From 79e820957771159993b67d80ec56d3a91e7e0456 Mon Sep 17 00:00:00 2001 From: jiaxiaoyu Date: Thu, 22 Oct 2026 13:37:37 +0800 Subject: [PATCH 1/6] [RISCV] Add native ZAMES and ZMASYNC support Add matrix and synchronization intrinsics, target extension types, register classes, instruction encodings, assembly and disassembly. Implement RV64 SelectionDAG lowering with pre-RA configuration materialization and post-RA expansion. Track matrix memory operands, validate intrinsic immediate ranges, and diagnose unsupported RV32 code generation. Add IR, MC, MIR and CodeGen coverage. Keep vector exp2, MXFP and XX8 conversion extensions out of this change. Validation: - Build LLVM, Clang and MLIR with RISCV and X86 backends. - Pass 25 patch-related test files and 11 smoke scenarios. Matrix copy/spill/reload and cross-function ABI support remain outside the supported scope. --- .../Driver/print-supported-extensions-riscv.c | 2 + llvm/include/llvm/CodeGen/ValueTypes.td | 9 + llvm/include/llvm/IR/IntrinsicsRISCV.td | 4 + .../llvm/IR/IntrinsicsRISCVZamesZmasync.td | 107 +++ llvm/lib/CodeGen/ValueTypes.cpp | 58 +- llvm/lib/IR/Type.cpp | 23 + .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 43 ++ .../RISCV/Disassembler/RISCVDisassembler.cpp | 48 ++ .../RISCV/MCTargetDesc/RISCVInstPrinter.cpp | 8 + .../RISCV/MCTargetDesc/RISCVInstPrinter.h | 2 + llvm/lib/Target/RISCV/RISCV.td | 1 + .../Target/RISCV/RISCVExpandPseudoPreEmit.cpp | 301 +++++++++ .../Target/RISCV/RISCVExpandPseudoPreRA.cpp | 490 ++++++++++++++ llvm/lib/Target/RISCV/RISCVFeatures.td | 21 + llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h | 30 + llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 83 +++ llvm/lib/Target/RISCV/RISCVInstrInfo.td | 4 + llvm/lib/Target/RISCV/RISCVInstrInfoZames.td | 308 +++++++++ .../RISCV/RISCVInstrInfoZamesPatterns.td | 595 +++++++++++++++++ .../Target/RISCV/RISCVRegisterInfoZames.td | 96 +++ llvm/test/Assembler/riscv-zames-intrinsics.ll | 130 ++++ llvm/test/CodeGen/RISCV/features-info.ll | 2 + .../CodeGen/RISCV/zames/rv32-unsupported.ll | 26 + .../CodeGen/RISCV/zames/zames-attributes.ll | 16 + .../test/CodeGen/RISCV/zames/zames-codegen.ll | 266 ++++++++ .../CodeGen/RISCV/zames/zames-fp-expand.mir | 54 ++ .../zames/zames-fp-types-negative-acc.ll | 21 + .../RISCV/zames/zames-fp-types-negative.ll | 22 + .../CodeGen/RISCV/zames/zames-fp-types.ll | 621 ++++++++++++++++++ .../zames/zames-immediate-range-invalid.ll | 55 ++ .../CodeGen/RISCV/zames/zames-isel-pseudos.ll | 115 ++++ .../RISCV/zames/zames-memory-operands.ll | 127 ++++ .../CodeGen/RISCV/zames/zames-no-feature.ll | 12 + .../RISCV/zames/zames-prera-cfg-pseudos.ll | 140 ++++ .../CodeGen/RISCV/zames/zmasync-codegen.ll | 20 + llvm/test/MC/RISCV/zames-invalid.s | 23 + llvm/test/MC/RISCV/zames-no-feature.s | 62 ++ llvm/test/MC/RISCV/zames.s | 147 +++++ llvm/test/MC/RISCV/zmasync-implies-zames.s | 11 + llvm/test/MC/RISCV/zmasync-invalid.s | 19 + llvm/test/MC/RISCV/zmasync-no-feature.s | 18 + llvm/test/MC/RISCV/zmasync.s | 22 + .../riscv-zames-intrinsics-invalid.ll | 140 ++++ 43 files changed, 4301 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td create mode 100644 llvm/lib/Target/RISCV/RISCVInstrInfoZames.td create mode 100644 llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td create mode 100644 llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td create mode 100644 llvm/test/Assembler/riscv-zames-intrinsics.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-attributes.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-codegen.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll create mode 100644 llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll create mode 100644 llvm/test/MC/RISCV/zames-invalid.s create mode 100644 llvm/test/MC/RISCV/zames-no-feature.s create mode 100644 llvm/test/MC/RISCV/zames.s create mode 100644 llvm/test/MC/RISCV/zmasync-implies-zames.s create mode 100644 llvm/test/MC/RISCV/zmasync-invalid.s create mode 100644 llvm/test/MC/RISCV/zmasync-no-feature.s create mode 100644 llvm/test/MC/RISCV/zmasync.s create mode 100644 llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c index 30c45aa6e4bdb3..7b526f8055d40a 100644 --- a/clang/test/Driver/print-supported-extensions-riscv.c +++ b/clang/test/Driver/print-supported-extensions-riscv.c @@ -35,6 +35,7 @@ // CHECK-NEXT: zihpm 2.0 'Zihpm' (Hardware Performance Counters) // CHECK-NEXT: zilsd 1.0 'Zilsd' (Load/Store Pair Instructions) // CHECK-NEXT: zimop 1.0 'Zimop' (May-Be-Operations) +// CHECK: zmasync{{ +}}1.0{{( +|$)}} // CHECK-NEXT: zmmul 1.0 'Zmmul' (Integer Multiplication) // CHECK-NEXT: za128rs 1.0 'Za128rs' (Reservation Set Size of at Most 128 Bytes) // CHECK-NEXT: za64rs 1.0 'Za64rs' (Reservation Set Size of at Most 64 Bytes) @@ -44,6 +45,7 @@ // CHECK-NEXT: zalasr 1.0 'Zalasr' (Load-Acquire and Store-Release Instructions) // CHECK-NEXT: zalrsc 1.0 'Zalrsc' (Load-Reserved/Store-Conditional) // CHECK-NEXT: zama16b 1.0 'Zama16b' (Atomic 16-byte misaligned loads, stores and AMOs) +// CHECK: zames{{ +}}1.0{{( +|$)}} // CHECK-NEXT: zawrs 1.0 'Zawrs' (Wait on Reservation Set) // CHECK-NEXT: zfa 1.0 'Zfa' (Additional Floating-Point) // CHECK-NEXT: zfbfmin 1.0 'Zfbfmin' (Scalar BF16 Converts) diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td index b5ac120f92509e..b20f67a68699fe 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.td +++ b/llvm/include/llvm/CodeGen/ValueTypes.td @@ -409,6 +409,15 @@ def amdgpuBufferStridedPointer : ValueType<192>; def aarch64mfp8 : ValueType<8>; // 8-bit value in FPR (AArch64) +def riscv_ame_tile_i8 : ValueType<0>; // RISC-V AME/ZAMES tile with i8 elements +def riscv_ame_tile_i32 : ValueType<0>; // RISC-V AME/ZAMES tile with i32 elements +def riscv_ame_sync : ValueType<0>; // RISC-V Zmasync sync register +def riscv_ame_tile_fp8e4 : ValueType<0>; // RISC-V ZAMES tile with fp8e4m3 elements +def riscv_ame_tile_fp8e5 : ValueType<0>; // RISC-V ZAMES tile with fp8e5m2 elements +def riscv_ame_tile_f16 : ValueType<0>; // RISC-V ZAMES tile with fp16 elements +def riscv_ame_tile_bf16 : ValueType<0>; // RISC-V ZAMES tile with bf16 elements +def riscv_ame_tile_f32 : ValueType<0>; // RISC-V ZAMES tile with fp32 elements + // CHERI capabilities. Pointer-like values that carry additional metadata // for enforcing safety guarantees on CHERI-enabled targets. def c64 : VTCheriCapability<64>; // 64-bit CHERI capability value diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 840b0568f942e6..28e7325e70d25d 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2498,3 +2498,7 @@ include "llvm/IR/IntrinsicsRISCVExt.td" include "llvm/IR/IntrinsicsRISCVBOSCExt.td" include "llvm/IR/IntrinsicsRISCVXTExt.td" include "llvm/IR/IntrinsicsRISCVBuckyballExt.td" + +// ZAMES/ZMASYNC extensions +//===----------------------------------------------------------------------===// +include "llvm/IR/IntrinsicsRISCVZamesZmasync.td" diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td b/llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td new file mode 100644 index 00000000000000..75c39337a6d703 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td @@ -0,0 +1,107 @@ +//===- IntrinsicsRISCVZamesZmasync.td - Zames/Zmasync intrinsics -*- tablegen -*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in { + class ZAMESSetCfgIntrinsic + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 8>]>; + class ZAMESGetCfgIntrinsic + : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 8>]>; + class ZAMESSetTileIntrinsic + : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 1024>]>; + class ZAMESSetTileRegIntrinsic + : Intrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>; + class ZAMESInitIntrinsic + : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; + + class ZAMESMZeroIntrinsic + : Intrinsic<[llvm_any_ty], [], [IntrNoMem]>; + class ZAMESLoadIntrinsic + : Intrinsic<[llvm_any_ty], [llvm_ptr_ty, llvm_i64_ty], + [NoCapture>, IntrReadMem], "", + [SDNPMemOperand]>; + class ZAMESStoreIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty, llvm_i64_ty], + [NoCapture>, IntrWriteMem], "", + [SDNPMemOperand]>; + class ZAMESWholeLoadIntrinsic + : Intrinsic<[llvm_any_ty], [llvm_ptr_ty], + [NoCapture>, IntrReadMem], "", + [SDNPMemOperand]>; + class ZAMESWholeStoreIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty], + [NoCapture>, IntrWriteMem], "", + [SDNPMemOperand]>; + class ZAMESMMAccIntrinsic + : Intrinsic<[llvm_any_ty], + [llvm_any_ty, llvm_any_ty, LLVMMatchType<0>], + [IntrNoMem]>; + + def int_riscv_zames_msetcfg : ZAMESSetCfgIntrinsic; + def int_riscv_zames_mgetcfg : ZAMESGetCfgIntrinsic; + def int_riscv_zames_msettilemi : ZAMESSetTileIntrinsic; + def int_riscv_zames_msettileni : ZAMESSetTileIntrinsic; + def int_riscv_zames_msettileki : ZAMESSetTileIntrinsic; + def int_riscv_zames_msettilem : ZAMESSetTileRegIntrinsic; + def int_riscv_zames_msettilen : ZAMESSetTileRegIntrinsic; + def int_riscv_zames_msettilek : ZAMESSetTileRegIntrinsic; + + def int_riscv_zames_minit : ZAMESInitIntrinsic; + + def int_riscv_zames_mzero : ZAMESMZeroIntrinsic; + def int_riscv_zames_mmacc : ZAMESMMAccIntrinsic; + + def int_riscv_zames_mla : ZAMESLoadIntrinsic; + def int_riscv_zames_mlb : ZAMESLoadIntrinsic; + def int_riscv_zames_mlc : ZAMESLoadIntrinsic; + def int_riscv_zames_mlat : ZAMESLoadIntrinsic; + def int_riscv_zames_mlbt : ZAMESLoadIntrinsic; + def int_riscv_zames_mlct : ZAMESLoadIntrinsic; + def int_riscv_zames_mla_whole : ZAMESWholeLoadIntrinsic; + def int_riscv_zames_mlb_whole : ZAMESWholeLoadIntrinsic; + def int_riscv_zames_mlc_whole : ZAMESWholeLoadIntrinsic; + + def int_riscv_zames_msa : ZAMESStoreIntrinsic; + def int_riscv_zames_msb : ZAMESStoreIntrinsic; + def int_riscv_zames_msc : ZAMESStoreIntrinsic; + def int_riscv_zames_msat : ZAMESStoreIntrinsic; + def int_riscv_zames_msbt : ZAMESStoreIntrinsic; + def int_riscv_zames_msct : ZAMESStoreIntrinsic; + def int_riscv_zames_msa_whole : ZAMESWholeStoreIntrinsic; + def int_riscv_zames_msb_whole : ZAMESWholeStoreIntrinsic; + def int_riscv_zames_msc_whole : ZAMESWholeStoreIntrinsic; + + class ZMASYNCSyncResetIntrinsic + : Intrinsic<[llvm_any_ty], [], [IntrNoMem, IntrHasSideEffects]>; + class ZMASYNCReleaseIntrinsic + : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], + [IntrNoMem, IntrHasSideEffects]>; + class ZMASYNCAcquireIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + class ZMASYNCFenceIntrinsic + : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; + + def int_riscv_zmasync_msyncregreset : ZMASYNCSyncResetIntrinsic; + def int_riscv_zmasync_mrelease : ZMASYNCReleaseIntrinsic; + def int_riscv_zmasync_macquire : ZMASYNCAcquireIntrinsic; + def int_riscv_zmasync_mfence : ZMASYNCFenceIntrinsic; +} diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp index 3bba70cd670016..db3d2e08be4659 100644 --- a/llvm/lib/CodeGen/ValueTypes.cpp +++ b/llvm/lib/CodeGen/ValueTypes.cpp @@ -158,7 +158,7 @@ TypeSize EVT::getExtendedSizeInBits() const { llvm_unreachable("Unrecognized extended type!"); } -/// getEVTString - This function returns value type as a string, e.g. "i32". +/// Print the value type, e.g. "i32". void EVT::print(raw_ostream &OS) const { switch (V.SimpleTy) { default: @@ -242,6 +242,30 @@ void EVT::print(raw_ostream &OS) const { case MVT::aarch64mfp8: OS << "aarch64mfp8"; return; + case MVT::riscv_ame_tile_i8: + OS << "riscv_ame_tile_i8"; + return; + case MVT::riscv_ame_tile_i32: + OS << "riscv_ame_tile_i32"; + return; + case MVT::riscv_ame_sync: + OS << "riscv_ame_sync"; + return; + case MVT::riscv_ame_tile_fp8e4: + OS << "riscv_ame_tile_fp8e4"; + return; + case MVT::riscv_ame_tile_fp8e5: + OS << "riscv_ame_tile_fp8e5"; + return; + case MVT::riscv_ame_tile_f16: + OS << "riscv_ame_tile_f16"; + return; + case MVT::riscv_ame_tile_bf16: + OS << "riscv_ame_tile_bf16"; + return; + case MVT::riscv_ame_tile_f32: + OS << "riscv_ame_tile_f32"; + return; } } @@ -267,6 +291,22 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const { return TargetExtType::get(Context, "aarch64.svcount"); case MVT::aarch64mfp8: return FixedVectorType::get(IntegerType::get(Context, 8), 1); + case MVT::riscv_ame_tile_i8: + return TargetExtType::get(Context, "riscv.ame.tile.int8"); + case MVT::riscv_ame_tile_i32: + return TargetExtType::get(Context, "riscv.ame.tile.int32"); + case MVT::riscv_ame_sync: + return TargetExtType::get(Context, "riscv.ame.sync"); + case MVT::riscv_ame_tile_fp8e4: + return TargetExtType::get(Context, "riscv.ame.tile.fp8e4"); + case MVT::riscv_ame_tile_fp8e5: + return TargetExtType::get(Context, "riscv.ame.tile.fp8e5"); + case MVT::riscv_ame_tile_f16: + return TargetExtType::get(Context, "riscv.ame.tile.fp16"); + case MVT::riscv_ame_tile_bf16: + return TargetExtType::get(Context, "riscv.ame.tile.bf16"); + case MVT::riscv_ame_tile_f32: + return TargetExtType::get(Context, "riscv.ame.tile.fp32"); case MVT::x86amx: return Type::getX86_AMXTy(Context); case MVT::i64x8: return IntegerType::get(Context, 512); case MVT::amdgpuBufferFatPointer: return IntegerType::get(Context, 160); @@ -322,6 +362,22 @@ MVT MVT::getVT(Type *Ty, bool HandleUnknown){ return MVT::getRISCVVectorTupleVT(Sz * NF, NF); } + if (TargetExtTy->getName() == "riscv.ame.tile.int8") + return MVT(MVT::riscv_ame_tile_i8); + if (TargetExtTy->getName() == "riscv.ame.tile.int32") + return MVT(MVT::riscv_ame_tile_i32); + if (TargetExtTy->getName() == "riscv.ame.sync") + return MVT(MVT::riscv_ame_sync); + if (TargetExtTy->getName() == "riscv.ame.tile.fp8e4") + return MVT(MVT::riscv_ame_tile_fp8e4); + if (TargetExtTy->getName() == "riscv.ame.tile.fp8e5") + return MVT(MVT::riscv_ame_tile_fp8e5); + if (TargetExtTy->getName() == "riscv.ame.tile.fp16") + return MVT(MVT::riscv_ame_tile_f16); + if (TargetExtTy->getName() == "riscv.ame.tile.bf16") + return MVT(MVT::riscv_ame_tile_bf16); + if (TargetExtTy->getName() == "riscv.ame.tile.fp32") + return MVT(MVT::riscv_ame_tile_f32); if (HandleUnknown) return MVT(MVT::Other); llvm_unreachable("Unknown target ext type!"); diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 7af638499cd868..b0cbe17f772c59 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -981,6 +981,17 @@ Expected TargetExtType::checkParams(TargetExtType *TTy) { "target extension type riscv.vector.tuple should have one " "type parameter and one integer parameter"); + if ((TTy->Name == "riscv.ame.tile.int8" || + TTy->Name == "riscv.ame.tile.int32" || + TTy->Name == "riscv.ame.tile.fp8e4" || + TTy->Name == "riscv.ame.tile.fp8e5" || + TTy->Name == "riscv.ame.tile.fp16" || + TTy->Name == "riscv.ame.tile.bf16" || + TTy->Name == "riscv.ame.tile.fp32" || TTy->Name == "riscv.ame.sync") && + (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 0)) + return createStringError("target extension type " + TTy->Name + + " should have no parameters"); + // Opaque types in the AMDGPU name space. if (TTy->Name == "amdgcn.named.barrier" && (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 1)) { @@ -1077,6 +1088,18 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); } + if (Name == "riscv.ame.tile.int8" || Name == "riscv.ame.tile.int32" || + Name == "riscv.ame.tile.fp8e4" || Name == "riscv.ame.tile.fp8e5" || + Name == "riscv.ame.tile.fp16" || Name == "riscv.ame.tile.bf16" || + Name == "riscv.ame.tile.fp32") + return TargetTypeInfo(FixedVectorType::get(Type::getInt8Ty(C), 16), + TargetExtType::CanBeLocal, + TargetExtType::HasZeroInit); + + if (Name == "riscv.ame.sync") + return TargetTypeInfo(Type::getInt64Ty(C), TargetExtType::CanBeLocal, + TargetExtType::HasZeroInit); + // DirectX resources if (Name == "dx.Padding") return TargetTypeInfo( diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 0cb8e2800b29cf..172dc5c9cad1a4 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -240,6 +240,7 @@ class RISCVAsmParser : public MCTargetAsmParser { } ParseStatus parseRegReg(OperandVector &Operands); + ParseStatus parseMCFGSel(OperandVector &Operands); ParseStatus parseXSfmmVType(OperandVector &Operands); ParseStatus parseZcmpStackAdj(OperandVector &Operands, bool ExpectNegative = false); @@ -761,6 +762,8 @@ struct RISCVOperand final : public MCParsedAsmOperand { return IsConstantImm && isUInt(Imm); } + bool isMCFGSel() const { return isUImm<3>(); } + template bool isUImmShifted() const { int64_t Imm; if (!isExpr()) @@ -1428,6 +1431,19 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp, getRISCVMCRegisterClass(RISCV::FPR64CRegClassID).contains(Reg); bool IsRegVR = getRISCVMCRegisterClass(RISCV::VRRegClassID).contains(Reg); + // ZAMES shares assembly names with BOSC AME, but has distinct register + // records and accumulator encodings. Coerce only for a ZAMES operand class. + if ((Kind == MCK_ZamesTileReg || Kind == MCK_ZamesTileOrAccReg) && + Reg >= RISCV::TR0 && Reg <= RISCV::TR3) { + Op.Reg.Reg = RISCV::ZAMES_TR0 + (Reg - RISCV::TR0); + return Match_Success; + } + if ((Kind == MCK_ZamesAccReg || Kind == MCK_ZamesTileOrAccReg) && + Reg >= RISCV::ACC0 && Reg <= RISCV::ACC3) { + Op.Reg.Reg = RISCV::ZAMES_ACC0 + (Reg - RISCV::ACC0); + return Match_Success; + } + if (Op.isGPR() && Kind == MCK_YGPR) { // GPR and capability GPR use the same register names, convert if required. Op.Reg.Reg = convertGPRToYGPR(Reg); @@ -1887,6 +1903,12 @@ MCRegister RISCVAsmParser::matchRegisterNameHelper(StringRef Name) const { static_assert(RISCV::F0_D < RISCV::F0_H, "FPR matching must be updated"); static_assert(RISCV::F0_D < RISCV::F0_F, "FPR matching must be updated"); static_assert(RISCV::F0_D < RISCV::F0_Q, "FPR matching must be updated"); + // Shared matrix names initially resolve to BOSC AME; operand validation + // converts them to ZAMES registers only when the instruction requires it. + static_assert(RISCV::TR0 < RISCV::ZAMES_TR0, + "Matrix register matching must be updated"); + static_assert(RISCV::ACC0 < RISCV::ZAMES_ACC0, + "Matrix register matching must be updated"); if (!Reg) Reg = MatchRegisterAltName(Name); if (isRVE() && Reg >= RISCV::X16 && Reg <= RISCV::X31) @@ -2573,6 +2595,27 @@ bool RISCVAsmParser::generateVTypeError(SMLoc ErrorLoc) { "e[8|8alt|16|16alt|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]"); } +ParseStatus RISCVAsmParser::parseMCFGSel(OperandVector &Operands) { + const AsmToken &Tok = getLexer().getTok(); + if (!Tok.is(AsmToken::Identifier)) + return ParseStatus::NoMatch; + + StringRef Name = Tok.getIdentifier(); + if (!Name.consume_front("mcfg")) + return ParseStatus::NoMatch; + + unsigned Cfg = 0; + if (Name.consumeInteger(10, Cfg) || !Name.empty() || Cfg > 7) + return ParseStatus::NoMatch; + + SMLoc S = getLoc(); + SMLoc E = Tok.getEndLoc(); + Operands.push_back(RISCVOperand::createExpr( + MCConstantExpr::create(Cfg, getContext()), S, E, isRV64())); + Lex(); + return ParseStatus::Success; +} + ParseStatus RISCVAsmParser::parseXSfmmVType(OperandVector &Operands) { SMLoc S = getLoc(); diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp index 3853cefd59517b..1fb3cbdc3e6277 100644 --- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp +++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp @@ -318,6 +318,54 @@ static DecodeStatus DecodeYBNDSWImm(MCInst &Inst, uint64_t Imm, int64_t Address, return MCDisassembler::Success; } +static DecodeStatus +DecodeZamesTileRegRegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo >= 4) + return MCDisassembler::Fail; + + MCRegister Reg = RISCV::ZAMES_TR0 + RegNo; + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + +static DecodeStatus +DecodeZamesAccRegRegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo < 4 || RegNo >= 8) + return MCDisassembler::Fail; + + MCRegister Reg = RISCV::ZAMES_ACC0 + (RegNo - 4); + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + +// Union of ZamesTileReg (0-3) and ZamesAccReg (4-7) for real instructions whose +// md field accepts either register file (e.g. mzero). +static DecodeStatus +DecodeZamesTileOrAccRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo >= 8) + return MCDisassembler::Fail; + + MCRegister Reg = + RegNo < 4 ? RISCV::ZAMES_TR0 + RegNo : RISCV::ZAMES_ACC0 + (RegNo - 4); + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + +static DecodeStatus DecodeSyncRRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo >= 16) + return MCDisassembler::Fail; + + MCRegister Reg = RISCV::SYNC0 + RegNo; + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + static DecodeStatus decodeVMaskReg(MCInst &Inst, uint32_t RegNo, uint64_t Address, const MCDisassembler *Decoder) { diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp index 5d941ff8205ae6..8f5a467711dbe2 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp @@ -259,6 +259,14 @@ void RISCVInstPrinter::printXSfmmVType(const MCInst *MI, unsigned OpNo, O << ", w" << Widen; } +void RISCVInstPrinter::printMCFGSel(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, + raw_ostream &O) { + unsigned Imm = MI->getOperand(OpNo).getImm(); + assert(Imm < 8 && "Invalid MCFG selector immediate"); + markup(O, Markup::Register) << "mcfg" << Imm; +} + // Print a Zcmp RList. If we are printing architectural register names rather // than ABI register names, we need to print "{x1, x8-x9, x18-x27}" for all // registers. Otherwise, we print "{ra, s0-s11}". diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h index 324e131e50089c..66031cb37da763 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h @@ -54,6 +54,8 @@ class RISCVInstPrinter : public MCInstPrinter { raw_ostream &O); void printXSfmmVType(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); + void printMCFGSel(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, + raw_ostream &O); void printVMaskReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); void printMatrixReg(const MCInst *MI, unsigned OpNo, diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index 786120abb207d7..08fc2bd4cf2afc 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -31,6 +31,7 @@ include "RISCVSystemOperands.td" //===----------------------------------------------------------------------===// include "RISCVRegisterInfo.td" +include "RISCVRegisterInfoZames.td" include "RISCVSchedule.td" include "RISCVCallingConv.td" include "RISCVInstrInfo.td" diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp index fb96ae371c73e5..d5c9c71bfc70c1 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/Support/ErrorHandling.h" using namespace llvm; @@ -62,6 +63,9 @@ class RISCVExpandPseudoPreEmitImpl final : public RISCVExpandPseudoImplBase { bool expandBOSCAMEOrderedTile(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; + + bool expandZamesMatrixPseudo(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; }; class RISCVExpandPseudoPreEmitLegacy : public MachineFunctionPass { @@ -177,6 +181,68 @@ bool RISCVExpandPseudoPreEmitImpl::expandMI( return expandPseudoReadVLENBViaVSETVLIX0(MBB, MBBI); case RISCV::PseudoClearFPR64: return expandPseudoClearFPR64(MBB, MBBI); + case RISCV::PSEUDO_ZAMES_MLA_A_I8: + case RISCV::PSEUDO_ZAMES_MLB_B_I8: + case RISCV::PSEUDO_ZAMES_MLBT_B_I8: + case RISCV::PSEUDO_ZAMES_MLAT_A_I8: + case RISCV::PSEUDO_ZAMES_MLC_C_I32: + case RISCV::PSEUDO_ZAMES_MLCT_C_I32: + case RISCV::PSEUDO_ZAMES_MZERO_C_I32: + case RISCV::PSEUDO_ZAMES_MMACC_I8_I32: + case RISCV::PSEUDO_ZAMES_MSC_C_I32: + case RISCV::PSEUDO_ZAMES_MSA_A_I8: + case RISCV::PSEUDO_ZAMES_MSAT_A_I8: + case RISCV::PSEUDO_ZAMES_MSB_B_I8: + case RISCV::PSEUDO_ZAMES_MSBT_B_I8: + case RISCV::PSEUDO_ZAMES_MSCT_C_I32: + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MLA_A_F16: + case RISCV::PSEUDO_ZAMES_MLA_A_BF16: + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MLAT_A_F16: + case RISCV::PSEUDO_ZAMES_MLAT_A_BF16: + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MLB_B_F16: + case RISCV::PSEUDO_ZAMES_MLB_B_BF16: + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MLBT_B_F16: + case RISCV::PSEUDO_ZAMES_MLBT_B_BF16: + case RISCV::PSEUDO_ZAMES_MLC_C_F32: + case RISCV::PSEUDO_ZAMES_MLCT_C_F32: + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MSA_A_F16: + case RISCV::PSEUDO_ZAMES_MSA_A_BF16: + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MSAT_A_F16: + case RISCV::PSEUDO_ZAMES_MSAT_A_BF16: + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MSB_B_F16: + case RISCV::PSEUDO_ZAMES_MSB_B_BF16: + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MSBT_B_F16: + case RISCV::PSEUDO_ZAMES_MSBT_B_BF16: + case RISCV::PSEUDO_ZAMES_MSC_C_F32: + case RISCV::PSEUDO_ZAMES_MSCT_C_F32: + case RISCV::PSEUDO_ZAMES_MZERO_T_I8: + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4: + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5: + case RISCV::PSEUDO_ZAMES_MZERO_T_F16: + case RISCV::PSEUDO_ZAMES_MZERO_T_BF16: + case RISCV::PSEUDO_ZAMES_MZERO_C_F32: + case RISCV::PSEUDO_ZAMES_MMACC_I8_F32: + case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32: + case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32: + case RISCV::PSEUDO_ZAMES_MMACC_F16_F32: + case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32: + return expandZamesMatrixPseudo(MBB, MBBI); } return false; @@ -274,6 +340,241 @@ bool RISCVExpandPseudoPreEmitImpl::expandBOSCAMEOrderedTile( return true; } +static unsigned getZamesMcfgSelector(Register Reg) { + switch (Reg) { + case RISCV::ZAMES_TR0: + return 0; + case RISCV::ZAMES_TR1: + return 1; + case RISCV::ZAMES_TR2: + return 2; + case RISCV::ZAMES_TR3: + return 3; + case RISCV::ZAMES_ACC0: + return 4; + case RISCV::ZAMES_ACC1: + return 5; + case RISCV::ZAMES_ACC2: + return 6; + case RISCV::ZAMES_ACC3: + return 7; + default: + report_fatal_error( + "ZAMES msetcfg pseudo expansion requires tr0-tr3 or acc0-acc3"); + } +} + +static void emitZamesConfig(const RISCVInstrInfo *TII, MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, DebugLoc DL, + Register MatrixReg, Register ConfigReg) { + unsigned Selector = getZamesMcfgSelector(MatrixReg); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSETCFG)) + .addImm(Selector) + .addReg(ConfigReg); +} + +bool RISCVExpandPseudoPreEmitImpl::expandZamesMatrixPseudo( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + unsigned Opc = MI.getOpcode(); + + switch (Opc) { + default: + llvm_unreachable("Unexpected ZAMES pseudo opcode"); + case RISCV::PSEUDO_ZAMES_MLA_A_I8: + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MLA_A_F16: + case RISCV::PSEUDO_ZAMES_MLA_A_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLA)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLB_B_I8: + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MLB_B_F16: + case RISCV::PSEUDO_ZAMES_MLB_B_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLB)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLBT_B_I8: + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MLBT_B_F16: + case RISCV::PSEUDO_ZAMES_MLBT_B_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLBT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLAT_A_I8: + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MLAT_A_F16: + case RISCV::PSEUDO_ZAMES_MLAT_A_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLAT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLC_C_I32: + case RISCV::PSEUDO_ZAMES_MLC_C_F32: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLC)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLCT_C_I32: + case RISCV::PSEUDO_ZAMES_MLCT_C_F32: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLCT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MZERO_C_I32: + case RISCV::PSEUDO_ZAMES_MZERO_T_I8: + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4: + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5: + case RISCV::PSEUDO_ZAMES_MZERO_T_F16: + case RISCV::PSEUDO_ZAMES_MZERO_T_BF16: + case RISCV::PSEUDO_ZAMES_MZERO_C_F32: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(1).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MZERO)).add(MI.getOperand(0)); + break; + } + case RISCV::PSEUDO_ZAMES_MMACC_I8_I32: + case RISCV::PSEUDO_ZAMES_MMACC_I8_F32: + case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32: + case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32: + case RISCV::PSEUDO_ZAMES_MMACC_F16_F32: + case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(1).getReg(), + MI.getOperand(4).getReg()); + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(2).getReg(), + MI.getOperand(5).getReg()); + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(3).getReg(), + MI.getOperand(6).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MMACC)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(3)); + break; + } + case RISCV::PSEUDO_ZAMES_MSC_C_I32: + case RISCV::PSEUDO_ZAMES_MSC_C_F32: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSC)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSA_A_I8: + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MSA_A_F16: + case RISCV::PSEUDO_ZAMES_MSA_A_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSA)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSAT_A_I8: + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4: + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5: + case RISCV::PSEUDO_ZAMES_MSAT_A_F16: + case RISCV::PSEUDO_ZAMES_MSAT_A_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSAT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSB_B_I8: + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MSB_B_F16: + case RISCV::PSEUDO_ZAMES_MSB_B_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSB)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSBT_B_I8: + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4: + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5: + case RISCV::PSEUDO_ZAMES_MSBT_B_F16: + case RISCV::PSEUDO_ZAMES_MSBT_B_BF16: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSBT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSCT_C_I32: + case RISCV::PSEUDO_ZAMES_MSCT_C_F32: { + emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSCT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + } + + MI.eraseFromParent(); + return true; +} + bool RISCVExpandPseudoPreEmitImpl::expandCCOp( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NextMBBI) const { diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp index d64605fd27feb0..a1ada1957c0ff7 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp @@ -17,7 +17,9 @@ #include "RISCVSubtarget.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/MC/MCContext.h" +#include "llvm/Support/ErrorHandling.h" using namespace llvm; @@ -48,6 +50,9 @@ class RISCVExpandPseudoPreRAImpl final : public RISCVExpandPseudoImplBase { bool expandLoadTLSDescAddress(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; + + bool expandZamesConfigPseudo(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; }; class RISCVExpandPseudoPreRALegacy : public MachineFunctionPass { @@ -90,11 +95,496 @@ bool RISCVExpandPseudoPreRAImpl::expandMI( return expandLoadTLSGDAddress(MBB, MBBI); case RISCV::PseudoLA_TLSDESC: return expandLoadTLSDescAddress(MBB, MBBI); + case RISCV::PSEUDO_ZAMES_MLA_A_I8_PRE: + case RISCV::PSEUDO_ZAMES_MLB_B_I8_PRE: + case RISCV::PSEUDO_ZAMES_MLBT_B_I8_PRE: + case RISCV::PSEUDO_ZAMES_MLAT_A_I8_PRE: + case RISCV::PSEUDO_ZAMES_MLC_C_I32_PRE: + case RISCV::PSEUDO_ZAMES_MLCT_C_I32_PRE: + case RISCV::PSEUDO_ZAMES_MZERO_C_I32_PRE: + case RISCV::PSEUDO_ZAMES_MMACC_I8_I32_PRE: + case RISCV::PSEUDO_ZAMES_MSC_C_I32_PRE: + case RISCV::PSEUDO_ZAMES_MSA_A_I8_PRE: + case RISCV::PSEUDO_ZAMES_MSAT_A_I8_PRE: + case RISCV::PSEUDO_ZAMES_MSB_B_I8_PRE: + case RISCV::PSEUDO_ZAMES_MSBT_B_I8_PRE: + case RISCV::PSEUDO_ZAMES_MSCT_C_I32_PRE: + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MLA_A_F16_PRE: + case RISCV::PSEUDO_ZAMES_MLA_A_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MLAT_A_F16_PRE: + case RISCV::PSEUDO_ZAMES_MLAT_A_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MLB_B_F16_PRE: + case RISCV::PSEUDO_ZAMES_MLB_B_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MLBT_B_F16_PRE: + case RISCV::PSEUDO_ZAMES_MLBT_B_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MLC_C_F32_PRE: + case RISCV::PSEUDO_ZAMES_MLCT_C_F32_PRE: + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MSA_A_F16_PRE: + case RISCV::PSEUDO_ZAMES_MSA_A_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MSAT_A_F16_PRE: + case RISCV::PSEUDO_ZAMES_MSAT_A_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MSB_B_F16_PRE: + case RISCV::PSEUDO_ZAMES_MSB_B_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MSBT_B_F16_PRE: + case RISCV::PSEUDO_ZAMES_MSBT_B_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MSC_C_F32_PRE: + case RISCV::PSEUDO_ZAMES_MSCT_C_F32_PRE: + case RISCV::PSEUDO_ZAMES_MZERO_T_I8_PRE: + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4_PRE: + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5_PRE: + case RISCV::PSEUDO_ZAMES_MZERO_T_F16_PRE: + case RISCV::PSEUDO_ZAMES_MZERO_T_BF16_PRE: + case RISCV::PSEUDO_ZAMES_MZERO_C_F32_PRE: + case RISCV::PSEUDO_ZAMES_MMACC_I8_F32_PRE: + case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE: + case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE: + case RISCV::PSEUDO_ZAMES_MMACC_F16_F32_PRE: + case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32_PRE: + return expandZamesConfigPseudo(MBB, MBBI); } return false; } +static constexpr int64_t ZamesCfgI8 = 0x02; +static constexpr int64_t ZamesCfgI32 = 0x04; +static constexpr int64_t ZamesCfgF8E5 = 0x07; +static constexpr int64_t ZamesCfgF8E4 = 0x08; +static constexpr int64_t ZamesCfgF16 = 0x09; +static constexpr int64_t ZamesCfgBF16 = 0x0a; +static constexpr int64_t ZamesCfgF32 = 0x0c; + +static Register createZamesConfigPayload(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + DebugLoc DL, int64_t Payload) { + MachineFunction *MF = MBB.getParent(); + Register ConfigReg = + MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), ConfigReg) + .addReg(RISCV::X0) + .addImm(Payload); + return ConfigReg; +} + +// Expand a single-config _PRE pseudo: materialize the payload GPR, rebuild +// the second-level pseudo with all original operands plus the payload +// register, and carry the memory operands over. +static void expandZamesPreRASingleConfig(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineInstr &MI, unsigned NewOpc, + int64_t Payload) { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, MI.getDebugLoc(), Payload); + auto MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc)); + for (const MachineOperand &MO : MI.operands()) + MIB.add(MO); + MIB.addReg(ConfigReg); + MIB.cloneMemRefs(MI); +} + +// Expand an mmacc _PRE pseudo: materialize the accumulator and matrix +// payload GPRs and rebuild the second-level pseudo. The B and A matrices +// always share the element type in every supported combination, so one +// payload register feeds both config operands. +static void expandZamesPreRAMMAcc(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineInstr &MI, unsigned NewOpc, + int64_t AccPayload, int64_t MatPayload) { + DebugLoc DL = MI.getDebugLoc(); + Register AccConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, AccPayload); + Register MatConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, MatPayload); + BuildMI(MBB, MBBI, DL, TII->get(NewOpc)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(3)) + .addReg(AccConfigReg) + .addReg(MatConfigReg) + .addReg(MatConfigReg); +} + +bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + unsigned Opc = MI.getOpcode(); + + switch (Opc) { + default: + llvm_unreachable("Unexpected ZAMES pre-RA pseudo opcode"); + case RISCV::PSEUDO_ZAMES_MLA_A_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLA_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLB_B_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLB_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLBT_B_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLBT_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLAT_A_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLAT_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLC_C_I32_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLC_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLCT_C_I32_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLCT_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MZERO_C_I32_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MZERO_C_I32)) + .add(MI.getOperand(0)) + .addReg(ConfigReg); + break; + } + case RISCV::PSEUDO_ZAMES_MMACC_I8_I32_PRE: { + Register AccConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); + Register MatConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MMACC_I8_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(3)) + .addReg(AccConfigReg) + .addReg(MatConfigReg) + .addReg(MatConfigReg); + break; + } + case RISCV::PSEUDO_ZAMES_MSC_C_I32_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSC_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSA_A_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSA_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSAT_A_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSAT_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSB_B_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSB_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSBT_B_I8_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSBT_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MSCT_C_I32_PRE: { + Register ConfigReg = + createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSCT_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLA_A_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLA_A_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MLA_A_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLA_A_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MLA_A_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLA_A_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MLAT_A_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLAT_A_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MLAT_A_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLAT_A_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLB_B_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLB_B_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MLB_B_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLB_B_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MLB_B_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLB_B_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MLBT_B_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLBT_B_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MLBT_B_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLBT_B_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MLC_C_F32_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLC_C_F32, ZamesCfgF32); + break; + case RISCV::PSEUDO_ZAMES_MLCT_C_F32_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MLCT_C_F32, ZamesCfgF32); + break; + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSA_A_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSA_A_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MSA_A_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSA_A_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MSA_A_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSA_A_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MSAT_A_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSAT_A_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MSAT_A_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSAT_A_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSB_B_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSB_B_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MSB_B_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSB_B_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MSB_B_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSB_B_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MSBT_B_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSBT_B_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MSBT_B_BF16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSBT_B_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MSC_C_F32_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSC_C_F32, ZamesCfgF32); + break; + case RISCV::PSEUDO_ZAMES_MSCT_C_F32_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MSCT_C_F32, ZamesCfgF32); + break; + case RISCV::PSEUDO_ZAMES_MZERO_T_I8_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MZERO_T_I8, ZamesCfgI8); + break; + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4, ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5, ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MZERO_T_F16_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MZERO_T_F16, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MZERO_T_BF16_PRE: + expandZamesPreRASingleConfig( + TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MZERO_T_BF16, ZamesCfgBF16); + break; + case RISCV::PSEUDO_ZAMES_MZERO_C_F32_PRE: + expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MZERO_C_F32, ZamesCfgF32); + break; + case RISCV::PSEUDO_ZAMES_MMACC_I8_F32_PRE: + expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MMACC_I8_F32, + ZamesCfgF32, ZamesCfgI8); + break; + case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE: + expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32, ZamesCfgF32, + ZamesCfgF8E4); + break; + case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE: + expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32, ZamesCfgF32, + ZamesCfgF8E5); + break; + case RISCV::PSEUDO_ZAMES_MMACC_F16_F32_PRE: + expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MMACC_F16_F32, + ZamesCfgF32, ZamesCfgF16); + break; + case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32_PRE: + expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_ZAMES_MMACC_BF16_F32, ZamesCfgF32, + ZamesCfgBF16); + break; + } + + MI.eraseFromParent(); + return true; +} + bool RISCVExpandPseudoPreRAImpl::expandAuipcInstPair( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned FlagsHi, unsigned SecondOpcode) const { diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index c4c5115a973028..91a40b85072bc8 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1008,6 +1008,27 @@ def HasStdExtZvfbdota32f AssemblerPredicate<(all_of FeatureStdExtZvfbdota32f), "'Zvfbdota32f' (FP32 batched dot-product extension)">; +// Buddy matrix extensions + +def FeatureStdExtZames + : RISCVExtension<1, 0, "Buddy simplified matrix extension", [], + "HasZames">; + +def FeatureStdExtZmasync + : RISCVExtension<1, 0, "Buddy matrix async sync extension", + [FeatureStdExtZames], + "HasZmasync">; + +def HasStdExtZames + : Predicate<"Subtarget->hasZames()">, + AssemblerPredicate<(all_of FeatureStdExtZames), + "'ZAMES' (simplified matrix extension)">; + +def HasStdExtZmasync + : Predicate<"Subtarget->hasZmasync()">, + AssemblerPredicate<(all_of FeatureStdExtZmasync), + "'ZMASYNC' (matrix async sync extension)">; + // Vector instruction predicates def HasVInstructions : Predicate<"Subtarget->hasVInstructions()">, diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h index 7ee67ae085e22d..0c34439597f6a4 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -16,6 +16,9 @@ #include "RISCV.h" #include "RISCVTargetMachine.h" #include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/KnownBits.h" // RISC-V specific code to select RISC-V machine instructions for @@ -33,6 +36,33 @@ class RISCVDAGToDAGISel : public SelectionDAGISel { bool runOnMachineFunction(MachineFunction &MF) override { Subtarget = &MF.getSubtarget(); + // ZAMES/ZMASYNC intrinsics use i64 scalar interfaces and their + // SelectionDAG patterns and pseudos are constrained to RV64. Diagnose + // them before instruction selection on RV32 instead of failing with an + // uncontrolled "Cannot select" or a crash. + if (!Subtarget->is64Bit()) { + Function &F = MF.getFunction(); + for (Instruction &I : instructions(F)) { + auto *II = dyn_cast(&I); + if (!II) + continue; + StringRef Name = Intrinsic::getBaseName(II->getIntrinsicID()); + if (Name.starts_with("llvm.riscv.zames.") || + Name.starts_with("llvm.riscv.zmasync.")) { + F.getContext().diagnose(DiagnosticInfoUnsupported( + F, "ZAMES/ZMASYNC code generation requires RV64", + I.getDebugLoc())); + // Leave a minimal, valid function body so later passes do not + // trip over the unselected machine function; compilation still + // fails with the error emitted above. + MachineBasicBlock *Entry = MF.CreateMachineBasicBlock(); + MF.push_back(Entry); + BuildMI(Entry, DebugLoc(), + Subtarget->getInstrInfo()->get(RISCV::PseudoRET)); + return true; + } + } + } return SelectionDAGISel::runOnMachineFunction(MF); } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 189ae725cf5738..db8def4774dd8a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -196,6 +196,18 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, addRegisterClass(VT, &RISCV::MatrixRegRegClass); } + if (Subtarget.hasZames()) { + addRegisterClass(MVT::riscv_ame_tile_i8, &RISCV::ZamesTileRegRegClass); + addRegisterClass(MVT::riscv_ame_tile_i32, &RISCV::ZamesAccRegRegClass); + addRegisterClass(MVT::riscv_ame_tile_fp8e4, &RISCV::ZamesTileRegRegClass); + addRegisterClass(MVT::riscv_ame_tile_fp8e5, &RISCV::ZamesTileRegRegClass); + addRegisterClass(MVT::riscv_ame_tile_f16, &RISCV::ZamesTileRegRegClass); + addRegisterClass(MVT::riscv_ame_tile_bf16, &RISCV::ZamesTileRegRegClass); + addRegisterClass(MVT::riscv_ame_tile_f32, &RISCV::ZamesAccRegRegClass); + } + if (Subtarget.hasZmasync()) + addRegisterClass(MVT::riscv_ame_sync, &RISCV::SyncRRegClass); + static const MVT::SimpleValueType BoolVecVTs[] = { MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1, MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1}; @@ -2276,6 +2288,41 @@ void RISCVTargetLowering::getTgtMemIntrinsic( Infos.push_back(Info); }; + auto GetZamesMemoryVT = [&](bool IsStore) { + Type *TileTy = IsStore ? I.getArgOperand(0)->getType() : I.getType(); + auto *TargetTy = dyn_cast(TileTy); + if (!TargetTy) + return MVT(MVT::Other); + return StringSwitch(TargetTy->getName()) + .Cases({"riscv.ame.tile.int8", "riscv.ame.tile.fp8e4", + "riscv.ame.tile.fp8e5"}, + MVT::i8) + .Cases({"riscv.ame.tile.fp16", "riscv.ame.tile.bf16"}, MVT::i16) + .Cases({"riscv.ame.tile.int32", "riscv.ame.tile.fp32"}, MVT::i32) + .Default(MVT::Other); + }; + + auto SetZamesMemoryInfo = [&](unsigned PtrOp, bool IsStore, bool UsePtrVal) { + Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN; + Info.memVT = GetZamesMemoryVT(IsStore); + if (Info.memVT == MVT::Other) + return; + + const Value *Ptr = I.getArgOperand(PtrOp); + if (UsePtrVal) + Info.ptrVal = Ptr; + else + Info.fallbackAddressSpace = Ptr->getType()->getPointerAddressSpace(); + + Info.offset = 0; + Info.size = MemoryLocation::UnknownSize; + if (MaybeAlign Alignment = I.getParamAlign(PtrOp)) + Info.align = Alignment; + Info.flags |= + IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad; + Infos.push_back(Info); + }; + if (I.hasMetadata(LLVMContext::MD_nontemporal)) Info.flags |= MachineMemOperand::MONonTemporal; @@ -2604,6 +2651,42 @@ void RISCVTargetLowering::getTgtMemIntrinsic( Info.flags |= MachineMemOperand::MOStore; Infos.push_back(Info); return; + case Intrinsic::riscv_zames_mla: + case Intrinsic::riscv_zames_mlat: + case Intrinsic::riscv_zames_mlb: + case Intrinsic::riscv_zames_mlbt: + SetZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_zames_mlc: + case Intrinsic::riscv_zames_mlct: + SetZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_zames_msa: + case Intrinsic::riscv_zames_msat: + case Intrinsic::riscv_zames_msb: + case Intrinsic::riscv_zames_msbt: + SetZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_zames_msc: + case Intrinsic::riscv_zames_msct: + SetZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_zames_mla_whole: + case Intrinsic::riscv_zames_mlb_whole: + case Intrinsic::riscv_zames_mlc_whole: + SetZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ true); + return; + case Intrinsic::riscv_zames_msa_whole: + case Intrinsic::riscv_zames_msb_whole: + case Intrinsic::riscv_zames_msc_whole: + SetZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ true); + return; } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index ba9331d368a235..d641d2138fdfc0 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -2528,6 +2528,10 @@ include "RISCVInstrInfoSFB.td" // CHERI include "RISCVInstrInfoY.td" +// ZAMES/ZMASYNC matrix extensions +include "RISCVInstrInfoZames.td" +include "RISCVInstrInfoZamesPatterns.td" + //===----------------------------------------------------------------------===// // Vendor extensions //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZames.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZames.td new file mode 100644 index 00000000000000..32f44ae3ce4228 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZames.td @@ -0,0 +1,308 @@ +//===-- RISCVInstrInfoZames.td - ZAMES/ZMASYNC instructions -*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC real instruction definitions (MC layer). Codegen-only +// pseudos and SelectionDAG patterns are intentionally not part of this file. +// +//===----------------------------------------------------------------------===// + +def zames_uimm10 : RISCVUImmLeafOp<10>; + +// mcfg0..mcfg7 select the hardware matrix configuration register configured by +// msetcfg/mgetcfg. The value is encoded as a 3-bit immediate. +def MCFGSelAsmOperand : AsmOperandClass { + let Name = "MCFGSel"; + let RenderMethod = "addImmOperands"; + let ParserMethod = "parseMCFGSel"; +} + +def mcfgsel : RISCVUImmLeafOp<3> { + let ParserMatchClass = MCFGSelAsmOperand; + let PrintMethod = "printMCFGSel"; +} + +class ZAMESInstConfigImm func4, string opcodestr> + : RVInst<(outs), (ins zames_uimm10:$imm), opcodestr, "$imm", [], + InstFormatI> { + bits<10> imm; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = 0b0; + let Inst{24-15} = imm; + let Inst{14-12} = 0b000; + let Inst{11-7} = 0b00000; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstConfigReg func4, string opcodestr> + : RVInst<(outs), (ins GPR:$rs1), opcodestr, "$rs1", [], + InstFormatI> { + bits<5> rs1; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = 0b1; + let Inst{24-20} = 0b00000; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; + let Inst{11-7} = 0b00000; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstSetCfg + : RVInst<(outs), (ins mcfgsel:$cfg, GPR:$rs1), opcodestr, "$cfg, $rs1", + [], InstFormatI> { + bits<3> cfg; + bits<5> rs1; + + let Inst{31-28} = 0b0100; + let Inst{27-26} = 0b00; + let Inst{25} = 0b1; + let Inst{24-20} = 0b00000; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; + let Inst{11-10} = 0b00; + let Inst{9-7} = cfg; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstGetCfg + : RVInst<(outs GPR:$rd), (ins mcfgsel:$cfg), opcodestr, "$rd, $cfg", + [], InstFormatI> { + bits<5> rd; + bits<3> cfg; + + let Inst{31-28} = 0b0101; + let Inst{27-26} = 0b00; + let Inst{25} = 0b0; + let Inst{24-20} = 0b00000; + let Inst{19-18} = 0b00; + let Inst{17-15} = cfg; + let Inst{14-12} = 0b000; + let Inst{11-7} = rd; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstInit + : RVInst<(outs), (ins), opcodestr, "", [], InstFormatI> { + let Inst{31-7} = 0; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstLoadStore func4, bit ls, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + bits<5> rs1; + bits<5> rs2; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = ls; + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b001; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstWholeLoadStore func4, bit ls, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + bits<5> rs1; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = ls; + let Inst{24-20} = 0b00000; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b001; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstMMAcc func4, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + bits<5> ms1; + bits<5> ms2; + + let Inst{31-28} = func4; + let Inst{27-25} = 0b000; + let Inst{24-20} = ms2; + let Inst{19-15} = ms1; + let Inst{14-12} = 0b010; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class ZAMESInstMisc func4, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + + let Inst{31-28} = func4; + let Inst{27-15} = 0b0000000000000; + let Inst{14-12} = 0b011; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def ZAMES_MSETTILEMI : ZAMESInstConfigImm<0b0010, "msettilemi">; + def ZAMES_MSETTILENI : ZAMESInstConfigImm<0b0011, "msettileni">; + def ZAMES_MSETTILEKI : ZAMESInstConfigImm<0b0001, "msettileki">; + + def ZAMES_MSETTILEM : ZAMESInstConfigReg<0b0010, "msettilem">; + def ZAMES_MSETTILEN : ZAMESInstConfigReg<0b0011, "msettilen">; + def ZAMES_MSETTILEK : ZAMESInstConfigReg<0b0001, "msettilek">; + + def ZAMES_MSETCFG : ZAMESInstSetCfg<"msetcfg">; + def ZAMES_MGETCFG : ZAMESInstGetCfg<"mgetcfg">; + + def ZAMES_MINIT : ZAMESInstInit<"minit">; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 1, + mayStore = 0 in { + def ZAMES_MLA : ZAMESInstLoadStore<0b0000, 0b0, (outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mla">; + def ZAMES_MLB : ZAMESInstLoadStore<0b0001, 0b0, (outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlb">; + def ZAMES_MLC : ZAMESInstLoadStore<0b0010, 0b0, (outs ZamesAccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlc">; + def ZAMES_MLBT : ZAMESInstLoadStore<0b0101, 0b0, (outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlbt">; + def ZAMES_MLAT : ZAMESInstLoadStore<0b0100, 0b0, (outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlat">; + def ZAMES_MLCT : ZAMESInstLoadStore<0b0110, 0b0, (outs ZamesAccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlct">; + + def ZAMES_MLA_WHOLE + : ZAMESInstWholeLoadStore<0b1000, 0b0, (outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1), "mla.whole">; + def ZAMES_MLB_WHOLE + : ZAMESInstWholeLoadStore<0b1001, 0b0, (outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1), "mlb.whole">; + def ZAMES_MLC_WHOLE + : ZAMESInstWholeLoadStore<0b1010, 0b0, (outs ZamesAccReg:$md), + (ins GPRMemZeroOffset:$rs1), "mlc.whole">; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, + mayStore = 1 in { + def ZAMES_MSC : ZAMESInstLoadStore<0b0010, 0b1, (outs), + (ins ZamesAccReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msc">; + def ZAMES_MSA : ZAMESInstLoadStore<0b0000, 0b1, (outs), + (ins ZamesTileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msa">; + def ZAMES_MSB : ZAMESInstLoadStore<0b0001, 0b1, (outs), + (ins ZamesTileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msb">; + def ZAMES_MSAT : ZAMESInstLoadStore<0b0100, 0b1, (outs), + (ins ZamesTileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msat">; + def ZAMES_MSBT : ZAMESInstLoadStore<0b0101, 0b1, (outs), + (ins ZamesTileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msbt">; + def ZAMES_MSCT : ZAMESInstLoadStore<0b0110, 0b1, (outs), + (ins ZamesAccReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msct">; + + def ZAMES_MSA_WHOLE + : ZAMESInstWholeLoadStore<0b1000, 0b1, (outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1), + "msa.whole">; + def ZAMES_MSB_WHOLE + : ZAMESInstWholeLoadStore<0b1001, 0b1, (outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1), + "msb.whole">; + def ZAMES_MSC_WHOLE + : ZAMESInstWholeLoadStore<0b1010, 0b1, (outs), + (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1), + "msc.whole">; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def ZAMES_MZERO : ZAMESInstMisc<0b0000, (outs ZamesTileOrAccReg:$md), (ins), + "mzero">; + + def ZAMES_MMACC : ZAMESInstMMAcc<0b0000, (outs ZamesAccReg:$md_wb), + (ins ZamesAccReg:$md, ZamesTileReg:$ms2, + ZamesTileReg:$ms1), + "mmacc"> { + let Constraints = "$md_wb = $md"; + } +} + +//===----------------------------------------------------------------------===// +// ZMASYNC Matrix Async Synchronization Instructions +//===----------------------------------------------------------------------===// + +class ZMASYNCInst func4, dag outs, dag ins, + string opcodestr, string argstr> + : RVInst { + bits<5> rs1; + bits<5> sync; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = 0b0; + let Inst{24-20} = sync; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b100; + let Inst{11-7} = 0b00000; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +let Predicates = [HasStdExtZmasync], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def ZMASYNC_MSYNCREGRESET + : ZMASYNCInst<0b1000, (outs SyncR:$sync), (ins), + "msyncregreset", "$sync"> { + let rs1 = 0b00000; + } + + let Constraints = "$sync = $src" in { + def ZMASYNC_MRELEASE + : ZMASYNCInst<0b1001, (outs SyncR:$sync), (ins SyncR:$src), + "mrelease", "$sync"> { + let rs1 = 0b00000; + } + } + + def ZMASYNC_MACQUIRE + : ZMASYNCInst<0b1010, (outs), (ins SyncR:$sync, GPR:$rs1), + "macquire", "$sync, $rs1">; + + def ZMASYNC_MFENCE + : ZMASYNCInst<0b1011, (outs), (ins), "mfence", ""> { + let rs1 = 0b00000; + let sync = 0b00000; + } +} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td new file mode 100644 index 00000000000000..26a10558dc5d18 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td @@ -0,0 +1,595 @@ +//===-- RISCVInstrInfoZamesPatterns.td - ZAMES/ZMASYNC pseudos -*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC codegen-only pseudos and SelectionDAG patterns. +// +// Ordinary load/store, mzero and mmacc go through a two-level pseudo: +// intrinsic -> PSEUDO_ZAMES_*_PRE -> (pre-RA: materialize cfg payload GPR) +// -> PSEUDO_ZAMES_* with cfg operands -> RA -> (post-RA: emit msetcfg by +// final TR/ACC encoding) -> real ZAMES instruction. +// +// msettile*, msetcfg/mgetcfg, minit, *.whole and the four ZMASYNC +// instructions are selected directly to real instructions. +// +// mmacc operand reorder: LLVM IR operands are A, B, C (the intrinsic +// overload suffix lists the C, A, B *type* sequence, not the operand order); +// the typed pseudo and real instruction use the machine order C, B, A. +// +//===----------------------------------------------------------------------===// + +// Defensive second-layer immediate constraints. The ImmArg/Range intrinsic +// attributes reject out-of-range constants in the IR verifier; these leaves +// additionally prevent ISel from silently truncating an out-of-range target +// constant into the 10-bit tile field or the 3-bit config selector when the +// verifier is disabled or a transform built an illegal DAG. +def zames_tuimm10 : TImmLeaf(Imm); }]>; +def zames_tuimm3 : TImmLeaf(Imm); }]>; + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 1, + mayStore = 0, isCodeGenOnly = 1 in { + def PSEUDO_ZAMES_MLA_A_I8_PRE + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MLB_B_I8_PRE + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MLBT_B_I8_PRE + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MLAT_A_I8_PRE + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MLC_C_I32_PRE + : Pseudo<(outs ZamesAccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MLCT_C_I32_PRE + : Pseudo<(outs ZamesAccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + + def PSEUDO_ZAMES_MLA_A_I8 + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_ZAMES_MLB_B_I8 + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_ZAMES_MLBT_B_I8 + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_ZAMES_MLAT_A_I8 + : Pseudo<(outs ZamesTileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_ZAMES_MLC_C_I32 + : Pseudo<(outs ZamesAccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_ZAMES_MLCT_C_I32 + : Pseudo<(outs ZamesAccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, + mayStore = 0, isCodeGenOnly = 1 in { + def PSEUDO_ZAMES_MZERO_C_I32_PRE : Pseudo<(outs ZamesAccReg:$md), (ins), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MMACC_I8_I32_PRE + : Pseudo<(outs ZamesAccReg:$md_wb), + (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1), []> { + let Constraints = "$md_wb = $md"; + let Size = 28; + } + + def PSEUDO_ZAMES_MZERO_C_I32 + : Pseudo<(outs ZamesAccReg:$md), (ins GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_ZAMES_MMACC_I8_I32 + : Pseudo<(outs ZamesAccReg:$md_wb), + (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1, GPR:$cfg_c, + GPR:$cfg_b, GPR:$cfg_a), []> { + let Constraints = "$md_wb = $md"; + let Size = 16; + } +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, + mayStore = 1, isCodeGenOnly = 1 in { + def PSEUDO_ZAMES_MSC_C_I32_PRE + : Pseudo<(outs), + (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MSA_A_I8_PRE + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MSAT_A_I8_PRE + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MSB_B_I8_PRE + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MSBT_B_I8_PRE + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_ZAMES_MSCT_C_I32_PRE + : Pseudo<(outs), + (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + + def PSEUDO_ZAMES_MSC_C_I32 + : Pseudo<(outs), + (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_ZAMES_MSA_A_I8 + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_ZAMES_MSAT_A_I8 + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_ZAMES_MSB_B_I8 + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_ZAMES_MSBT_B_I8 + : Pseudo<(outs), + (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_ZAMES_MSCT_C_I32 + : Pseudo<(outs), + (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } +} + +class PatZAMESSetTileReg + : Pat<(OpNode (XLenVT GPR:$rs1)), (Inst GPR:$rs1)>; + +// Shared shells for the typed ZAMES pseudo instructions. Concrete defs carry +// the PSEUDO_ZAMES___ names so every instance stays +// reachable from the pre-RA and late C++ expansion switches. +class ZAMESLdPseudoPRE + : Pseudo<(outs RC:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; +} +class ZAMESLdPseudoCFG + : Pseudo<(outs RC:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; +} +class ZAMESStPseudoPRE + : Pseudo<(outs), (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; +} +class ZAMESStPseudoCFG + : Pseudo<(outs), + (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; +} +class ZAMESMZeroPseudoPRE + : Pseudo<(outs RC:$md), (ins), []> { + let Size = 12; +} +class ZAMESMZeroPseudoCFG + : Pseudo<(outs RC:$md), (ins GPR:$cfg), []> { + let Size = 8; +} +class ZAMESMMAccPseudoPRE + : Pseudo<(outs ZamesAccReg:$md_wb), + (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1), []> { + let Constraints = "$md_wb = $md"; + let Size = 28; +} +class ZAMESMMAccPseudoCFG + : Pseudo<(outs ZamesAccReg:$md_wb), + (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1, GPR:$cfg_c, + GPR:$cfg_b, GPR:$cfg_a), []> { + let Constraints = "$md_wb = $md"; + let Size = 16; +} + +// Typed load pseudo instances: A/B tiles use every supported input element; +// C accumulators use fp32. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 1, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_ZAMES_MLA_A_FP8E4_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLA_A_FP8E4 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLA_A_FP8E5_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLA_A_FP8E5 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLA_A_F16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLA_A_F16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLA_A_BF16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLA_A_BF16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLAT_A_FP8E4_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLAT_A_FP8E4 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLAT_A_FP8E5_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLAT_A_FP8E5 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLAT_A_F16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLAT_A_F16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLAT_A_BF16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLAT_A_BF16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLB_B_FP8E4_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLB_B_FP8E4 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLB_B_FP8E5_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLB_B_FP8E5 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLB_B_F16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLB_B_F16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLB_B_BF16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLB_B_BF16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLBT_B_FP8E4_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLBT_B_FP8E4 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLBT_B_FP8E5_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLBT_B_FP8E5 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLBT_B_F16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLBT_B_F16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLBT_B_BF16_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLBT_B_BF16 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLC_C_F32_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLC_C_F32 : ZAMESLdPseudoCFG; + def PSEUDO_ZAMES_MLCT_C_F32_PRE : ZAMESLdPseudoPRE; + def PSEUDO_ZAMES_MLCT_C_F32 : ZAMESLdPseudoCFG; +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 1, + isCodeGenOnly = 1 in { + def PSEUDO_ZAMES_MSA_A_FP8E4_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSA_A_FP8E4 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSA_A_FP8E5_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSA_A_FP8E5 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSA_A_F16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSA_A_F16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSA_A_BF16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSA_A_BF16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSAT_A_FP8E4_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSAT_A_FP8E4 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSAT_A_FP8E5_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSAT_A_FP8E5 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSAT_A_F16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSAT_A_F16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSAT_A_BF16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSAT_A_BF16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSB_B_FP8E4_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSB_B_FP8E4 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSB_B_FP8E5_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSB_B_FP8E5 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSB_B_F16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSB_B_F16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSB_B_BF16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSB_B_BF16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSBT_B_FP8E4_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSBT_B_FP8E4 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSBT_B_FP8E5_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSBT_B_FP8E5 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSBT_B_F16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSBT_B_F16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSBT_B_BF16_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSBT_B_BF16 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSC_C_F32_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSC_C_F32 : ZAMESStPseudoCFG; + def PSEUDO_ZAMES_MSCT_C_F32_PRE : ZAMESStPseudoPRE; + def PSEUDO_ZAMES_MSCT_C_F32 : ZAMESStPseudoCFG; +} + +// Typed mzero pseudo instances preserve the input/output register partition. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_ZAMES_MZERO_T_I8_PRE : ZAMESMZeroPseudoPRE; + def PSEUDO_ZAMES_MZERO_T_I8 : ZAMESMZeroPseudoCFG; + def PSEUDO_ZAMES_MZERO_T_FP8E4_PRE : ZAMESMZeroPseudoPRE; + def PSEUDO_ZAMES_MZERO_T_FP8E4 : ZAMESMZeroPseudoCFG; + def PSEUDO_ZAMES_MZERO_T_FP8E5_PRE : ZAMESMZeroPseudoPRE; + def PSEUDO_ZAMES_MZERO_T_FP8E5 : ZAMESMZeroPseudoCFG; + def PSEUDO_ZAMES_MZERO_T_F16_PRE : ZAMESMZeroPseudoPRE; + def PSEUDO_ZAMES_MZERO_T_F16 : ZAMESMZeroPseudoCFG; + def PSEUDO_ZAMES_MZERO_T_BF16_PRE : ZAMESMZeroPseudoPRE; + def PSEUDO_ZAMES_MZERO_T_BF16 : ZAMESMZeroPseudoCFG; + def PSEUDO_ZAMES_MZERO_C_F32_PRE : ZAMESMZeroPseudoPRE; + def PSEUDO_ZAMES_MZERO_C_F32 : ZAMESMZeroPseudoCFG; +} + +// Typed mmacc pseudo instances for the six supported combinations. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_ZAMES_MMACC_I8_F32_PRE : ZAMESMMAccPseudoPRE; + def PSEUDO_ZAMES_MMACC_I8_F32 : ZAMESMMAccPseudoCFG; + def PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE : ZAMESMMAccPseudoPRE; + def PSEUDO_ZAMES_MMACC_FP8E4_F32 : ZAMESMMAccPseudoCFG; + def PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE : ZAMESMMAccPseudoPRE; + def PSEUDO_ZAMES_MMACC_FP8E5_F32 : ZAMESMMAccPseudoCFG; + def PSEUDO_ZAMES_MMACC_F16_F32_PRE : ZAMESMMAccPseudoPRE; + def PSEUDO_ZAMES_MMACC_F16_F32 : ZAMESMMAccPseudoCFG; + def PSEUDO_ZAMES_MMACC_BF16_F32_PRE : ZAMESMMAccPseudoPRE; + def PSEUDO_ZAMES_MMACC_BF16_F32 : ZAMESMMAccPseudoCFG; +} + +class PatZAMESLoad + : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1, GPR:$rs2)), + (Inst GPRMemZeroOffset:$rs1, GPR:$rs2)>; + +class PatZAMESStore + : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2), + (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2)>; + +class PatZAMESWholeLoad + : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1)), + (Inst GPRMemZeroOffset:$rs1)>; + +class PatZAMESWholeStore + : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1), + (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1)>; + +let Predicates = [HasStdExtZames, IsRV64] in { + def : Pat<(int_riscv_zames_msettilemi zames_tuimm10:$imm), + (ZAMES_MSETTILEMI timm:$imm)>; + def : Pat<(int_riscv_zames_msettileni zames_tuimm10:$imm), + (ZAMES_MSETTILENI timm:$imm)>; + def : Pat<(int_riscv_zames_msettileki zames_tuimm10:$imm), + (ZAMES_MSETTILEKI timm:$imm)>; + + def : PatZAMESSetTileReg; + def : PatZAMESSetTileReg; + def : PatZAMESSetTileReg; + + def : Pat<(int_riscv_zames_msetcfg zames_tuimm3:$cfg, (XLenVT GPR:$rs1)), + (ZAMES_MSETCFG timm:$cfg, GPR:$rs1)>; + + def : Pat<(int_riscv_zames_minit), (ZAMES_MINIT)>; + def : Pat<(XLenVT (int_riscv_zames_mgetcfg zames_tuimm3:$cfg)), + (ZAMES_MGETCFG timm:$cfg)>; + + def : Pat<(mi32_t (int_riscv_zames_mzero)), + (PSEUDO_ZAMES_MZERO_C_I32_PRE)>; + + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + + def : Pat<(mi32_t (int_riscv_zames_mmacc (mi8_t ZamesTileReg:$rs1), + (mi8_t ZamesTileReg:$rs2), + (mi32_t ZamesAccReg:$rs3))), + (PSEUDO_ZAMES_MMACC_I8_I32_PRE (mi32_t ZamesAccReg:$rs3), + (mi8_t ZamesTileReg:$rs2), + (mi8_t ZamesTileReg:$rs1))>; + + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + def : PatZAMESLoad; + + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeLoad; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + def : PatZAMESWholeStore; + + def : Pat<(mi8_t (int_riscv_zames_mzero)), (PSEUDO_ZAMES_MZERO_T_I8_PRE)>; + def : Pat<(mf8e4_t (int_riscv_zames_mzero)), + (PSEUDO_ZAMES_MZERO_T_FP8E4_PRE)>; + def : Pat<(mf8e5_t (int_riscv_zames_mzero)), + (PSEUDO_ZAMES_MZERO_T_FP8E5_PRE)>; + def : Pat<(mf16_t (int_riscv_zames_mzero)), (PSEUDO_ZAMES_MZERO_T_F16_PRE)>; + def : Pat<(mbf16_t (int_riscv_zames_mzero)), + (PSEUDO_ZAMES_MZERO_T_BF16_PRE)>; + def : Pat<(mf32_t (int_riscv_zames_mzero)), (PSEUDO_ZAMES_MZERO_C_F32_PRE)>; + + def : Pat<(mf32_t (int_riscv_zames_mmacc (mi8_t ZamesTileReg:$rs1), + (mi8_t ZamesTileReg:$rs2), + (mf32_t ZamesAccReg:$rs3))), + (PSEUDO_ZAMES_MMACC_I8_F32_PRE (mf32_t ZamesAccReg:$rs3), + (mi8_t ZamesTileReg:$rs2), + (mi8_t ZamesTileReg:$rs1))>; + def : Pat<(mf32_t (int_riscv_zames_mmacc (mf8e4_t ZamesTileReg:$rs1), + (mf8e4_t ZamesTileReg:$rs2), + (mf32_t ZamesAccReg:$rs3))), + (PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE (mf32_t ZamesAccReg:$rs3), + (mf8e4_t ZamesTileReg:$rs2), + (mf8e4_t ZamesTileReg:$rs1))>; + def : Pat<(mf32_t (int_riscv_zames_mmacc (mf8e5_t ZamesTileReg:$rs1), + (mf8e5_t ZamesTileReg:$rs2), + (mf32_t ZamesAccReg:$rs3))), + (PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE (mf32_t ZamesAccReg:$rs3), + (mf8e5_t ZamesTileReg:$rs2), + (mf8e5_t ZamesTileReg:$rs1))>; + def : Pat<(mf32_t (int_riscv_zames_mmacc (mf16_t ZamesTileReg:$rs1), + (mf16_t ZamesTileReg:$rs2), + (mf32_t ZamesAccReg:$rs3))), + (PSEUDO_ZAMES_MMACC_F16_F32_PRE (mf32_t ZamesAccReg:$rs3), + (mf16_t ZamesTileReg:$rs2), + (mf16_t ZamesTileReg:$rs1))>; + def : Pat<(mf32_t (int_riscv_zames_mmacc (mbf16_t ZamesTileReg:$rs1), + (mbf16_t ZamesTileReg:$rs2), + (mf32_t ZamesAccReg:$rs3))), + (PSEUDO_ZAMES_MMACC_BF16_F32_PRE (mf32_t ZamesAccReg:$rs3), + (mbf16_t ZamesTileReg:$rs2), + (mbf16_t ZamesTileReg:$rs1))>; + + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; + def : PatZAMESStore; +} + +let Predicates = [HasStdExtZmasync, IsRV64] in { + def : Pat<(msync_t (int_riscv_zmasync_msyncregreset)), + (ZMASYNC_MSYNCREGRESET)>; + def : Pat<(msync_t (int_riscv_zmasync_mrelease (msync_t SyncR:$sync))), + (ZMASYNC_MRELEASE (msync_t SyncR:$sync))>; + def : Pat<(int_riscv_zmasync_macquire (msync_t SyncR:$sync), + (XLenVT GPR:$rs1)), + (ZMASYNC_MACQUIRE (msync_t SyncR:$sync), (XLenVT GPR:$rs1))>; + def : Pat<(int_riscv_zmasync_mfence), (ZMASYNC_MFENCE)>; +} + diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td b/llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td new file mode 100644 index 00000000000000..8b82d60fd0a7c7 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td @@ -0,0 +1,96 @@ +//===-- RISCVRegisterInfoZames.td - ZAMES/ZMASYNC registers -*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC register definitions. +// +// - 4 Tile Registers (tr0-tr3): input matrices A and B +// - 4 Accumulation Registers (acc0-acc3): output/accumulation matrix C +// - 16 Sync Registers (sync0-sync15): ZMASYNC synchronization +// +// RegisterClass Size values are placeholders; the real MLEN/AMUL ABI is not +// part of the MC-level migration. +// +//===----------------------------------------------------------------------===// + +let Namespace = "RISCV" in { + +// Base class for Tile Registers +class AMETileReg Enc, string n> : Register { + let HWEncoding{2-0} = Enc; + let HWEncoding{4-3} = 0b00; +} + +// Keep ZAMES records distinct from BOSC AME's identically spelled registers. +def ZAMES_TR0 : AMETileReg<0, "tr0">; +def ZAMES_TR1 : AMETileReg<1, "tr1">; +def ZAMES_TR2 : AMETileReg<2, "tr2">; +def ZAMES_TR3 : AMETileReg<3, "tr3">; + +// Base class for Accumulation Registers +class AMEAccReg Enc, string n> : Register { + let HWEncoding{2-0} = Enc; + let HWEncoding{4-3} = 0b00; +} + +// Define 4 Accumulation Registers: acc0-acc3 occupy the hardware register +// slots 4-7. +def ZAMES_ACC0 : AMEAccReg<4, "acc0">; +def ZAMES_ACC1 : AMEAccReg<5, "acc1">; +def ZAMES_ACC2 : AMEAccReg<6, "acc2">; +def ZAMES_ACC3 : AMEAccReg<7, "acc3">; + +// ZMASYNC Sync Registers (sync0-sync15) +class ZMASyncReg Enc, string n> : Register { + let HWEncoding{4-0} = Enc; +} + +foreach Index = 0-15 in { + def SYNC#Index : ZMASyncReg; +} + +} // End Namespace = "RISCV" + +defvar mi8_t = riscv_ame_tile_i8; +defvar mi32_t = riscv_ame_tile_i32; +defvar msync_t = riscv_ame_sync; +defvar mf8e4_t = riscv_ame_tile_fp8e4; +defvar mf8e5_t = riscv_ame_tile_fp8e5; +defvar mf16_t = riscv_ame_tile_f16; +defvar mbf16_t = riscv_ame_tile_bf16; +defvar mf32_t = riscv_ame_tile_f32; + +// Tile Register class (tr0-tr3) +// Note: Size is set to 256 as a placeholder; actual size depends on MLEN +def ZamesTileReg + : RegisterClass<"RISCV", [mi8_t, mf8e4_t, mf8e5_t, mf16_t, mbf16_t], 256, + (add ZAMES_TR0, ZAMES_TR1, ZAMES_TR2, ZAMES_TR3)> { + let Size = 256; // Placeholder: actual MLEN is hardware-defined +} + +// Accumulation Register class (acc0-acc3) +def ZamesAccReg + : RegisterClass<"RISCV", [mi32_t, mf32_t], 1024, + (add ZAMES_ACC0, ZAMES_ACC1, ZAMES_ACC2, ZAMES_ACC3)> { + let Size = 1024; // Placeholder: actual MLEN×AMUL is hardware-defined +} + +// Union of ZamesTileReg and ZamesAccReg used only as a machine-instruction +// operand class for real instructions whose encoding accepts both register +// files (e.g. mzero). Virtual registers are still allocated through +// ZamesTileReg or ZamesAccReg according to their MVT; this class never +// participates in register allocation. +def ZamesTileOrAccReg : RegisterClass<"RISCV", + [mi8_t, mf8e4_t, mf8e5_t, mi32_t, mf16_t, + mbf16_t, mf32_t], 1024, + (add ZamesTileReg, ZamesAccReg)> { + let Size = 1024; // Placeholder, matches ZamesAccReg +} + +// Sync register class used by ZMASYNC instructions. +def SyncR : RegisterClass<"RISCV", [msync_t], 64, + (add (sequence "SYNC%u", 0, 15))>; diff --git a/llvm/test/Assembler/riscv-zames-intrinsics.ll b/llvm/test/Assembler/riscv-zames-intrinsics.ll new file mode 100644 index 00000000000000..1673760fb3cfdc --- /dev/null +++ b/llvm/test/Assembler/riscv-zames-intrinsics.ll @@ -0,0 +1,130 @@ +; RUN: llvm-as %s -o %t.bc +; RUN: llvm-dis %t.bc -o - | FileCheck %s +; RUN: opt -passes=verify -disable-output %s + +; Round-trip every ZAMES/ZMASYNC intrinsic and every target extension type +; through the standard LLVM assembly/disassembly path. + +; --- Non-overloaded configuration intrinsics ---------------------------- + +; CHECK-LABEL: define void @config( +; CHECK: call void @llvm.riscv.zames.msettilemi(i64 4) +; CHECK: call void @llvm.riscv.zames.msettileni(i64 8) +; CHECK: call void @llvm.riscv.zames.msettileki(i64 16) +; CHECK: call void @llvm.riscv.zames.msettilem(i64 %m) +; CHECK: call void @llvm.riscv.zames.msettilen(i64 %n) +; CHECK: call void @llvm.riscv.zames.msettilek(i64 %k) +; CHECK: call void @llvm.riscv.zames.msetcfg(i64 4, i64 %payload) +; CHECK: %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 4) +; CHECK: call void @llvm.riscv.zames.minit() +define void @config(i64 %m, i64 %n, i64 %k, i64 %payload) { + call void @llvm.riscv.zames.msettilemi(i64 4) + call void @llvm.riscv.zames.msettileni(i64 8) + call void @llvm.riscv.zames.msettileki(i64 16) + call void @llvm.riscv.zames.msettilem(i64 %m) + call void @llvm.riscv.zames.msettilen(i64 %n) + call void @llvm.riscv.zames.msettilek(i64 %k) + call void @llvm.riscv.zames.msetcfg(i64 4, i64 %payload) + %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 4) + call void @llvm.riscv.zames.minit() + ret void +} + +; --- Overloaded load/store on int8 tiles -------------------------------- + +; CHECK-LABEL: define void @load_store_i8( +; CHECK: %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %pa, i64 %s) +; CHECK: %at = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %pa, i64 %s) +; CHECK: %aw = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %pa) +; CHECK: call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %at, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %aw, ptr %pa) +define void @load_store_i8(ptr %pa, i64 %s) { + %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %pa, i64 %s) + %at = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %pa, i64 %s) + %aw = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %pa) + call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %pa, i64 %s) + call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %at, ptr %pa, i64 %s) + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %aw, ptr %pa) + ret void +} + +; --- B/C tile class + transposed/whole on int32 accumulator ------------- + +; CHECK-LABEL: define void @load_store_bc( +; CHECK: %b = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.triscv.ame.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bt = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlbt.triscv.ame.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int32t(ptr %pb) +; CHECK: %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %pc, i64 %s) +; CHECK: %ct = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %pc, i64 %s) +; CHECK: %cw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %pc) +; CHECK: call void @llvm.riscv.zames.msb.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %b, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.zames.msbt.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bt, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bw, ptr %pb) +; CHECK: call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %ct, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %cw, ptr %pc) +define void @load_store_bc(ptr %pb, ptr %pc, i64 %s) { + %b = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.triscv.ame.tile.int32t(ptr %pb, i64 %s) + %bt = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlbt.triscv.ame.tile.int32t(ptr %pb, i64 %s) + %bw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int32t(ptr %pb) + %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %pc, i64 %s) + %ct = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %pc, i64 %s) + %cw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %pc) + call void @llvm.riscv.zames.msb.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %b, ptr %pb, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bt, ptr %pb, i64 %s) + call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bw, ptr %pb) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %pc, i64 %s) + call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %ct, ptr %pc, i64 %s) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %cw, ptr %pc) + ret void +} + +; --- mzero / mmacc ------------------------------------------------------ + +; CHECK-LABEL: define target("riscv.ame.tile.int32") @zero_acc() { +; CHECK: %z = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() +; CHECK: ret target("riscv.ame.tile.int32") %z +define target("riscv.ame.tile.int32") @zero_acc() { + %z = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + ret target("riscv.ame.tile.int32") %z +} + +; CHECK-LABEL: define target("riscv.ame.tile.int32") @matmul( +; CHECK: %r = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, target("riscv.ame.tile.int8") %b, target("riscv.ame.tile.int32") %c) +define target("riscv.ame.tile.int32") @matmul(target("riscv.ame.tile.int8") %a, target("riscv.ame.tile.int8") %b, target("riscv.ame.tile.int32") %c) { + %r = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, target("riscv.ame.tile.int8") %b, target("riscv.ame.tile.int32") %c) + ret target("riscv.ame.tile.int32") %r +} + +; --- Every floating tile type parses and prints ------------------------- + +; CHECK-LABEL: define void @fp_types( +; CHECK: call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr %p, i64 %s) +; CHECK: call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr %p, i64 %s) +; CHECK: call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr %p, i64 %s) +; CHECK: call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr %p, i64 %s) +; CHECK: call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mla.triscv.ame.tile.fp32t(ptr %p, i64 %s) +define void @fp_types(ptr %p, i64 %s) { + %e4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr %p, i64 %s) + %e5 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr %p, i64 %s) + %f16 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr %p, i64 %s) + %bf16 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr %p, i64 %s) + %f32 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mla.triscv.ame.tile.fp32t(ptr %p, i64 %s) + ret void +} + +; --- ZMASYNC ------------------------------------------------------------ + +; CHECK-LABEL: define void @sync_ops( +; CHECK: %s0 = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset.triscv.ame.synct() +; CHECK: %s1 = call target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease.triscv.ame.synct(target("riscv.ame.sync") %s0) +; CHECK: call void @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync") %s1, i64 %sem) +; CHECK: call void @llvm.riscv.zmasync.mfence() +define void @sync_ops(i64 %sem) { + %s0 = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset.triscv.ame.synct() + %s1 = call target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease.triscv.ame.synct(target("riscv.ame.sync") %s0) + call void @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync") %s1, i64 %sem) + call void @llvm.riscv.zmasync.mfence() + ret void +} diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll index c7042ce2431968..917cae82bc871d 100644 --- a/llvm/test/CodeGen/RISCV/features-info.ll +++ b/llvm/test/CodeGen/RISCV/features-info.ll @@ -349,6 +349,7 @@ ; CHECK-NEXT: zalasr - 'Zalasr' (Load-Acquire and Store-Release Instructions). ; CHECK-NEXT: zalrsc - 'Zalrsc' (Load-Reserved/Store-Conditional). ; CHECK-NEXT: zama16b - 'Zama16b' (Atomic 16-byte misaligned loads, stores and AMOs). +; CHECK: zames ; CHECK-NEXT: zawrs - 'Zawrs' (Wait on Reservation Set). ; CHECK-NEXT: zba - 'Zba' (Address Generation Instructions). ; CHECK-NEXT: zbb - 'Zbb' (Basic Bit-Manipulation). @@ -405,6 +406,7 @@ ; CHECK-NEXT: zksed - 'Zksed' (ShangMi Suite: SM4 Block Cipher Instructions). ; CHECK-NEXT: zksh - 'Zksh' (ShangMi Suite: SM3 Hash Function Instructions). ; CHECK-NEXT: zkt - 'Zkt' (Data Independent Execution Latency). +; CHECK: zmasync ; CHECK-NEXT: zmmul - 'Zmmul' (Integer Multiplication). ; CHECK-NEXT: ztso - 'Ztso' (Memory Model - Total Store Order). ; CHECK-NEXT: zvbb - 'Zvbb' (Vector basic bit-manipulation instructions). diff --git a/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll new file mode 100644 index 00000000000000..8700076394ed88 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll @@ -0,0 +1,26 @@ +; RV32 is not a supported ZAMES/ZMASYNC code generation target: the +; intrinsic scalar interfaces are i64 and the SelectionDAG patterns and +; pseudos are constrained to RV64. llc must reject RV32 with a stable +; diagnostic before instruction selection, without "Cannot select", a +; stack trace, or a crash. +; +; RUN: not llc -mtriple=riscv32 -mattr=+zames,+zmasync %s -o /dev/null 2>&1 | FileCheck %s + +; CHECK-NOT: Cannot select +; CHECK: error: {{.*}}ZAMES/ZMASYNC code generation requires RV64 +; CHECK-NOT: Cannot select + +define target("riscv.ame.tile.int8") @rv32_zames(ptr %p, i64 %stride) { + %t = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %p, i64 %stride) + ret target("riscv.ame.tile.int8") %t +} + +define void @rv32_zmasync() { + %s = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() + call void @llvm.riscv.zmasync.mfence() + ret void +} + +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() +declare void @llvm.riscv.zmasync.mfence() diff --git a/llvm/test/CodeGen/RISCV/zames/zames-attributes.ll b/llvm/test/CodeGen/RISCV/zames/zames-attributes.ll new file mode 100644 index 00000000000000..0c78edf4b78db8 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-attributes.ll @@ -0,0 +1,16 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zmasync < %s \ +; RUN: | FileCheck %s --check-prefix=ASM +; RUN: llc -mtriple=riscv64 -mattr=+zmasync -filetype=obj -o %t.o < %s +; RUN: llvm-readelf -A %t.o | FileCheck %s --check-prefix=OBJ + +; Pins the exact ELF arch-attribute string for ZMASYNC: +zmasync implies +; +zames and the canonical ordering lists zmasync before zames. + +; ASM: .attribute 5, "rv64i2p1_zmasync1p0_zames1p0" + +; OBJ: TagName: arch +; OBJ-NEXT: Value: rv64i2p1_zmasync1p0_zames1p0 + +define void @empty() { + ret void +} diff --git a/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll new file mode 100644 index 00000000000000..681ca14c369d6e --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll @@ -0,0 +1,266 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames -verify-machineinstrs < %s | FileCheck %s --implicit-check-not=llvm.riscv.buddy +; RUN: llc -mtriple=riscv64 -mattr=+zames -filetype=obj %s -o %t.o +; RUN: llvm-objdump --mattr=+zames -d %t.o | FileCheck %s --check-prefix=CHECK-OBJ + +; This file intentionally keeps every matrix value in one basic block with +; short live ranges. Matrix-register copy, spill, reload, frame layout, and ABI +; handling are outside the current backend contract. + +define void @zames_settile_immediates() { +; CHECK-LABEL: zames_settile_immediates: +; CHECK: msettilemi 4 +; CHECK-NEXT: msettileni 8 +; CHECK-NEXT: msettileki 16 +; CHECK-NEXT: ret + call void @llvm.riscv.zames.msettilemi(i64 4) + call void @llvm.riscv.zames.msettileni(i64 8) + call void @llvm.riscv.zames.msettileki(i64 16) + ret void +} + +; Encoding boundary of the uimm10 tile field and the 3-bit config selector: +; the inclusive hardware endpoints must still select. tile=0 is an LLVM +; encoding-boundary check only; the ZAMES dialect never emits zero sizes. +define void @zames_settile_immediate_boundaries() { +; CHECK-LABEL: zames_settile_immediate_boundaries: +; CHECK: msettilemi 0 +; CHECK-NEXT: msettileni 1023 +; CHECK-NEXT: msettileki 0 +; CHECK-NEXT: ret + call void @llvm.riscv.zames.msettilemi(i64 0) + call void @llvm.riscv.zames.msettileni(i64 1023) + call void @llvm.riscv.zames.msettileki(i64 0) + ret void +} + +define void @zames_settile_registers(i64 %m, i64 %n, i64 %k) { +; CHECK-LABEL: zames_settile_registers: +; CHECK: msettilem a0 +; CHECK-NEXT: msettilen a1 +; CHECK-NEXT: msettilek a2 +; CHECK-NEXT: ret +; CHECK-OBJ-LABEL: : +; CHECK-OBJ: msettilem a0 +; CHECK-OBJ-NEXT: msettilen a1 +; CHECK-OBJ-NEXT: msettilek a2 + call void @llvm.riscv.zames.msettilem(i64 %m) + call void @llvm.riscv.zames.msettilen(i64 %n) + call void @llvm.riscv.zames.msettilek(i64 %k) + ret void +} + +define void @zames_minit() { +; CHECK-LABEL: zames_minit: +; CHECK: # %bb.0: +; CHECK-NEXT: minit +; CHECK-NEXT: ret +; CHECK-OBJ-LABEL: : +; CHECK-OBJ: minit{{$}} + call void @llvm.riscv.zames.minit() + ret void +} + +define i64 @zames_cfg_intrinsics(i64 %value) { +; CHECK-LABEL: zames_cfg_intrinsics: +; CHECK: msetcfg mcfg3, a0 +; CHECK-NEXT: mgetcfg a0, mcfg3 +; CHECK-NEXT: ret + call void @llvm.riscv.zames.msetcfg(i64 3, i64 %value) + %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 3) + ret i64 %cfg +} + +define i64 @zames_cfg_immediate_boundaries(i64 %value) { +; CHECK-LABEL: zames_cfg_immediate_boundaries: +; CHECK: msetcfg mcfg0, a0 +; CHECK-NEXT: mgetcfg a0, mcfg7 +; CHECK-NEXT: ret + call void @llvm.riscv.zames.msetcfg(i64 0, i64 %value) + %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 7) + ret i64 %cfg +} + +; Check matrix register/configuration relationships, not GPR materialization +; order or a particular physical register allocation. +define void @zames_straight_line(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %stride_b, i64 %stride_c) { +; CHECK-LABEL: zames_straight_line: +; CHECK: msettilemi 4 +; CHECK: msettileni 4 +; CHECK: msettileki 8 +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mla tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlbt tr[[#B]], (a1), a4 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mzero acc[[#C-4]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: mmacc acc[[#C-4]], tr[[#B]], tr[[#A]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msc acc[[#C-4]], (a2), a5 +; CHECK: ret + call void @llvm.riscv.zames.msettilemi(i64 4) + call void @llvm.riscv.zames.msettileni(i64 4) + call void @llvm.riscv.zames.msettileki(i64 8) + %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( + target("riscv.ame.tile.int8") %tile_a, + target("riscv.ame.tile.int8") %tile_b, + target("riscv.ame.tile.int32") %zero) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_mlb_straight_line(ptr %a, ptr %b, ptr %c, + i64 %stride_a, i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: zames_mlb_straight_line: +; CHECK: msettilemi 4 +; CHECK: msettileni 4 +; CHECK: msettileki 8 +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mla tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlb tr[[#B]], (a1), a4 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mzero acc[[#C-4]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: mmacc acc[[#C-4]], tr[[#B]], tr[[#A]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msc acc[[#C-4]], (a2), a5 +; CHECK: ret + call void @llvm.riscv.zames.msettilemi(i64 4) + call void @llvm.riscv.zames.msettileni(i64 4) + call void @llvm.riscv.zames.msettileki(i64 8) + %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( + target("riscv.ame.tile.int8") %tile_a, + target("riscv.ame.tile.int8") %tile_b, + target("riscv.ame.tile.int32") %zero) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_mlc_straight_line(ptr %c, i64 %stride_c) { +; CHECK-LABEL: zames_mlc_straight_line: +; CHECK: msettilemi 4 +; CHECK: msettileni 4 +; CHECK: msettileki 8 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlc acc[[#C-4]], (a0), a1 +; CHECK: ret + call void @llvm.riscv.zames.msettilemi(i64 4) + call void @llvm.riscv.zames.msettileni(i64 4) + call void @llvm.riscv.zames.msettileki(i64 8) + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) + ret void +} + +define void @zames_transpose_load_store(ptr %a, ptr %c, i64 %stride_a, i64 %stride_c) { +; CHECK-LABEL: zames_transpose_load_store: +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlat tr[[#A]], (a0), a2 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlct acc[[#C-4]], (a1), a3 +; CHECK: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: msat tr[[#A]], (a0), a2 +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msct acc[[#C-4]], (a1), a3 +; CHECK: ret + %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) + %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) + call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %stride_a) + call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_store_variants(ptr %a, ptr %b, ptr %c, i64 %stride) { +; CHECK-LABEL: zames_store_variants: +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mla tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlb tr[[#B]], (a1), a3 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlc acc[[#C-4]], (a2), a3 +; CHECK: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: msa tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msb tr[[#B]], (a1), a3 +; CHECK: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msbt tr[[#B]], (a1), a3 +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msct acc[[#C-4]], (a2), a3 +; CHECK: ret + %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride) + %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride) + %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride) + call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %stride) + call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %stride) + ret void +} + +define void @zames_whole(ptr %src, ptr %dst) { +; CHECK-LABEL: zames_whole: +; CHECK-NOT: msetcfg +; CHECK: mla.whole tr0, (a0) +; CHECK-NEXT: mlb.whole tr1, (a0) +; CHECK-NEXT: mlc.whole acc0, (a0) +; CHECK-NEXT: msa.whole tr0, (a1) +; CHECK-NEXT: msb.whole tr1, (a1) +; CHECK-NEXT: msc.whole acc0, (a1) +; CHECK-NEXT: ret +; CHECK-OBJ-LABEL: : +; CHECK-OBJ-NOT: msetcfg +; CHECK-OBJ: mla.whole tr0, (a0) +; CHECK-OBJ-NEXT: mlb.whole tr1, (a0) +; CHECK-OBJ-NEXT: mlc.whole acc0, (a0) +; CHECK-OBJ-NEXT: msa.whole tr0, (a1) +; CHECK-OBJ-NEXT: msb.whole tr1, (a1) +; CHECK-OBJ-NEXT: msc.whole acc0, (a1) + %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) + %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) + %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) + ret void +} + +declare void @llvm.riscv.zames.msettilemi(i64) +declare void @llvm.riscv.zames.msettileni(i64) +declare void @llvm.riscv.zames.msettileki(i64) +declare void @llvm.riscv.zames.msettilem(i64) +declare void @llvm.riscv.zames.msettilen(i64) +declare void @llvm.riscv.zames.msettilek(i64) +declare void @llvm.riscv.zames.minit() +declare void @llvm.riscv.zames.msetcfg(i64, i64) +declare i64 @llvm.riscv.zames.mgetcfg(i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) +declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir b/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir new file mode 100644 index 00000000000000..b388c5207bea89 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir @@ -0,0 +1,54 @@ +# RUN: llc -mtriple=riscv64 -mattr=+zames \ +# RUN: -run-pass=riscv-expand-pseudo-pre-emit -o - %s | FileCheck %s + +# Late expansion must derive the msetcfg selector from the physical matrix +# register on the pseudo operand: tr0-tr3 -> 0-3, acc0-acc3 -> 4-7. It must +# also preserve the machine operand order C, B, A on the final mmacc. +# These relationships cannot be checked in the .ll tests because register +# allocation picks the registers there. +# Matrix operands are live on entry to this post-RA pass fixture. + +--- | + define void @zames_mir_cfg() { ret void } +... +--- +name: zames_mir_cfg +body: | + bb.0: + liveins: $zames_acc2, $zames_tr3, $zames_tr1 + + ; f16 x f16 -> f32 with C = acc2, B = tr3, A = tr1 + $x10 = ADDI $x0, 12 + $x11 = ADDI $x0, 9 + ; CHECK: ZAMES_MSETCFG 6, {{[^:]*}}$x10{{$}} + ; CHECK-NEXT: ZAMES_MSETCFG 3, {{[^:]*}}$x11{{$}} + ; CHECK-NEXT: ZAMES_MSETCFG 1, {{[^:]*}}$x11{{$}} + ; CHECK-NEXT: $zames_acc2 = ZAMES_MMACC {{[^:]*}}$zames_acc2, {{[^:]*}}$zames_tr3, {{[^:]*}}$zames_tr1{{$}} + $zames_acc2 = PSEUDO_ZAMES_MMACC_F16_F32 $zames_acc2, $zames_tr3, $zames_tr1, $x10, $x11, $x11 + + ; fp16 tile load into tr2 + $x12 = ADDI $x0, 9 + ; CHECK: ZAMES_MSETCFG 2, {{[^:]*}}$x12{{$}} + ; CHECK-NEXT: $zames_tr2 = ZAMES_MLA + $zames_tr2 = PSEUDO_ZAMES_MLA_A_F16 $x10, $x11, $x12 + + ; bf16 tile store from tr3 + $x13 = ADDI $x0, 10 + ; CHECK: ZAMES_MSETCFG 3, {{[^:]*}}$x13{{$}} + ; CHECK-NEXT: ZAMES_MSA + PSEUDO_ZAMES_MSA_A_BF16 $zames_tr3, $x10, $x11, $x13 + + ; mzero into an fp16 tile register + $x14 = ADDI $x0, 9 + ; CHECK: ZAMES_MSETCFG 1, {{[^:]*}}$x14{{$}} + ; CHECK-NEXT: $zames_tr1 = ZAMES_MZERO{{$}} + $zames_tr1 = PSEUDO_ZAMES_MZERO_T_F16 $x14 + + ; mzero into accumulator acc2 + $x15 = ADDI $x0, 12 + ; CHECK: ZAMES_MSETCFG 6, {{[^:]*}}$x15{{$}} + ; CHECK-NEXT: $zames_acc2 = ZAMES_MZERO{{$}} + $zames_acc2 = PSEUDO_ZAMES_MZERO_C_F32 $x15 + + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll new file mode 100644 index 00000000000000..b83d1d03324a07 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll @@ -0,0 +1,21 @@ +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames %s -o /dev/null 2>&1 \ +; RUN: | FileCheck %s + +; FP8 inputs only accumulate into fp32. An i32 accumulator has no mmacc +; pattern and must fail selection explicitly. + +; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.zames.mmacc + +define void @bad_mmacc_acc(ptr %src, ptr %dst) { + %a4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) + %b4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %dst) + %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) + %r = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %a4, target("riscv.ame.tile.fp8e4") %b4, target("riscv.ame.tile.int32") %c) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %r, ptr %dst) + ret void +} + +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.int32")) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll new file mode 100644 index 00000000000000..593b383e613d61 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll @@ -0,0 +1,22 @@ +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames %s -o /dev/null 2>&1 \ +; RUN: | FileCheck %s + +; Mixed input element types have no mmacc pattern: fp8e4 x fp8e5 is not a +; supported combination and must fail selection explicitly. + +; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.zames.mmacc + +define void @bad_mmacc(ptr %src, ptr %dst) { + %a4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) + %a5 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr %src) + %c = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) + %r = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e4") %a4, target("riscv.ame.tile.fp8e5") %a5, target("riscv.ame.tile.fp32") %c) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r, ptr %dst) + ret void +} + +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr) +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp8e5"), target("riscv.ame.tile.fp32")) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll new file mode 100644 index 00000000000000..15a55ead9dae02 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll @@ -0,0 +1,621 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=finalize-isel -o - %s \ +; RUN: | FileCheck %s --check-prefix=ISEL +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-ra -o - %s \ +; RUN: | FileCheck %s --check-prefix=PRERA +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-emit -o - %s \ +; RUN: | FileCheck %s --check-prefix=POSTRA +; RUN: llc -mtriple=riscv64 -mattr=+zames -filetype=obj %s -o %t.o +; RUN: llvm-objdump --mattr=+zames -d %t.o \ +; RUN: | FileCheck %s --check-prefix=OBJ + +; FP8E4/FP8E5/FP16/BF16 are input tile types; FP32 is an accumulator type. +; Typed loads, stores, mzero and mmacc expand through the two-level pseudo +; chain with the type payload from the spec (i8=0x02, i32=0x04, +; fp8e5m2=0x07, fp8e4m3=0x08, fp16=0x09, bf16=0x0a, fp32=0x0c). Pre-RA, +; payload materialization immediately precedes its typed pseudo; post-RA, +; msetcfg immediately precedes the real instruction. + +define void @zames_fp_ab_ls(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_fp_ab_ls +; ISEL: PSEUDO_ZAMES_MLA_A_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MSA_A_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MLAT_A_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MSAT_A_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MLB_B_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MSB_B_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MLBT_B_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MSBT_B_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MLA_A_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MSA_A_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MLAT_A_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MSAT_A_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MLB_B_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MSB_B_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MLBT_B_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MSBT_B_FP8E5_PRE +; PRERA-LABEL: name: zames_fp_ab_ls +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_FP8E5 {{.*}}, %[[C7]] :: +; POSTRA-LABEL: name: zames_fp_ab_ls +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; OBJ-LABEL: : +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} + %t1 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t8, ptr %dst, i64 %s) + ret void +} + +define void @zames_fp_c_ls(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_fp_c_ls +; ISEL: PSEUDO_ZAMES_MLC_C_F32_PRE +; ISEL: PSEUDO_ZAMES_MSC_C_F32_PRE +; ISEL: PSEUDO_ZAMES_MLCT_C_F32_PRE +; ISEL: PSEUDO_ZAMES_MSCT_C_F32_PRE +; PRERA-LABEL: name: zames_fp_c_ls +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MLC_C_F32 {{.*}}, %[[C12]] :: +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: PSEUDO_ZAMES_MSC_C_F32 {{.*}}, %[[C12]] :: +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MLCT_C_F32 {{.*}}, %[[C12]] :: +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: PSEUDO_ZAMES_MSCT_C_F32 {{.*}}, %[[C12]] :: +; POSTRA-LABEL: name: zames_fp_c_ls +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLC {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSC {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLCT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSCT {{.*}}:: (store unknown-size +; OBJ-LABEL: : +; OBJ: mlc{{[ \t]}} +; OBJ: msc{{[ \t]}} +; OBJ: mlct{{[ \t]}} +; OBJ: msct{{[ \t]}} + %c5 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.triscv.ame.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msc.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %c5, ptr %dst, i64 %s) + %c6 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlct.triscv.ame.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msct.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %c6, ptr %dst, i64 %s) + ret void +} + +define void @zames_fp16_bf16_ab_ls(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_fp16_bf16_ab_ls +; ISEL: PSEUDO_ZAMES_MLA_A_F16_PRE +; ISEL: PSEUDO_ZAMES_MSA_A_F16_PRE +; ISEL: PSEUDO_ZAMES_MLAT_A_F16_PRE +; ISEL: PSEUDO_ZAMES_MSAT_A_F16_PRE +; ISEL: PSEUDO_ZAMES_MLB_B_F16_PRE +; ISEL: PSEUDO_ZAMES_MSB_B_F16_PRE +; ISEL: PSEUDO_ZAMES_MLBT_B_F16_PRE +; ISEL: PSEUDO_ZAMES_MSBT_B_F16_PRE +; ISEL: PSEUDO_ZAMES_MLA_A_BF16_PRE +; ISEL: PSEUDO_ZAMES_MSA_A_BF16_PRE +; ISEL: PSEUDO_ZAMES_MLAT_A_BF16_PRE +; ISEL: PSEUDO_ZAMES_MSAT_A_BF16_PRE +; ISEL: PSEUDO_ZAMES_MLB_B_BF16_PRE +; ISEL: PSEUDO_ZAMES_MSB_B_BF16_PRE +; ISEL: PSEUDO_ZAMES_MLBT_B_BF16_PRE +; ISEL: PSEUDO_ZAMES_MSBT_B_BF16_PRE +; PRERA-LABEL: name: zames_fp16_bf16_ab_ls +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_BF16 {{.*}}, %[[C10]] :: +; POSTRA-LABEL: name: zames_fp16_bf16_ab_ls +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; OBJ-LABEL: : +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} + %t1 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msa.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlat.triscv.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msat.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.triscv.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msb.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msa.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlat.triscv.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msat.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.triscv.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msb.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlbt.triscv.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t8, ptr %dst, i64 %s) + ret void +} + +; Whole loads and stores select the real instruction directly; the transfer +; does not read the type config, so no msetcfg is emitted for them. +define void @zames_fp_whole(ptr %src, ptr %dst) { +; ISEL-LABEL: name: zames_fp_whole +; ISEL: ZAMES_MLA_WHOLE +; ISEL: ZAMES_MSA_WHOLE +; ISEL: ZAMES_MLB_WHOLE +; ISEL: ZAMES_MSB_WHOLE +; ISEL: ZAMES_MLA_WHOLE +; ISEL: ZAMES_MSA_WHOLE +; ISEL: ZAMES_MLB_WHOLE +; ISEL: ZAMES_MSB_WHOLE +; ISEL: ZAMES_MLC_WHOLE +; ISEL: ZAMES_MSC_WHOLE +; OBJ-LABEL: : +; OBJ-NOT: msetcfg +; OBJ: mla.whole{{[ \t]}} +; OBJ: msa.whole{{[ \t]}} +; OBJ: mlb.whole{{[ \t]}} +; OBJ: msb.whole{{[ \t]}} +; OBJ: mla.whole{{[ \t]}} +; OBJ: msa.whole{{[ \t]}} +; OBJ: mlb.whole{{[ \t]}} +; OBJ: msb.whole{{[ \t]}} +; OBJ: mlc.whole{{[ \t]}} +; OBJ: msc.whole{{[ \t]}} + %a = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %a, ptr %dst) + %b = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e5t(ptr %src) + call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %b, ptr %dst) + %a16 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp16t(ptr %src) + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %a16, ptr %dst) + %bb16 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.bf16t(ptr %src) + call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %bb16, ptr %dst) + %c = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %c, ptr %dst) + ret void +} + +; mzero covers all five input tile types and both accumulator types, each +; bound to its payload config and register class. +define void @zames_fp_mzero(ptr %dst) { +; ISEL-LABEL: name: zames_fp_mzero +; ISEL: PSEUDO_ZAMES_MZERO_T_I8_PRE +; ISEL: PSEUDO_ZAMES_MZERO_T_FP8E4_PRE +; ISEL: PSEUDO_ZAMES_MZERO_T_FP8E5_PRE +; ISEL: PSEUDO_ZAMES_MZERO_C_I32_PRE +; ISEL: PSEUDO_ZAMES_MZERO_T_F16_PRE +; ISEL: PSEUDO_ZAMES_MZERO_T_BF16_PRE +; ISEL: PSEUDO_ZAMES_MZERO_C_F32_PRE +; PRERA-LABEL: name: zames_fp_mzero +; PRERA: %[[C2:[0-9]+]]:gpr = ADDI $x0, 2 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_I8 %[[C2]]{{$}} +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_FP8E4 %[[C8]]{{$}} +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_FP8E5 %[[C7]]{{$}} +; PRERA: %[[C4:[0-9]+]]:gpr = ADDI $x0, 4 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MZERO_C_I32 %[[C4]]{{$}} +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_F16 %[[C9]]{{$}} +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_BF16 %[[C10]]{{$}} +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MZERO_C_F32 %[[C12]]{{$}} +; POSTRA-LABEL: name: zames_fp_mzero +; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} +; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} +; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MZERO{{$}} +; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} +; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MZERO{{$}} +; OBJ-LABEL: : +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}acc{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}acc{{[0-3]}} + %z1 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mzero.triscv.ame.tile.int8t() + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %z1, ptr %dst) + %z2 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e4t() + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %z2, ptr %dst) + %z3 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e5t() + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %z3, ptr %dst) + %z4 = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %z4, ptr %dst) + %z5 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mzero.triscv.ame.tile.fp16t() + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %z5, ptr %dst) + %z6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mzero.triscv.ame.tile.bf16t() + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %z6, ptr %dst) + %z7 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mzero.triscv.ame.tile.fp32t() + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %z7, ptr %dst) + ret void +} + +; mmacc covers the six supported element combinations. Distinct A/B pointers +; and load opcodes keep the required machine operand order C, B, A observable. +; Pre-RA materializes the accumulator and input payloads; late expansion emits +; one msetcfg per physical matrix operand. +define void @zames_fp_mmacc(ptr %src, ptr %src2, ptr %dst) { +; ISEL-LABEL: name: zames_fp_mmacc +; ISEL: %[[A1:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE +; ISEL: %[[B1:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE +; ISEL: %[[C1:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE +; ISEL: PSEUDO_ZAMES_MMACC_I8_I32_PRE %[[C1]], {{.*}}%[[B1]], {{.*}}%[[A1]]{{$}} +; ISEL: %[[A2:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE +; ISEL: %[[B2:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE +; ISEL: %[[C2:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE +; ISEL: PSEUDO_ZAMES_MMACC_I8_F32_PRE %[[C2]], {{.*}}%[[B2]], {{.*}}%[[A2]]{{$}} +; ISEL: %[[A3:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE +; ISEL: %[[B3:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE +; ISEL: %[[C3:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE +; ISEL: PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE %[[C3]], {{.*}}%[[B3]], {{.*}}%[[A3]]{{$}} +; ISEL: %[[A4:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE +; ISEL: %[[B4:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE +; ISEL: %[[C4:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE +; ISEL: PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE %[[C4]], {{.*}}%[[B4]], {{.*}}%[[A4]]{{$}} +; ISEL: %[[A5:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE +; ISEL: %[[B5:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE +; ISEL: %[[C5:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE +; ISEL: PSEUDO_ZAMES_MMACC_F16_F32_PRE %[[C5]], {{.*}}%[[B5]], {{.*}}%[[A5]]{{$}} +; ISEL: %[[A6:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE +; ISEL: %[[B6:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE +; ISEL: %[[C6:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE +; ISEL: PSEUDO_ZAMES_MMACC_BF16_F32_PRE %[[C6]], {{.*}}%[[B6]], {{.*}}%[[A6]]{{$}} +; PRERA-LABEL: name: zames_fp_mmacc +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 4 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 2 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_I32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 2 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_FP8E4_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_FP8E5_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_F16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_BF16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; POSTRA-LABEL: name: zames_fp_mmacc +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} +; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} +; OBJ-LABEL: : +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} + %a1 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) + %b1 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src2) + %c1 = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) + %r1 = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a1, target("riscv.ame.tile.int8") %b1, target("riscv.ame.tile.int32") %c1) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %r1, ptr %dst) + %a2 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) + %b2 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src2) + %c2 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) + %r2 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a2, target("riscv.ame.tile.int8") %b2, target("riscv.ame.tile.fp32") %c2) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r2, ptr %dst) + %a3 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) + %b3 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e4t(ptr %src2) + %c3 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) + %r3 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %a3, target("riscv.ame.tile.fp8e4") %b3, target("riscv.ame.tile.fp32") %c3) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r3, ptr %dst) + %a4 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr %src) + %b4 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e5t(ptr %src2) + %c4 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) + %r4 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e5t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %a4, target("riscv.ame.tile.fp8e5") %b4, target("riscv.ame.tile.fp32") %c4) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r4, ptr %dst) + %a5 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp16t(ptr %src) + %b5 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp16t(ptr %src2) + %c5 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) + %r5 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp16t.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %a5, target("riscv.ame.tile.fp16") %b5, target("riscv.ame.tile.fp32") %c5) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r5, ptr %dst) + %a6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.bf16t(ptr %src) + %b6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.bf16t(ptr %src2) + %c6 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) + %r6 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.bf16t.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %a6, target("riscv.ame.tile.bf16") %b6, target("riscv.ame.tile.fp32") %c6) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r6, ptr %dst) + ret void +} + +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr, i64) +declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlat.triscv.ame.tile.fp16t(ptr, i64) +declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.triscv.ame.tile.fp16t(ptr, i64) +declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp16t(ptr, i64) +declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr, i64) +declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlat.triscv.ame.tile.bf16t(ptr, i64) +declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.triscv.ame.tile.bf16t(ptr, i64) +declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlbt.triscv.ame.tile.bf16t(ptr, i64) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.triscv.ame.tile.fp32t(ptr, i64) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlct.triscv.ame.tile.fp32t(ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.zames.msc.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr, i64) +declare void @llvm.riscv.zames.msct.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr, i64) +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr) +declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp16t(ptr) +declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.bf16t(ptr) +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e5t(ptr) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e4t(ptr) +declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp16t(ptr) +declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.bf16t(ptr) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr) +declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr) +declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mzero.triscv.ame.tile.int8t() +declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e4t() +declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e5t() +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() +declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mzero.triscv.ame.tile.fp16t() +declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mzero.triscv.ame.tile.bf16t() +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mzero.triscv.ame.tile.fp32t() +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.fp32")) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp32")) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e5t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), target("riscv.ame.tile.fp8e5"), target("riscv.ame.tile.fp32")) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp16t.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), target("riscv.ame.tile.fp16"), target("riscv.ame.tile.fp32")) +declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.bf16t.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), target("riscv.ame.tile.bf16"), target("riscv.ame.tile.fp32")) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll b/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll new file mode 100644 index 00000000000000..cd43377d0d909c --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll @@ -0,0 +1,55 @@ +; Defensive ISel negatives: with the IR verifier disabled, an out-of-range +; immediate must still fail selection instead of being truncated into the +; 10-bit tile field or the 3-bit config selector. + +; RUN: split-file %s %t +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/settilemi-neg.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=MI-NEG +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/settileni-1024.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=NI-1024 +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/settileki-1024.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=KI-1024 +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/msetcfg-8.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SETCFG-8 +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/mgetcfg-neg.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=GETCFG-NEG + +;--- settilemi-neg.ll +declare void @llvm.riscv.zames.msettilemi(i64) + +define void @f() { +; MI-NEG: Cannot select + call void @llvm.riscv.zames.msettilemi(i64 -1) + ret void +} + +;--- settileni-1024.ll +declare void @llvm.riscv.zames.msettileni(i64) + +define void @f() { +; NI-1024: Cannot select + call void @llvm.riscv.zames.msettileni(i64 1024) + ret void +} + +;--- settileki-1024.ll +declare void @llvm.riscv.zames.msettileki(i64) + +define void @f() { +; KI-1024: Cannot select + call void @llvm.riscv.zames.msettileki(i64 1024) + ret void +} + +;--- msetcfg-8.ll +declare void @llvm.riscv.zames.msetcfg(i64, i64) + +define void @f(i64 %payload) { +; SETCFG-8: Cannot select + call void @llvm.riscv.zames.msetcfg(i64 8, i64 %payload) + ret void +} + +;--- mgetcfg-neg.ll +declare i64 @llvm.riscv.zames.mgetcfg(i64) + +define i64 @f() { +; GETCFG-NEG: Cannot select + %r = call i64 @llvm.riscv.zames.mgetcfg(i64 -1) + ret i64 %r +} diff --git a/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll new file mode 100644 index 00000000000000..4261ce7f53dcc6 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll @@ -0,0 +1,115 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames -stop-after=finalize-isel -o - %s | FileCheck %s + +; The matrix data intrinsics must select to typed pre-RA pseudos first. cfg GPR +; payloads are materialized by riscv-expand-pseudo-pre-ra before RA. + +define void @zames_selects_typed_pseudos(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %stride_b, i64 %stride_c) { +; CHECK-LABEL: name: zames_selects_typed_pseudos +; CHECK: PSEUDO_ZAMES_MLA_A_I8_PRE +; CHECK: PSEUDO_ZAMES_MLBT_B_I8_PRE +; CHECK: PSEUDO_ZAMES_MZERO_C_I32_PRE +; CHECK: PSEUDO_ZAMES_MMACC_I8_I32_PRE +; CHECK: PSEUDO_ZAMES_MSC_C_I32_PRE + %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( + target("riscv.ame.tile.int8") %tile_a, + target("riscv.ame.tile.int8") %tile_b, + target("riscv.ame.tile.int32") %zero) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_selects_mlb_pseudo(ptr %a, ptr %b, ptr %c, + i64 %stride_a, i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: name: zames_selects_mlb_pseudo +; CHECK: PSEUDO_ZAMES_MLA_A_I8_PRE +; CHECK: PSEUDO_ZAMES_MLB_B_I8_PRE +; CHECK: PSEUDO_ZAMES_MZERO_C_I32_PRE +; CHECK: PSEUDO_ZAMES_MMACC_I8_I32_PRE +; CHECK: PSEUDO_ZAMES_MSC_C_I32_PRE + %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( + target("riscv.ame.tile.int8") %tile_a, + target("riscv.ame.tile.int8") %tile_b, + target("riscv.ame.tile.int32") %zero) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_selects_mlc_pseudo(ptr %c, i64 %stride_c) { +; CHECK-LABEL: name: zames_selects_mlc_pseudo +; CHECK: PSEUDO_ZAMES_MLC_C_I32_PRE + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) + ret void +} + +define void @zames_selects_transpose_and_store_pseudos(ptr %a, ptr %c, + i64 %sa, i64 %sc) { +; CHECK-LABEL: name: zames_selects_transpose_and_store_pseudos +; CHECK: PSEUDO_ZAMES_MLAT_A_I8_PRE +; CHECK: PSEUDO_ZAMES_MLCT_C_I32_PRE +; CHECK: PSEUDO_ZAMES_MSAT_A_I8_PRE +; CHECK: PSEUDO_ZAMES_MSCT_C_I32_PRE + %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %sc) + ret void +} + +define void @zames_selects_ab_store_pseudos(ptr %a, ptr %b, i64 %s) { +; CHECK-LABEL: name: zames_selects_ab_store_pseudos +; CHECK: PSEUDO_ZAMES_MSA_A_I8_PRE +; CHECK: PSEUDO_ZAMES_MSB_B_I8_PRE +; CHECK: PSEUDO_ZAMES_MSBT_B_I8_PRE + %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) + ret void +} + +define void @zames_selects_whole_direct(ptr %src, ptr %dst) { +; CHECK-LABEL: name: zames_selects_whole_direct +; CHECK-NOT: PSEUDO +; CHECK: ZAMES_MLA_WHOLE +; CHECK: ZAMES_MLB_WHOLE +; CHECK: ZAMES_MLC_WHOLE +; CHECK: ZAMES_MSA_WHOLE +; CHECK: ZAMES_MSB_WHOLE +; CHECK: ZAMES_MSC_WHOLE + %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) + %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) + %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) + ret void +} + +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) +declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll new file mode 100644 index 00000000000000..8a19cb530be51d --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll @@ -0,0 +1,127 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=finalize-isel -o - %s \ +; RUN: | FileCheck %s --check-prefix=ISEL +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-ra -o - %s \ +; RUN: | FileCheck %s --check-prefix=PRERA +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-emit -o - %s \ +; RUN: | FileCheck %s --check-prefix=POSTRA + +; Every ZAMES load/store MachineInstr must carry a MachineMemOperand through +; all three expansion stages. Normal/transpose accesses use the conservative +; address-space fallback (no %ir value), whole accesses keep the exact base +; pointer. Load results feed matching stores so nothing is deleted as unused. + +define void @zames_mem_ab_i8(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_mem_ab_i8 +; ISEL: PSEUDO_ZAMES_MLA_A_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_ZAMES_MLAT_A_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_ZAMES_MLB_B_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_ZAMES_MLBT_B_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_ZAMES_MSA_A_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_ZAMES_MSAT_A_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_ZAMES_MSB_B_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_ZAMES_MSBT_B_I8_PRE {{.*}}:: (store unknown-size +; PRERA-LABEL: name: zames_mem_ab_i8 +; PRERA: PSEUDO_ZAMES_MLA_A_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_ZAMES_MLAT_A_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_ZAMES_MLB_B_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_ZAMES_MLBT_B_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_ZAMES_MSA_A_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_ZAMES_MSAT_A_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_ZAMES_MSB_B_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_ZAMES_MSBT_B_I8 {{.*}}:: (store unknown-size +; POSTRA-LABEL: name: zames_mem_ab_i8 +; POSTRA: ZAMES_MSETCFG {{[^:]*$}} +; POSTRA: ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: ZAMES_MSBT {{.*}}:: (store unknown-size + %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %src, i64 %s) + %tat = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %src, i64 %s) + %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %src, i64 %s) + %tbt = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %dst, i64 %s) + call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tat, ptr %dst, i64 %s) + call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %dst, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tbt, ptr %dst, i64 %s) + ret void +} + +define void @zames_mem_c_i32(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_mem_c_i32 +; ISEL: PSEUDO_ZAMES_MLC_C_I32_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_ZAMES_MLCT_C_I32_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_ZAMES_MSC_C_I32_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_ZAMES_MSCT_C_I32_PRE {{.*}}:: (store unknown-size +; PRERA-LABEL: name: zames_mem_c_i32 +; PRERA: PSEUDO_ZAMES_MLC_C_I32 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_ZAMES_MLCT_C_I32 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_ZAMES_MSC_C_I32 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_ZAMES_MSCT_C_I32 {{.*}}:: (store unknown-size +; POSTRA-LABEL: name: zames_mem_c_i32 +; POSTRA: ZAMES_MLC {{.*}}:: (load unknown-size +; POSTRA: ZAMES_MLCT {{.*}}:: (load unknown-size +; POSTRA: ZAMES_MSC {{.*}}:: (store unknown-size +; POSTRA: ZAMES_MSCT {{.*}}:: (store unknown-size + %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %src, i64 %s) + %tct = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %src, i64 %s) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %dst, i64 %s) + call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tct, ptr %dst, i64 %s) + ret void +} + +define void @zames_mem_whole(ptr %src, ptr %dst) { +; ISEL-LABEL: name: zames_mem_whole +; ISEL: ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; ISEL: ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; ISEL: ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA-LABEL: name: zames_mem_whole +; PRERA: ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA: ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA: ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA-LABEL: name: zames_mem_whole +; POSTRA: ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA: ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA: ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst + %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) + %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) + %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) + ret void +} + +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll new file mode 100644 index 00000000000000..d83c3a07db0b28 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll @@ -0,0 +1,12 @@ +; RUN: not --crash llc -mtriple=riscv64 -verify-machineinstrs < %s 2>&1 | FileCheck %s + +; Without +zames, the target must not legally select ZAMES matrix intrinsics. + +define void @zames_requires_feature(ptr %c, i64 %stride) { +; CHECK: Cannot select: intrinsic %llvm.riscv.zames.msc + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t( + target("riscv.ame.tile.int32") undef, ptr %c, i64 %stride) + ret void +} + +declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll new file mode 100644 index 00000000000000..90d8bf7804f835 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll @@ -0,0 +1,140 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames -stop-after=riscv-expand-pseudo-pre-ra -o - %s | FileCheck %s + +define void @zames_prera_materializes_cfg(ptr %a, ptr %b, ptr %c, + i64 %stride_a, i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: name: zames_prera_materializes_cfg +; CHECK: %[[CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_A]] +; CHECK: %[[CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[TILE_B:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLBT_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_B]] +; CHECK: %[[CFG_ZERO:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[ZERO:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MZERO_C_I32 %[[CFG_ZERO]] +; CHECK: %[[CFG_ACC:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[CFG_MAT:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[ACC:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_I32 %[[ZERO]], %[[TILE_B]], %[[TILE_A]], %[[CFG_ACC]], %[[CFG_MAT]], %[[CFG_MAT]] +; CHECK: %[[CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: PSEUDO_ZAMES_MSC_C_I32 %[[ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_STORE]] + %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( + target("riscv.ame.tile.int8") %tile_a, + target("riscv.ame.tile.int8") %tile_b, + target("riscv.ame.tile.int32") %zero) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_prera_materializes_mlb_cfg(ptr %a, ptr %b, ptr %c, + i64 %stride_a, + i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: name: zames_prera_materializes_mlb_cfg +; CHECK: %[[MLB_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[MLB_TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_A]] +; CHECK: %[[MLB_CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[MLB_TILE_B:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_B]] +; CHECK: %[[MLB_CFG_ZERO:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[MLB_ZERO:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MZERO_C_I32 %[[MLB_CFG_ZERO]] +; CHECK: %[[MLB_CFG_ACC:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[MLB_CFG_MAT:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[MLB_ACC:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_I32 %[[MLB_ZERO]], %[[MLB_TILE_B]], %[[MLB_TILE_A]], %[[MLB_CFG_ACC]], %[[MLB_CFG_MAT]], %[[MLB_CFG_MAT]] +; CHECK: %[[MLB_CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: PSEUDO_ZAMES_MSC_C_I32 %[[MLB_ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_STORE]] + %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( + target("riscv.ame.tile.int8") %tile_a, + target("riscv.ame.tile.int8") %tile_b, + target("riscv.ame.tile.int32") %zero) + call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_prera_materializes_mlc_cfg(ptr %c, i64 %stride_c) { +; CHECK-LABEL: name: zames_prera_materializes_mlc_cfg +; CHECK: %[[MLC_CFG:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[MLC_ACC:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MLC_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLC_CFG]] + %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) + ret void +} + +define void @zames_prera_transpose_store_cfg(ptr %a, ptr %c, + i64 %sa, i64 %sc) { +; CHECK-LABEL: name: zames_prera_transpose_store_cfg +; CHECK: %[[T_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[T_TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLAT_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_A]] +; CHECK: %[[T_CFG_C:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[T_ACC_C:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MLCT_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_C]] +; CHECK: %[[T_CFG_SA:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_ZAMES_MSAT_A_I8 %[[T_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SA]] +; CHECK: %[[T_CFG_SC:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: PSEUDO_ZAMES_MSCT_C_I32 %[[T_ACC_C]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SC]] + %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %sc) + ret void +} + +define void @zames_prera_ab_store_cfg(ptr %a, ptr %b, i64 %s) { +; CHECK-LABEL: name: zames_prera_ab_store_cfg +; CHECK: %[[AB_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[AB_TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_A]] +; CHECK: %[[AB_CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[AB_TILE_B:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_B]] +; CHECK: %[[AB_CFG_SA:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_ZAMES_MSA_A_I8 %[[AB_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SA]] +; CHECK: %[[AB_CFG_SB:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_ZAMES_MSB_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SB]] +; CHECK: %[[AB_CFG_ST:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_ZAMES_MSBT_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_ST]] + %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) + ret void +} + +define void @zames_prera_whole_no_cfg(ptr %src, ptr %dst) { +; CHECK-LABEL: name: zames_prera_whole_no_cfg +; CHECK-NOT: ADDI +; CHECK-NOT: PSEUDO +; CHECK: ZAMES_MLA_WHOLE +; CHECK: ZAMES_MLB_WHOLE +; CHECK: ZAMES_MLC_WHOLE +; CHECK: ZAMES_MSA_WHOLE +; CHECK: ZAMES_MSB_WHOLE +; CHECK: ZAMES_MSC_WHOLE + %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) + %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) + %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) + ret void +} + +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) +declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) +declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) +declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll new file mode 100644 index 00000000000000..7d88ccd3586de1 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll @@ -0,0 +1,20 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zmasync -verify-machineinstrs < %s | FileCheck %s + +define void @zmasync_intrinsics(i64 %target) { +; CHECK-LABEL: zmasync_intrinsics: +; CHECK: msyncregreset sync0 +; CHECK-NEXT: mrelease sync0 +; CHECK-NEXT: macquire sync0, a0 +; CHECK-NEXT: mfence +; CHECK-NEXT: ret + %sync0 = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() + %sync1 = call target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease(target("riscv.ame.sync") %sync0) + call void @llvm.riscv.zmasync.macquire(target("riscv.ame.sync") %sync1, i64 %target) + call void @llvm.riscv.zmasync.mfence() + ret void +} + +declare target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() +declare target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease(target("riscv.ame.sync")) +declare void @llvm.riscv.zmasync.macquire(target("riscv.ame.sync"), i64) +declare void @llvm.riscv.zmasync.mfence() diff --git a/llvm/test/MC/RISCV/zames-invalid.s b/llvm/test/MC/RISCV/zames-invalid.s new file mode 100644 index 00000000000000..f6057e4257918b --- /dev/null +++ b/llvm/test/MC/RISCV/zames-invalid.s @@ -0,0 +1,23 @@ +# RUN: not llvm-mc %s -triple=riscv64 -mattr=+zames 2>&1 | FileCheck %s + +# ACC register where only TileReg is accepted. +mla acc0, (a0), a1 +# CHECK: error: + +# Out-of-range tile/accumulator registers. +mla tr4, (a0), a1 +# CHECK: error: +mzero acc4 +# CHECK: error: + +# Wrong operand count. +msetcfg a0 +# CHECK: error: + +# Illegal mcfg selector. +msetcfg mcfg8, a0 +# CHECK: error: + +# Wrong operand type. +msetcfg mcfg0, mcfg1 +# CHECK: error: diff --git a/llvm/test/MC/RISCV/zames-no-feature.s b/llvm/test/MC/RISCV/zames-no-feature.s new file mode 100644 index 00000000000000..045a7bb338c19e --- /dev/null +++ b/llvm/test/MC/RISCV/zames-no-feature.s @@ -0,0 +1,62 @@ +# RUN: not llvm-mc %s -triple=riscv64 2>&1 | FileCheck %s + +# Every ZAMES mnemonic must be rejected when the extension is not enabled. + +# CHECK: error: instruction requires the following: 'ZAMES' +msettilemi 4 +# CHECK: error: instruction requires the following: 'ZAMES' +msettileni 4 +# CHECK: error: instruction requires the following: 'ZAMES' +msettileki 8 +# CHECK: error: instruction requires the following: 'ZAMES' +msettilem a0 +# CHECK: error: instruction requires the following: 'ZAMES' +msettilen a1 +# CHECK: error: instruction requires the following: 'ZAMES' +msettilek a2 +# CHECK: error: instruction requires the following: 'ZAMES' +msetcfg mcfg0, a0 +# CHECK: error: instruction requires the following: 'ZAMES' +mgetcfg a0, mcfg5 +# CHECK: error: instruction requires the following: 'ZAMES' +minit +# CHECK: error: instruction requires the following: 'ZAMES' +mzero acc0 +# CHECK: error: instruction requires the following: 'ZAMES' +mmacc acc0, tr1, tr0 +# CHECK: error: instruction requires the following: 'ZAMES' +mla tr0, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlb tr2, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlbt tr1, (a1), a4 +# CHECK: error: instruction requires the following: 'ZAMES' +mlc acc2, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlat tr1, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlct acc1, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +msa tr0, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msb tr1, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msat tr2, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msbt tr3, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msc acc0, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msct acc0, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +mla.whole tr0, (a0) +# CHECK: error: instruction requires the following: 'ZAMES' +mlb.whole tr1, (a0) +# CHECK: error: instruction requires the following: 'ZAMES' +mlc.whole acc0, (a0) +# CHECK: error: instruction requires the following: 'ZAMES' +msa.whole tr0, (a2) +# CHECK: error: instruction requires the following: 'ZAMES' +msb.whole tr1, (a2) +# CHECK: error: instruction requires the following: 'ZAMES' +msc.whole acc0, (a2) diff --git a/llvm/test/MC/RISCV/zames.s b/llvm/test/MC/RISCV/zames.s new file mode 100644 index 00000000000000..238f5cb3560ac8 --- /dev/null +++ b/llvm/test/MC/RISCV/zames.s @@ -0,0 +1,147 @@ +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zames -show-encoding | FileCheck %s --check-prefix=CHECK-ASM +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zames -filetype=obj | llvm-objdump --mattr=+zames -d - | FileCheck %s --check-prefix=CHECK-OBJ + +# CHECK-ASM: msettilemi 4 # encoding: [0x2b,0x00,0x02,0x20] +# CHECK-OBJ: msettilemi 0x4 +msettilemi 4 + +# CHECK-ASM: msettileni 4 # encoding: [0x2b,0x00,0x02,0x30] +# CHECK-OBJ: msettileni 0x4 +msettileni 4 + +# CHECK-ASM: msettileki 8 # encoding: [0x2b,0x00,0x04,0x10] +# CHECK-OBJ: msettileki 0x8 +msettileki 8 + +# CHECK-ASM: msettilem a0 # encoding: [0x2b,0x00,0x05,0x22] +# CHECK-OBJ: msettilem a0 +msettilem a0 + +# CHECK-ASM: msettilen a1 # encoding: [0x2b,0x80,0x05,0x32] +# CHECK-OBJ: msettilen a1 +msettilen a1 + +# CHECK-ASM: msettilek a2 # encoding: [0x2b,0x00,0x06,0x12] +# CHECK-OBJ: msettilek a2 +msettilek a2 + +# CHECK-ASM: msetcfg mcfg0, a0 # encoding: [0x2b,0x00,0x05,0x42] +# CHECK-OBJ: msetcfg mcfg0, a0 +msetcfg mcfg0, a0 + +# CHECK-ASM: msetcfg mcfg4, a1 # encoding: [0x2b,0x82,0x05,0x42] +# CHECK-OBJ: msetcfg mcfg4, a1 +msetcfg mcfg4, a1 + +# CHECK-ASM: msetcfg mcfg7, a1 # encoding: [0xab,0x83,0x05,0x42] +# CHECK-OBJ: msetcfg mcfg7, a1 +msetcfg mcfg7, a1 + +# CHECK-ASM: mgetcfg a0, mcfg5 # encoding: [0x2b,0x85,0x02,0x50] +# CHECK-OBJ: mgetcfg a0, mcfg5 +mgetcfg a0, mcfg5 + +# CHECK-ASM: minit # encoding: [0x2b,0x00,0x00,0x00] +# CHECK-OBJ: minit +minit + +# CHECK-ASM: mla tr0, (a0), a3 # encoding: [0x2b,0x10,0xd5,0x00] +# CHECK-OBJ: mla tr0, (a0), a3 +mla tr0, (a0), a3 + +# CHECK-ASM: mlb tr2, (a0), a3 # encoding: [0x2b,0x11,0xd5,0x10] +# CHECK-OBJ: mlb tr2, (a0), a3 +mlb tr2, (a0), a3 + +# CHECK-ASM: mlbt tr1, (a1), a4 # encoding: [0xab,0x90,0xe5,0x50] +# CHECK-OBJ: mlbt tr1, (a1), a4 +mlbt tr1, (a1), a4 + +# CHECK-ASM: mlc acc2, (a0), a3 # encoding: [0x2b,0x13,0xd5,0x20] +# CHECK-OBJ: mlc acc2, (a0), a3 +mlc acc2, (a0), a3 + +# CHECK-ASM: mzero acc0 # encoding: [0x2b,0x32,0x00,0x00] +# CHECK-OBJ: mzero acc0 +mzero acc0 + +# CHECK-ASM: mzero tr0 # encoding: [0x2b,0x30,0x00,0x00] +# CHECK-OBJ: mzero tr0 +mzero tr0 + +# CHECK-ASM: mzero tr3 # encoding: [0xab,0x31,0x00,0x00] +# CHECK-OBJ: mzero tr3 +mzero tr3 + +# CHECK-ASM: mzero acc3 # encoding: [0xab,0x33,0x00,0x00] +# CHECK-OBJ: mzero acc3 +mzero acc3 + +# CHECK-ASM: mmacc acc0, tr1, tr0 # encoding: [0x2b,0x22,0x10,0x00] +# CHECK-OBJ: mmacc acc0, tr1, tr0 +mmacc acc0, tr1, tr0 + +# CHECK-ASM: mmacc acc3, tr2, tr1 # encoding: [0xab,0xa3,0x20,0x00] +# CHECK-OBJ: mmacc acc3, tr2, tr1 +mmacc acc3, tr2, tr1 + +# CHECK-ASM: msc acc0, (a2), a5 # encoding: [0x2b,0x12,0xf6,0x22] +# CHECK-OBJ: msc acc0, (a2), a5 +msc acc0, (a2), a5 + +# CHECK-ASM: msc acc3, (a2), a5 # encoding: [0xab,0x13,0xf6,0x22] +# CHECK-OBJ: msc acc3, (a2), a5 +msc acc3, (a2), a5 + +# CHECK-ASM: mlat tr1, (a0), a3 # encoding: [0xab,0x10,0xd5,0x40] +# CHECK-OBJ: mlat tr1, (a0), a3 +mlat tr1, (a0), a3 + +# CHECK-ASM: mlct acc1, (a0), a3 # encoding: [0xab,0x12,0xd5,0x60] +# CHECK-OBJ: mlct acc1, (a0), a3 +mlct acc1, (a0), a3 + +# CHECK-ASM: msa tr0, (a2), a5 # encoding: [0x2b,0x10,0xf6,0x02] +# CHECK-OBJ: msa tr0, (a2), a5 +msa tr0, (a2), a5 + +# CHECK-ASM: msb tr1, (a2), a5 # encoding: [0xab,0x10,0xf6,0x12] +# CHECK-OBJ: msb tr1, (a2), a5 +msb tr1, (a2), a5 + +# CHECK-ASM: msat tr2, (a2), a5 # encoding: [0x2b,0x11,0xf6,0x42] +# CHECK-OBJ: msat tr2, (a2), a5 +msat tr2, (a2), a5 + +# CHECK-ASM: msbt tr3, (a2), a5 # encoding: [0xab,0x11,0xf6,0x52] +# CHECK-OBJ: msbt tr3, (a2), a5 +msbt tr3, (a2), a5 + +# CHECK-ASM: msct acc0, (a2), a5 # encoding: [0x2b,0x12,0xf6,0x62] +# CHECK-OBJ: msct acc0, (a2), a5 +msct acc0, (a2), a5 + +# .whole loads/stores encode rs2 as zero. +# CHECK-ASM: mla.whole tr0, (a0) # encoding: [0x2b,0x10,0x05,0x80] +# CHECK-OBJ: mla.whole tr0, (a0) +mla.whole tr0, (a0) + +# CHECK-ASM: mlb.whole tr1, (a0) # encoding: [0xab,0x10,0x05,0x90] +# CHECK-OBJ: mlb.whole tr1, (a0) +mlb.whole tr1, (a0) + +# CHECK-ASM: mlc.whole acc0, (a0) # encoding: [0x2b,0x12,0x05,0xa0] +# CHECK-OBJ: mlc.whole acc0, (a0) +mlc.whole acc0, (a0) + +# CHECK-ASM: msa.whole tr0, (a2) # encoding: [0x2b,0x10,0x06,0x82] +# CHECK-OBJ: msa.whole tr0, (a2) +msa.whole tr0, (a2) + +# CHECK-ASM: msb.whole tr1, (a2) # encoding: [0xab,0x10,0x06,0x92] +# CHECK-OBJ: msb.whole tr1, (a2) +msb.whole tr1, (a2) + +# CHECK-ASM: msc.whole acc0, (a2) # encoding: [0x2b,0x12,0x06,0xa2] +# CHECK-OBJ: msc.whole acc0, (a2) +msc.whole acc0, (a2) diff --git a/llvm/test/MC/RISCV/zmasync-implies-zames.s b/llvm/test/MC/RISCV/zmasync-implies-zames.s new file mode 100644 index 00000000000000..66d631e4687f11 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync-implies-zames.s @@ -0,0 +1,11 @@ +# RUN: llvm-mc %s -triple=riscv64 -show-encoding | FileCheck %s + +# `zmasync` is a superset of `zames`: enabling it through `.option arch` +# must make ordinary ZAMES instructions available as well. +.option arch, rv64i_zmasync + +# CHECK: msettilemi 4 +msettilemi 4 + +# CHECK: msyncregreset sync0 +msyncregreset sync0 diff --git a/llvm/test/MC/RISCV/zmasync-invalid.s b/llvm/test/MC/RISCV/zmasync-invalid.s new file mode 100644 index 00000000000000..8aed77d984e777 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync-invalid.s @@ -0,0 +1,19 @@ +# RUN: not llvm-mc %s -triple=riscv64 -mattr=+zmasync 2>&1 | FileCheck %s + +# Wrong operand type. +msyncregreset a0 +# CHECK: error: +mrelease a0 +# CHECK: error: +mrelease sr0 +# CHECK: error: + +# Out-of-range sync registers. +msyncregreset sync16 +# CHECK: error: +macquire sync31, a0 +# CHECK: error: + +# Wrong operand type. +macquire sync0, sync1 +# CHECK: error: diff --git a/llvm/test/MC/RISCV/zmasync-no-feature.s b/llvm/test/MC/RISCV/zmasync-no-feature.s new file mode 100644 index 00000000000000..48193f3c83b214 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync-no-feature.s @@ -0,0 +1,18 @@ +# RUN: not llvm-mc %s -triple=riscv64 2>&1 | FileCheck %s --check-prefix=CHECK-NONE +# RUN: not llvm-mc %s -triple=riscv64 -mattr=+zames 2>&1 | FileCheck %s --check-prefix=CHECK-ZAMES + +# Every ZMASYNC mnemonic must be rejected when the extension is not enabled, +# including when only the base ZAMES extension is enabled. + +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +msyncregreset sync0 +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +mrelease sync1 +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +macquire sync15, a0 +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +mfence diff --git a/llvm/test/MC/RISCV/zmasync.s b/llvm/test/MC/RISCV/zmasync.s new file mode 100644 index 00000000000000..b923f70dae3940 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync.s @@ -0,0 +1,22 @@ +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zmasync -show-encoding | FileCheck %s --check-prefix=CHECK-ASM +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zmasync -filetype=obj | llvm-objdump --mattr=+zmasync -d - | FileCheck %s --check-prefix=CHECK-OBJ + +# CHECK-ASM: msyncregreset sync0 # encoding: [0x2b,0x40,0x00,0x80] +# CHECK-OBJ: msyncregreset sync0 +msyncregreset sync0 + +# CHECK-ASM: msyncregreset sync15 # encoding: [0x2b,0x40,0xf0,0x80] +# CHECK-OBJ: msyncregreset sync15 +msyncregreset sync15 + +# CHECK-ASM: mrelease sync1 # encoding: [0x2b,0x40,0x10,0x90] +# CHECK-OBJ: mrelease sync1 +mrelease sync1 + +# CHECK-ASM: macquire sync15, a0 # encoding: [0x2b,0x40,0xf5,0xa0] +# CHECK-OBJ: macquire sync15, a0 +macquire sync15, a0 + +# CHECK-ASM: mfence # encoding: [0x2b,0x40,0x00,0xb0] +# CHECK-OBJ: mfence +mfence diff --git a/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll new file mode 100644 index 00000000000000..551b092b36069c --- /dev/null +++ b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll @@ -0,0 +1,140 @@ +; Wrong signatures must be rejected by the verifier, proving the names are +; resolved as real intrinsics (an unregistered name would be accepted as an +; ordinary external function). + +; RUN: split-file %s %t +; RUN: not llvm-as %t/wrong-ret-settilem.ll -o /dev/null +; RUN: not llvm-as %t/whole-load-stride.ll -o /dev/null +; RUN: not llvm-as %t/acquire-wrong-ret.ll -o /dev/null +; RUN: not llvm-as %t/whole-store-stride.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-cfg.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-settilemi.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-settileni.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-settileki.ll -o /dev/null +; RUN: not llvm-as %t/range-settilemi-neg.ll -o /dev/null +; RUN: not llvm-as %t/range-settileni-1024.ll -o /dev/null +; RUN: not llvm-as %t/range-settileki-1024.ll -o /dev/null +; RUN: not llvm-as %t/range-msetcfg-8.ll -o /dev/null +; RUN: not llvm-as %t/range-mgetcfg-neg.ll -o /dev/null + +; Note: an overload suffix that disagrees with the argument types (e.g. +; msa.triscv.ame.tile.int8t taking an int32 tile) is NOT a verifier error — +; the parser re-mangles the name from the argument types. Signature +; violations must therefore probe arity/type/immarg constraints. + +;--- wrong-ret-settilem.ll +; msettilem returns void. +declare i64 @llvm.riscv.zames.msettilem(i64) + +define i64 @f(i64 %m) { + %r = call i64 @llvm.riscv.zames.msettilem(i64 %m) + ret i64 %r +} + +;--- whole-load-stride.ll +; .whole loads are pointer-only; a stride operand is not allowed. +declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr, i64) + +define target("riscv.ame.tile.int8") @f(ptr %p, i64 %s) { + %r = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %p, i64 %s) + ret target("riscv.ame.tile.int8") %r +} + +;--- acquire-wrong-ret.ll +; macquire returns void. +declare target("riscv.ame.sync") @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync"), i64) + +define target("riscv.ame.sync") @f(target("riscv.ame.sync") %s, i64 %sem) { + %r = call target("riscv.ame.sync") @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync") %s, i64 %sem) + ret target("riscv.ame.sync") %r +} + +;--- whole-store-stride.ll +; .whole stores take tile + pointer only. +declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) + +define void @f(target("riscv.ame.tile.int8") %t, ptr %p, i64 %s) { + call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %t, ptr %p, i64 %s) + ret void +} + +;--- nonimm-cfg.ll +; The msetcfg selector operand is an ImmArg. +declare void @llvm.riscv.zames.msetcfg(i64, i64) + +define void @f(i64 %sel, i64 %payload) { + call void @llvm.riscv.zames.msetcfg(i64 %sel, i64 %payload) + ret void +} + +;--- nonimm-settilemi.ll +; The immediate tile setters take a constant uimm10; a dynamic value is an +; ImmArg violation. Range selection (register form for >=1024) is done by +; the Buddy lowering, not the verifier. +declare void @llvm.riscv.zames.msettilemi(i64) + +define void @f(i64 %m) { + call void @llvm.riscv.zames.msettilemi(i64 %m) + ret void +} + +;--- nonimm-settileni.ll +declare void @llvm.riscv.zames.msettileni(i64) + +define void @f(i64 %n) { + call void @llvm.riscv.zames.msettileni(i64 %n) + ret void +} + +;--- nonimm-settileki.ll +declare void @llvm.riscv.zames.msettileki(i64) + +define void @f(i64 %k) { + call void @llvm.riscv.zames.msettileki(i64 %k) + ret void +} + +;--- range-settilemi-neg.ll +; The immediate tile setters encode a uimm10 field: the LLVM contract is the +; half-open range [0, 1024). Out-of-range constants must be rejected even +; though they are immediates. +declare void @llvm.riscv.zames.msettilemi(i64) + +define void @f() { + call void @llvm.riscv.zames.msettilemi(i64 -1) + ret void +} + +;--- range-settileni-1024.ll +declare void @llvm.riscv.zames.msettileni(i64) + +define void @f() { + call void @llvm.riscv.zames.msettileni(i64 1024) + ret void +} + +;--- range-settileki-1024.ll +declare void @llvm.riscv.zames.msettileki(i64) + +define void @f() { + call void @llvm.riscv.zames.msettileki(i64 1024) + ret void +} + +;--- range-msetcfg-8.ll +; The config selector is a 3-bit field: the LLVM contract is [0, 8). The +; payload argument is a full GPR value and is intentionally unconstrained. +declare void @llvm.riscv.zames.msetcfg(i64, i64) + +define void @f(i64 %payload) { + call void @llvm.riscv.zames.msetcfg(i64 8, i64 %payload) + ret void +} + +;--- range-mgetcfg-neg.ll +declare i64 @llvm.riscv.zames.mgetcfg(i64) + +define i64 @f() { + %r = call i64 @llvm.riscv.zames.mgetcfg(i64 -1) + ret i64 %r +} From 006120a6923eeb260468bddd616857619e830f3a Mon Sep 17 00:00:00 2001 From: jiaxiaoyu Date: Wed, 4 Nov 2026 17:36:02 +0800 Subject: [PATCH 2/6] [RISCV] Namespace native ZAMES and ZMASYNC interfaces under XSAI Use XSAI-prefixed target-extension types, MVTs, intrinsics, registers, opcodes, pseudos and TableGen filenames for the native matrix backend. Migrate the implementing consumers and IR/MIR checks together without retaining legacy aliases. Keep architectural features, assembly spellings and instruction encodings unchanged. Leave target type storage properties and sync memory effects for separate review. No numerical change is intended. Validation: 28 targeted LLVM/MC/IR/driver/MLIR tests passed. Representative integer, floating-point and sync codegen text sections are byte-identical; storage and O2/O3 sync-ordering non-change smokes passed. --- llvm/include/llvm/CodeGen/ValueTypes.td | 16 +- llvm/include/llvm/IR/IntrinsicsRISCV.td | 4 +- .../IR/IntrinsicsRISCVXSAIZamesZmasync.td | 107 +++ .../llvm/IR/IntrinsicsRISCVZamesZmasync.td | 107 --- llvm/lib/CodeGen/ValueTypes.cpp | 96 +-- llvm/lib/IR/Type.cpp | 27 +- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 18 +- .../RISCV/Disassembler/RISCVDisassembler.cpp | 37 +- .../RISCV/MCTargetDesc/RISCVInstPrinter.cpp | 6 +- .../RISCV/MCTargetDesc/RISCVInstPrinter.h | 4 +- llvm/lib/Target/RISCV/RISCV.td | 2 +- .../Target/RISCV/RISCVExpandPseudoPreEmit.cpp | 374 +++++---- .../Target/RISCV/RISCVExpandPseudoPreRA.cpp | 690 ++++++++------- llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h | 4 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 101 ++- llvm/lib/Target/RISCV/RISCVInstrInfo.td | 4 +- ...nfoZames.td => RISCVInstrInfoXSAIZames.td} | 150 ++-- .../RISCV/RISCVInstrInfoXSAIZamesPatterns.td | 594 +++++++++++++ .../RISCV/RISCVInstrInfoZamesPatterns.td | 595 ------------- .../RISCV/RISCVRegisterInfoXSAIZames.td | 96 +++ .../Target/RISCV/RISCVRegisterInfoZames.td | 96 --- llvm/test/Assembler/riscv-zames-intrinsics.ll | 166 ++-- .../CodeGen/RISCV/zames/rv32-unsupported.ll | 16 +- .../test/CodeGen/RISCV/zames/zames-codegen.ll | 172 ++-- .../CodeGen/RISCV/zames/zames-fp-expand.mir | 36 +- .../zames/zames-fp-types-negative-acc.ll | 20 +- .../RISCV/zames/zames-fp-types-negative.ll | 22 +- .../CodeGen/RISCV/zames/zames-fp-types.ll | 792 +++++++++--------- .../zames/zames-immediate-range-invalid.ll | 20 +- .../CodeGen/RISCV/zames/zames-isel-pseudos.ll | 152 ++-- .../RISCV/zames/zames-memory-operands.ll | 182 ++-- .../CodeGen/RISCV/zames/zames-no-feature.ll | 8 +- .../RISCV/zames/zames-prera-cfg-pseudos.ll | 156 ++-- .../CodeGen/RISCV/zames/zmasync-codegen.ll | 16 +- .../riscv-zames-intrinsics-invalid.ll | 64 +- 35 files changed, 2506 insertions(+), 2444 deletions(-) create mode 100644 llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td delete mode 100644 llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td rename llvm/lib/Target/RISCV/{RISCVInstrInfoZames.td => RISCVInstrInfoXSAIZames.td} (58%) create mode 100644 llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td delete mode 100644 llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td create mode 100644 llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td delete mode 100644 llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td index b20f67a68699fe..76eb92a55980ed 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.td +++ b/llvm/include/llvm/CodeGen/ValueTypes.td @@ -409,14 +409,14 @@ def amdgpuBufferStridedPointer : ValueType<192>; def aarch64mfp8 : ValueType<8>; // 8-bit value in FPR (AArch64) -def riscv_ame_tile_i8 : ValueType<0>; // RISC-V AME/ZAMES tile with i8 elements -def riscv_ame_tile_i32 : ValueType<0>; // RISC-V AME/ZAMES tile with i32 elements -def riscv_ame_sync : ValueType<0>; // RISC-V Zmasync sync register -def riscv_ame_tile_fp8e4 : ValueType<0>; // RISC-V ZAMES tile with fp8e4m3 elements -def riscv_ame_tile_fp8e5 : ValueType<0>; // RISC-V ZAMES tile with fp8e5m2 elements -def riscv_ame_tile_f16 : ValueType<0>; // RISC-V ZAMES tile with fp16 elements -def riscv_ame_tile_bf16 : ValueType<0>; // RISC-V ZAMES tile with bf16 elements -def riscv_ame_tile_f32 : ValueType<0>; // RISC-V ZAMES tile with fp32 elements +def riscv_xsai_ame_tile_i8 : ValueType<0>; // RISC-V AME/ZAMES tile with i8 elements +def riscv_xsai_ame_tile_i32 : ValueType<0>; // RISC-V AME/ZAMES tile with i32 elements +def riscv_xsai_ame_sync : ValueType<0>; // RISC-V Zmasync sync register +def riscv_xsai_ame_tile_fp8e4 : ValueType<0>; // RISC-V ZAMES tile with fp8e4m3 elements +def riscv_xsai_ame_tile_fp8e5 : ValueType<0>; // RISC-V ZAMES tile with fp8e5m2 elements +def riscv_xsai_ame_tile_f16 : ValueType<0>; // RISC-V ZAMES tile with fp16 elements +def riscv_xsai_ame_tile_bf16 : ValueType<0>; // RISC-V ZAMES tile with bf16 elements +def riscv_xsai_ame_tile_f32 : ValueType<0>; // RISC-V ZAMES tile with fp32 elements // CHERI capabilities. Pointer-like values that carry additional metadata // for enforcing safety guarantees on CHERI-enabled targets. diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 28e7325e70d25d..89425d0435ac6e 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2499,6 +2499,6 @@ include "llvm/IR/IntrinsicsRISCVBOSCExt.td" include "llvm/IR/IntrinsicsRISCVXTExt.td" include "llvm/IR/IntrinsicsRISCVBuckyballExt.td" -// ZAMES/ZMASYNC extensions +// XSAI ZAMES/ZMASYNC extensions //===----------------------------------------------------------------------===// -include "llvm/IR/IntrinsicsRISCVZamesZmasync.td" +include "llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td" diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td b/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td new file mode 100644 index 00000000000000..4f9493e7742d23 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td @@ -0,0 +1,107 @@ +//===- IntrinsicsRISCVXSAIZamesZmasync.td - Zames/Zmasync intrinsics -*- tablegen -*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in { + class XSAI_ZAMES_SetCfgIntrinsic + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 8>]>; + class XSAI_ZAMES_GetCfgIntrinsic + : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 8>]>; + class XSAI_ZAMES_SetTileIntrinsic + : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 1024>]>; + class XSAI_ZAMES_SetTileRegIntrinsic + : Intrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZAMES_InitIntrinsic + : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; + + class XSAI_ZAMES_MZeroIntrinsic + : Intrinsic<[llvm_any_ty], [], [IntrNoMem]>; + class XSAI_ZAMES_LoadIntrinsic + : Intrinsic<[llvm_any_ty], [llvm_ptr_ty, llvm_i64_ty], + [NoCapture>, IntrReadMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_StoreIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty, llvm_i64_ty], + [NoCapture>, IntrWriteMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_WholeLoadIntrinsic + : Intrinsic<[llvm_any_ty], [llvm_ptr_ty], + [NoCapture>, IntrReadMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_WholeStoreIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty], + [NoCapture>, IntrWriteMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_MMAccIntrinsic + : Intrinsic<[llvm_any_ty], + [llvm_any_ty, llvm_any_ty, LLVMMatchType<0>], + [IntrNoMem]>; + + def int_riscv_xsai_zames_msetcfg : XSAI_ZAMES_SetCfgIntrinsic; + def int_riscv_xsai_zames_mgetcfg : XSAI_ZAMES_GetCfgIntrinsic; + def int_riscv_xsai_zames_msettilemi : XSAI_ZAMES_SetTileIntrinsic; + def int_riscv_xsai_zames_msettileni : XSAI_ZAMES_SetTileIntrinsic; + def int_riscv_xsai_zames_msettileki : XSAI_ZAMES_SetTileIntrinsic; + def int_riscv_xsai_zames_msettilem : XSAI_ZAMES_SetTileRegIntrinsic; + def int_riscv_xsai_zames_msettilen : XSAI_ZAMES_SetTileRegIntrinsic; + def int_riscv_xsai_zames_msettilek : XSAI_ZAMES_SetTileRegIntrinsic; + + def int_riscv_xsai_zames_minit : XSAI_ZAMES_InitIntrinsic; + + def int_riscv_xsai_zames_mzero : XSAI_ZAMES_MZeroIntrinsic; + def int_riscv_xsai_zames_mmacc : XSAI_ZAMES_MMAccIntrinsic; + + def int_riscv_xsai_zames_mla : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlb : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlc : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlat : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlbt : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlct : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mla_whole : XSAI_ZAMES_WholeLoadIntrinsic; + def int_riscv_xsai_zames_mlb_whole : XSAI_ZAMES_WholeLoadIntrinsic; + def int_riscv_xsai_zames_mlc_whole : XSAI_ZAMES_WholeLoadIntrinsic; + + def int_riscv_xsai_zames_msa : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msb : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msc : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msat : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msbt : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msct : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msa_whole : XSAI_ZAMES_WholeStoreIntrinsic; + def int_riscv_xsai_zames_msb_whole : XSAI_ZAMES_WholeStoreIntrinsic; + def int_riscv_xsai_zames_msc_whole : XSAI_ZAMES_WholeStoreIntrinsic; + + class XSAI_ZMASYNC_SyncResetIntrinsic + : Intrinsic<[llvm_any_ty], [], [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZMASYNC_ReleaseIntrinsic + : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], + [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZMASYNC_AcquireIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZMASYNC_FenceIntrinsic + : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; + + def int_riscv_xsai_zmasync_msyncregreset : XSAI_ZMASYNC_SyncResetIntrinsic; + def int_riscv_xsai_zmasync_mrelease : XSAI_ZMASYNC_ReleaseIntrinsic; + def int_riscv_xsai_zmasync_macquire : XSAI_ZMASYNC_AcquireIntrinsic; + def int_riscv_xsai_zmasync_mfence : XSAI_ZMASYNC_FenceIntrinsic; +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td b/llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td deleted file mode 100644 index 75c39337a6d703..00000000000000 --- a/llvm/include/llvm/IR/IntrinsicsRISCVZamesZmasync.td +++ /dev/null @@ -1,107 +0,0 @@ -//===- IntrinsicsRISCVZamesZmasync.td - Zames/Zmasync intrinsics -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -let TargetPrefix = "riscv" in { - class ZAMESSetCfgIntrinsic - : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], - [IntrNoMem, IntrHasSideEffects, ImmArg>, - Range, 0, 8>]>; - class ZAMESGetCfgIntrinsic - : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], - [IntrNoMem, IntrHasSideEffects, ImmArg>, - Range, 0, 8>]>; - class ZAMESSetTileIntrinsic - : Intrinsic<[], [llvm_i64_ty], - [IntrNoMem, IntrHasSideEffects, ImmArg>, - Range, 0, 1024>]>; - class ZAMESSetTileRegIntrinsic - : Intrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>; - class ZAMESInitIntrinsic - : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; - - class ZAMESMZeroIntrinsic - : Intrinsic<[llvm_any_ty], [], [IntrNoMem]>; - class ZAMESLoadIntrinsic - : Intrinsic<[llvm_any_ty], [llvm_ptr_ty, llvm_i64_ty], - [NoCapture>, IntrReadMem], "", - [SDNPMemOperand]>; - class ZAMESStoreIntrinsic - : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty, llvm_i64_ty], - [NoCapture>, IntrWriteMem], "", - [SDNPMemOperand]>; - class ZAMESWholeLoadIntrinsic - : Intrinsic<[llvm_any_ty], [llvm_ptr_ty], - [NoCapture>, IntrReadMem], "", - [SDNPMemOperand]>; - class ZAMESWholeStoreIntrinsic - : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty], - [NoCapture>, IntrWriteMem], "", - [SDNPMemOperand]>; - class ZAMESMMAccIntrinsic - : Intrinsic<[llvm_any_ty], - [llvm_any_ty, llvm_any_ty, LLVMMatchType<0>], - [IntrNoMem]>; - - def int_riscv_zames_msetcfg : ZAMESSetCfgIntrinsic; - def int_riscv_zames_mgetcfg : ZAMESGetCfgIntrinsic; - def int_riscv_zames_msettilemi : ZAMESSetTileIntrinsic; - def int_riscv_zames_msettileni : ZAMESSetTileIntrinsic; - def int_riscv_zames_msettileki : ZAMESSetTileIntrinsic; - def int_riscv_zames_msettilem : ZAMESSetTileRegIntrinsic; - def int_riscv_zames_msettilen : ZAMESSetTileRegIntrinsic; - def int_riscv_zames_msettilek : ZAMESSetTileRegIntrinsic; - - def int_riscv_zames_minit : ZAMESInitIntrinsic; - - def int_riscv_zames_mzero : ZAMESMZeroIntrinsic; - def int_riscv_zames_mmacc : ZAMESMMAccIntrinsic; - - def int_riscv_zames_mla : ZAMESLoadIntrinsic; - def int_riscv_zames_mlb : ZAMESLoadIntrinsic; - def int_riscv_zames_mlc : ZAMESLoadIntrinsic; - def int_riscv_zames_mlat : ZAMESLoadIntrinsic; - def int_riscv_zames_mlbt : ZAMESLoadIntrinsic; - def int_riscv_zames_mlct : ZAMESLoadIntrinsic; - def int_riscv_zames_mla_whole : ZAMESWholeLoadIntrinsic; - def int_riscv_zames_mlb_whole : ZAMESWholeLoadIntrinsic; - def int_riscv_zames_mlc_whole : ZAMESWholeLoadIntrinsic; - - def int_riscv_zames_msa : ZAMESStoreIntrinsic; - def int_riscv_zames_msb : ZAMESStoreIntrinsic; - def int_riscv_zames_msc : ZAMESStoreIntrinsic; - def int_riscv_zames_msat : ZAMESStoreIntrinsic; - def int_riscv_zames_msbt : ZAMESStoreIntrinsic; - def int_riscv_zames_msct : ZAMESStoreIntrinsic; - def int_riscv_zames_msa_whole : ZAMESWholeStoreIntrinsic; - def int_riscv_zames_msb_whole : ZAMESWholeStoreIntrinsic; - def int_riscv_zames_msc_whole : ZAMESWholeStoreIntrinsic; - - class ZMASYNCSyncResetIntrinsic - : Intrinsic<[llvm_any_ty], [], [IntrNoMem, IntrHasSideEffects]>; - class ZMASYNCReleaseIntrinsic - : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], - [IntrNoMem, IntrHasSideEffects]>; - class ZMASYNCAcquireIntrinsic - : Intrinsic<[], [llvm_any_ty, llvm_i64_ty], - [IntrNoMem, IntrHasSideEffects]>; - class ZMASYNCFenceIntrinsic - : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; - - def int_riscv_zmasync_msyncregreset : ZMASYNCSyncResetIntrinsic; - def int_riscv_zmasync_mrelease : ZMASYNCReleaseIntrinsic; - def int_riscv_zmasync_macquire : ZMASYNCAcquireIntrinsic; - def int_riscv_zmasync_mfence : ZMASYNCFenceIntrinsic; -} diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp index db3d2e08be4659..046b239ff97b8e 100644 --- a/llvm/lib/CodeGen/ValueTypes.cpp +++ b/llvm/lib/CodeGen/ValueTypes.cpp @@ -242,29 +242,29 @@ void EVT::print(raw_ostream &OS) const { case MVT::aarch64mfp8: OS << "aarch64mfp8"; return; - case MVT::riscv_ame_tile_i8: - OS << "riscv_ame_tile_i8"; + case MVT::riscv_xsai_ame_tile_i8: + OS << "riscv_xsai_ame_tile_i8"; return; - case MVT::riscv_ame_tile_i32: - OS << "riscv_ame_tile_i32"; + case MVT::riscv_xsai_ame_tile_i32: + OS << "riscv_xsai_ame_tile_i32"; return; - case MVT::riscv_ame_sync: - OS << "riscv_ame_sync"; + case MVT::riscv_xsai_ame_sync: + OS << "riscv_xsai_ame_sync"; return; - case MVT::riscv_ame_tile_fp8e4: - OS << "riscv_ame_tile_fp8e4"; + case MVT::riscv_xsai_ame_tile_fp8e4: + OS << "riscv_xsai_ame_tile_fp8e4"; return; - case MVT::riscv_ame_tile_fp8e5: - OS << "riscv_ame_tile_fp8e5"; + case MVT::riscv_xsai_ame_tile_fp8e5: + OS << "riscv_xsai_ame_tile_fp8e5"; return; - case MVT::riscv_ame_tile_f16: - OS << "riscv_ame_tile_f16"; + case MVT::riscv_xsai_ame_tile_f16: + OS << "riscv_xsai_ame_tile_f16"; return; - case MVT::riscv_ame_tile_bf16: - OS << "riscv_ame_tile_bf16"; + case MVT::riscv_xsai_ame_tile_bf16: + OS << "riscv_xsai_ame_tile_bf16"; return; - case MVT::riscv_ame_tile_f32: - OS << "riscv_ame_tile_f32"; + case MVT::riscv_xsai_ame_tile_f32: + OS << "riscv_xsai_ame_tile_f32"; return; } } @@ -291,22 +291,22 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const { return TargetExtType::get(Context, "aarch64.svcount"); case MVT::aarch64mfp8: return FixedVectorType::get(IntegerType::get(Context, 8), 1); - case MVT::riscv_ame_tile_i8: - return TargetExtType::get(Context, "riscv.ame.tile.int8"); - case MVT::riscv_ame_tile_i32: - return TargetExtType::get(Context, "riscv.ame.tile.int32"); - case MVT::riscv_ame_sync: - return TargetExtType::get(Context, "riscv.ame.sync"); - case MVT::riscv_ame_tile_fp8e4: - return TargetExtType::get(Context, "riscv.ame.tile.fp8e4"); - case MVT::riscv_ame_tile_fp8e5: - return TargetExtType::get(Context, "riscv.ame.tile.fp8e5"); - case MVT::riscv_ame_tile_f16: - return TargetExtType::get(Context, "riscv.ame.tile.fp16"); - case MVT::riscv_ame_tile_bf16: - return TargetExtType::get(Context, "riscv.ame.tile.bf16"); - case MVT::riscv_ame_tile_f32: - return TargetExtType::get(Context, "riscv.ame.tile.fp32"); + case MVT::riscv_xsai_ame_tile_i8: + return TargetExtType::get(Context, "riscv.xsai.ame.tile.int8"); + case MVT::riscv_xsai_ame_tile_i32: + return TargetExtType::get(Context, "riscv.xsai.ame.tile.int32"); + case MVT::riscv_xsai_ame_sync: + return TargetExtType::get(Context, "riscv.xsai.ame.sync"); + case MVT::riscv_xsai_ame_tile_fp8e4: + return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp8e4"); + case MVT::riscv_xsai_ame_tile_fp8e5: + return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp8e5"); + case MVT::riscv_xsai_ame_tile_f16: + return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp16"); + case MVT::riscv_xsai_ame_tile_bf16: + return TargetExtType::get(Context, "riscv.xsai.ame.tile.bf16"); + case MVT::riscv_xsai_ame_tile_f32: + return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp32"); case MVT::x86amx: return Type::getX86_AMXTy(Context); case MVT::i64x8: return IntegerType::get(Context, 512); case MVT::amdgpuBufferFatPointer: return IntegerType::get(Context, 160); @@ -362,22 +362,22 @@ MVT MVT::getVT(Type *Ty, bool HandleUnknown){ return MVT::getRISCVVectorTupleVT(Sz * NF, NF); } - if (TargetExtTy->getName() == "riscv.ame.tile.int8") - return MVT(MVT::riscv_ame_tile_i8); - if (TargetExtTy->getName() == "riscv.ame.tile.int32") - return MVT(MVT::riscv_ame_tile_i32); - if (TargetExtTy->getName() == "riscv.ame.sync") - return MVT(MVT::riscv_ame_sync); - if (TargetExtTy->getName() == "riscv.ame.tile.fp8e4") - return MVT(MVT::riscv_ame_tile_fp8e4); - if (TargetExtTy->getName() == "riscv.ame.tile.fp8e5") - return MVT(MVT::riscv_ame_tile_fp8e5); - if (TargetExtTy->getName() == "riscv.ame.tile.fp16") - return MVT(MVT::riscv_ame_tile_f16); - if (TargetExtTy->getName() == "riscv.ame.tile.bf16") - return MVT(MVT::riscv_ame_tile_bf16); - if (TargetExtTy->getName() == "riscv.ame.tile.fp32") - return MVT(MVT::riscv_ame_tile_f32); + if (TargetExtTy->getName() == "riscv.xsai.ame.tile.int8") + return MVT(MVT::riscv_xsai_ame_tile_i8); + if (TargetExtTy->getName() == "riscv.xsai.ame.tile.int32") + return MVT(MVT::riscv_xsai_ame_tile_i32); + if (TargetExtTy->getName() == "riscv.xsai.ame.sync") + return MVT(MVT::riscv_xsai_ame_sync); + if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp8e4") + return MVT(MVT::riscv_xsai_ame_tile_fp8e4); + if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp8e5") + return MVT(MVT::riscv_xsai_ame_tile_fp8e5); + if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp16") + return MVT(MVT::riscv_xsai_ame_tile_f16); + if (TargetExtTy->getName() == "riscv.xsai.ame.tile.bf16") + return MVT(MVT::riscv_xsai_ame_tile_bf16); + if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp32") + return MVT(MVT::riscv_xsai_ame_tile_f32); if (HandleUnknown) return MVT(MVT::Other); llvm_unreachable("Unknown target ext type!"); diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index b0cbe17f772c59..73f495161e2b84 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -981,13 +981,14 @@ Expected TargetExtType::checkParams(TargetExtType *TTy) { "target extension type riscv.vector.tuple should have one " "type parameter and one integer parameter"); - if ((TTy->Name == "riscv.ame.tile.int8" || - TTy->Name == "riscv.ame.tile.int32" || - TTy->Name == "riscv.ame.tile.fp8e4" || - TTy->Name == "riscv.ame.tile.fp8e5" || - TTy->Name == "riscv.ame.tile.fp16" || - TTy->Name == "riscv.ame.tile.bf16" || - TTy->Name == "riscv.ame.tile.fp32" || TTy->Name == "riscv.ame.sync") && + if ((TTy->Name == "riscv.xsai.ame.tile.int8" || + TTy->Name == "riscv.xsai.ame.tile.int32" || + TTy->Name == "riscv.xsai.ame.tile.fp8e4" || + TTy->Name == "riscv.xsai.ame.tile.fp8e5" || + TTy->Name == "riscv.xsai.ame.tile.fp16" || + TTy->Name == "riscv.xsai.ame.tile.bf16" || + TTy->Name == "riscv.xsai.ame.tile.fp32" || + TTy->Name == "riscv.xsai.ame.sync") && (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 0)) return createStringError("target extension type " + TTy->Name + " should have no parameters"); @@ -1088,15 +1089,17 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); } - if (Name == "riscv.ame.tile.int8" || Name == "riscv.ame.tile.int32" || - Name == "riscv.ame.tile.fp8e4" || Name == "riscv.ame.tile.fp8e5" || - Name == "riscv.ame.tile.fp16" || Name == "riscv.ame.tile.bf16" || - Name == "riscv.ame.tile.fp32") + if (Name == "riscv.xsai.ame.tile.int8" || + Name == "riscv.xsai.ame.tile.int32" || + Name == "riscv.xsai.ame.tile.fp8e4" || + Name == "riscv.xsai.ame.tile.fp8e5" || + Name == "riscv.xsai.ame.tile.fp16" || + Name == "riscv.xsai.ame.tile.bf16" || Name == "riscv.xsai.ame.tile.fp32") return TargetTypeInfo(FixedVectorType::get(Type::getInt8Ty(C), 16), TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); - if (Name == "riscv.ame.sync") + if (Name == "riscv.xsai.ame.sync") return TargetTypeInfo(Type::getInt64Ty(C), TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 172dc5c9cad1a4..a83183b1abc222 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -240,7 +240,7 @@ class RISCVAsmParser : public MCTargetAsmParser { } ParseStatus parseRegReg(OperandVector &Operands); - ParseStatus parseMCFGSel(OperandVector &Operands); + ParseStatus parseXSAIZamesMCFGSel(OperandVector &Operands); ParseStatus parseXSfmmVType(OperandVector &Operands); ParseStatus parseZcmpStackAdj(OperandVector &Operands, bool ExpectNegative = false); @@ -762,7 +762,7 @@ struct RISCVOperand final : public MCParsedAsmOperand { return IsConstantImm && isUInt(Imm); } - bool isMCFGSel() const { return isUImm<3>(); } + bool isXSAIZamesMCFGSel() const { return isUImm<3>(); } template bool isUImmShifted() const { int64_t Imm; @@ -1433,14 +1433,14 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp, // ZAMES shares assembly names with BOSC AME, but has distinct register // records and accumulator encodings. Coerce only for a ZAMES operand class. - if ((Kind == MCK_ZamesTileReg || Kind == MCK_ZamesTileOrAccReg) && + if ((Kind == MCK_XSAI_ZAMES_TileReg || Kind == MCK_XSAI_ZAMES_TileOrAccReg) && Reg >= RISCV::TR0 && Reg <= RISCV::TR3) { - Op.Reg.Reg = RISCV::ZAMES_TR0 + (Reg - RISCV::TR0); + Op.Reg.Reg = RISCV::XSAI_ZAMES_TR0 + (Reg - RISCV::TR0); return Match_Success; } - if ((Kind == MCK_ZamesAccReg || Kind == MCK_ZamesTileOrAccReg) && + if ((Kind == MCK_XSAI_ZAMES_AccReg || Kind == MCK_XSAI_ZAMES_TileOrAccReg) && Reg >= RISCV::ACC0 && Reg <= RISCV::ACC3) { - Op.Reg.Reg = RISCV::ZAMES_ACC0 + (Reg - RISCV::ACC0); + Op.Reg.Reg = RISCV::XSAI_ZAMES_ACC0 + (Reg - RISCV::ACC0); return Match_Success; } @@ -1905,9 +1905,9 @@ MCRegister RISCVAsmParser::matchRegisterNameHelper(StringRef Name) const { static_assert(RISCV::F0_D < RISCV::F0_Q, "FPR matching must be updated"); // Shared matrix names initially resolve to BOSC AME; operand validation // converts them to ZAMES registers only when the instruction requires it. - static_assert(RISCV::TR0 < RISCV::ZAMES_TR0, + static_assert(RISCV::TR0 < RISCV::XSAI_ZAMES_TR0, "Matrix register matching must be updated"); - static_assert(RISCV::ACC0 < RISCV::ZAMES_ACC0, + static_assert(RISCV::ACC0 < RISCV::XSAI_ZAMES_ACC0, "Matrix register matching must be updated"); if (!Reg) Reg = MatchRegisterAltName(Name); @@ -2595,7 +2595,7 @@ bool RISCVAsmParser::generateVTypeError(SMLoc ErrorLoc) { "e[8|8alt|16|16alt|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]"); } -ParseStatus RISCVAsmParser::parseMCFGSel(OperandVector &Operands) { +ParseStatus RISCVAsmParser::parseXSAIZamesMCFGSel(OperandVector &Operands) { const AsmToken &Tok = getLexer().getTok(); if (!Tok.is(AsmToken::Identifier)) return ParseStatus::NoMatch; diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp index 1fb3cbdc3e6277..1a96878d4f1e63 100644 --- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp +++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp @@ -319,49 +319,52 @@ static DecodeStatus DecodeYBNDSWImm(MCInst &Inst, uint64_t Imm, int64_t Address, } static DecodeStatus -DecodeZamesTileRegRegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, - const MCDisassembler *Decoder) { +DecodeXSAI_ZAMES_TileRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { if (RegNo >= 4) return MCDisassembler::Fail; - MCRegister Reg = RISCV::ZAMES_TR0 + RegNo; + MCRegister Reg = RISCV::XSAI_ZAMES_TR0 + RegNo; Inst.addOperand(MCOperand::createReg(Reg)); return MCDisassembler::Success; } static DecodeStatus -DecodeZamesAccRegRegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, - const MCDisassembler *Decoder) { +DecodeXSAI_ZAMES_AccRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { if (RegNo < 4 || RegNo >= 8) return MCDisassembler::Fail; - MCRegister Reg = RISCV::ZAMES_ACC0 + (RegNo - 4); + MCRegister Reg = RISCV::XSAI_ZAMES_ACC0 + (RegNo - 4); Inst.addOperand(MCOperand::createReg(Reg)); return MCDisassembler::Success; } -// Union of ZamesTileReg (0-3) and ZamesAccReg (4-7) for real instructions whose -// md field accepts either register file (e.g. mzero). +// Union of XSAI_ZAMES_TileReg (0-3) and XSAI_ZAMES_AccReg (4-7) for real +// instructions whose md field accepts either register file (e.g. mzero). static DecodeStatus -DecodeZamesTileOrAccRegRegisterClass(MCInst &Inst, uint32_t RegNo, - uint64_t Address, - const MCDisassembler *Decoder) { +DecodeXSAI_ZAMES_TileOrAccRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { if (RegNo >= 8) return MCDisassembler::Fail; - MCRegister Reg = - RegNo < 4 ? RISCV::ZAMES_TR0 + RegNo : RISCV::ZAMES_ACC0 + (RegNo - 4); + MCRegister Reg = RegNo < 4 ? RISCV::XSAI_ZAMES_TR0 + RegNo + : RISCV::XSAI_ZAMES_ACC0 + (RegNo - 4); Inst.addOperand(MCOperand::createReg(Reg)); return MCDisassembler::Success; } -static DecodeStatus DecodeSyncRRegisterClass(MCInst &Inst, uint32_t RegNo, - uint64_t Address, - const MCDisassembler *Decoder) { +static DecodeStatus +DecodeXSAI_ZMASYNC_SyncRRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { if (RegNo >= 16) return MCDisassembler::Fail; - MCRegister Reg = RISCV::SYNC0 + RegNo; + MCRegister Reg = RISCV::XSAI_ZMASYNC_SYNC0 + RegNo; Inst.addOperand(MCOperand::createReg(Reg)); return MCDisassembler::Success; } diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp index 8f5a467711dbe2..ae4f3aa437f0d3 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp @@ -259,9 +259,9 @@ void RISCVInstPrinter::printXSfmmVType(const MCInst *MI, unsigned OpNo, O << ", w" << Widen; } -void RISCVInstPrinter::printMCFGSel(const MCInst *MI, unsigned OpNo, - const MCSubtargetInfo &STI, - raw_ostream &O) { +void RISCVInstPrinter::printXSAIZamesMCFGSel(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, + raw_ostream &O) { unsigned Imm = MI->getOperand(OpNo).getImm(); assert(Imm < 8 && "Invalid MCFG selector immediate"); markup(O, Markup::Register) << "mcfg" << Imm; diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h index 66031cb37da763..6e75a49d0bd41e 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h @@ -54,8 +54,8 @@ class RISCVInstPrinter : public MCInstPrinter { raw_ostream &O); void printXSfmmVType(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); - void printMCFGSel(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, - raw_ostream &O); + void printXSAIZamesMCFGSel(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O); void printVMaskReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); void printMatrixReg(const MCInst *MI, unsigned OpNo, diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index 08fc2bd4cf2afc..cfaad3cc603636 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -31,7 +31,7 @@ include "RISCVSystemOperands.td" //===----------------------------------------------------------------------===// include "RISCVRegisterInfo.td" -include "RISCVRegisterInfoZames.td" +include "RISCVRegisterInfoXSAIZames.td" include "RISCVSchedule.td" include "RISCVCallingConv.td" include "RISCVInstrInfo.td" diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp index d5c9c71bfc70c1..9f9ca9bd2054c6 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp @@ -64,8 +64,8 @@ class RISCVExpandPseudoPreEmitImpl final : public RISCVExpandPseudoImplBase { bool expandBOSCAMEOrderedTile(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; - bool expandZamesMatrixPseudo(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI) const; + bool expandXSAIZamesMatrixPseudo(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; }; class RISCVExpandPseudoPreEmitLegacy : public MachineFunctionPass { @@ -181,68 +181,68 @@ bool RISCVExpandPseudoPreEmitImpl::expandMI( return expandPseudoReadVLENBViaVSETVLIX0(MBB, MBBI); case RISCV::PseudoClearFPR64: return expandPseudoClearFPR64(MBB, MBBI); - case RISCV::PSEUDO_ZAMES_MLA_A_I8: - case RISCV::PSEUDO_ZAMES_MLB_B_I8: - case RISCV::PSEUDO_ZAMES_MLBT_B_I8: - case RISCV::PSEUDO_ZAMES_MLAT_A_I8: - case RISCV::PSEUDO_ZAMES_MLC_C_I32: - case RISCV::PSEUDO_ZAMES_MLCT_C_I32: - case RISCV::PSEUDO_ZAMES_MZERO_C_I32: - case RISCV::PSEUDO_ZAMES_MMACC_I8_I32: - case RISCV::PSEUDO_ZAMES_MSC_C_I32: - case RISCV::PSEUDO_ZAMES_MSA_A_I8: - case RISCV::PSEUDO_ZAMES_MSAT_A_I8: - case RISCV::PSEUDO_ZAMES_MSB_B_I8: - case RISCV::PSEUDO_ZAMES_MSBT_B_I8: - case RISCV::PSEUDO_ZAMES_MSCT_C_I32: - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MLA_A_F16: - case RISCV::PSEUDO_ZAMES_MLA_A_BF16: - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MLAT_A_F16: - case RISCV::PSEUDO_ZAMES_MLAT_A_BF16: - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MLB_B_F16: - case RISCV::PSEUDO_ZAMES_MLB_B_BF16: - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MLBT_B_F16: - case RISCV::PSEUDO_ZAMES_MLBT_B_BF16: - case RISCV::PSEUDO_ZAMES_MLC_C_F32: - case RISCV::PSEUDO_ZAMES_MLCT_C_F32: - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MSA_A_F16: - case RISCV::PSEUDO_ZAMES_MSA_A_BF16: - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MSAT_A_F16: - case RISCV::PSEUDO_ZAMES_MSAT_A_BF16: - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MSB_B_F16: - case RISCV::PSEUDO_ZAMES_MSB_B_BF16: - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MSBT_B_F16: - case RISCV::PSEUDO_ZAMES_MSBT_B_BF16: - case RISCV::PSEUDO_ZAMES_MSC_C_F32: - case RISCV::PSEUDO_ZAMES_MSCT_C_F32: - case RISCV::PSEUDO_ZAMES_MZERO_T_I8: - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4: - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5: - case RISCV::PSEUDO_ZAMES_MZERO_T_F16: - case RISCV::PSEUDO_ZAMES_MZERO_T_BF16: - case RISCV::PSEUDO_ZAMES_MZERO_C_F32: - case RISCV::PSEUDO_ZAMES_MMACC_I8_F32: - case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32: - case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32: - case RISCV::PSEUDO_ZAMES_MMACC_F16_F32: - case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32: - return expandZamesMatrixPseudo(MBB, MBBI); + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32: + return expandXSAIZamesMatrixPseudo(MBB, MBBI); } return false; @@ -340,23 +340,23 @@ bool RISCVExpandPseudoPreEmitImpl::expandBOSCAMEOrderedTile( return true; } -static unsigned getZamesMcfgSelector(Register Reg) { +static unsigned getXSAIZamesMcfgSelector(Register Reg) { switch (Reg) { - case RISCV::ZAMES_TR0: + case RISCV::XSAI_ZAMES_TR0: return 0; - case RISCV::ZAMES_TR1: + case RISCV::XSAI_ZAMES_TR1: return 1; - case RISCV::ZAMES_TR2: + case RISCV::XSAI_ZAMES_TR2: return 2; - case RISCV::ZAMES_TR3: + case RISCV::XSAI_ZAMES_TR3: return 3; - case RISCV::ZAMES_ACC0: + case RISCV::XSAI_ZAMES_ACC0: return 4; - case RISCV::ZAMES_ACC1: + case RISCV::XSAI_ZAMES_ACC1: return 5; - case RISCV::ZAMES_ACC2: + case RISCV::XSAI_ZAMES_ACC2: return 6; - case RISCV::ZAMES_ACC3: + case RISCV::XSAI_ZAMES_ACC3: return 7; default: report_fatal_error( @@ -364,16 +364,17 @@ static unsigned getZamesMcfgSelector(Register Reg) { } } -static void emitZamesConfig(const RISCVInstrInfo *TII, MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, DebugLoc DL, - Register MatrixReg, Register ConfigReg) { - unsigned Selector = getZamesMcfgSelector(MatrixReg); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSETCFG)) +static void emitXSAIZamesConfig(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, DebugLoc DL, + Register MatrixReg, Register ConfigReg) { + unsigned Selector = getXSAIZamesMcfgSelector(MatrixReg); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSETCFG)) .addImm(Selector) .addReg(ConfigReg); } -bool RISCVExpandPseudoPreEmitImpl::expandZamesMatrixPseudo( +bool RISCVExpandPseudoPreEmitImpl::expandXSAIZamesMatrixPseudo( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { MachineInstr &MI = *MBBI; DebugLoc DL = MI.getDebugLoc(); @@ -382,187 +383,188 @@ bool RISCVExpandPseudoPreEmitImpl::expandZamesMatrixPseudo( switch (Opc) { default: llvm_unreachable("Unexpected ZAMES pseudo opcode"); - case RISCV::PSEUDO_ZAMES_MLA_A_I8: - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MLA_A_F16: - case RISCV::PSEUDO_ZAMES_MLA_A_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLA)) + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLA)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLB_B_I8: - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MLB_B_F16: - case RISCV::PSEUDO_ZAMES_MLB_B_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLB)) + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLB)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLBT_B_I8: - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MLBT_B_F16: - case RISCV::PSEUDO_ZAMES_MLBT_B_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLBT)) + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLBT)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLAT_A_I8: - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MLAT_A_F16: - case RISCV::PSEUDO_ZAMES_MLAT_A_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLAT)) + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLAT)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLC_C_I32: - case RISCV::PSEUDO_ZAMES_MLC_C_F32: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLC)) + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLC)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLCT_C_I32: - case RISCV::PSEUDO_ZAMES_MLCT_C_F32: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MLCT)) + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLCT)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MZERO_C_I32: - case RISCV::PSEUDO_ZAMES_MZERO_T_I8: - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4: - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5: - case RISCV::PSEUDO_ZAMES_MZERO_T_F16: - case RISCV::PSEUDO_ZAMES_MZERO_T_BF16: - case RISCV::PSEUDO_ZAMES_MZERO_C_F32: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(1).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MZERO)).add(MI.getOperand(0)); + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(1).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MZERO)) + .add(MI.getOperand(0)); break; } - case RISCV::PSEUDO_ZAMES_MMACC_I8_I32: - case RISCV::PSEUDO_ZAMES_MMACC_I8_F32: - case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32: - case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32: - case RISCV::PSEUDO_ZAMES_MMACC_F16_F32: - case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(1).getReg(), - MI.getOperand(4).getReg()); - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(2).getReg(), - MI.getOperand(5).getReg()); - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(3).getReg(), - MI.getOperand(6).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MMACC)) + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(1).getReg(), + MI.getOperand(4).getReg()); + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(2).getReg(), + MI.getOperand(5).getReg()); + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(3).getReg(), + MI.getOperand(6).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MMACC)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .add(MI.getOperand(3)); break; } - case RISCV::PSEUDO_ZAMES_MSC_C_I32: - case RISCV::PSEUDO_ZAMES_MSC_C_F32: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSC)) + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSC)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSA_A_I8: - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MSA_A_F16: - case RISCV::PSEUDO_ZAMES_MSA_A_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSA)) + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSA)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSAT_A_I8: - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4: - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5: - case RISCV::PSEUDO_ZAMES_MSAT_A_F16: - case RISCV::PSEUDO_ZAMES_MSAT_A_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSAT)) + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSAT)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSB_B_I8: - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MSB_B_F16: - case RISCV::PSEUDO_ZAMES_MSB_B_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSB)) + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSB)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSBT_B_I8: - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4: - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5: - case RISCV::PSEUDO_ZAMES_MSBT_B_F16: - case RISCV::PSEUDO_ZAMES_MSBT_B_BF16: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSBT)) + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSBT)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSCT_C_I32: - case RISCV::PSEUDO_ZAMES_MSCT_C_F32: { - emitZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), - MI.getOperand(3).getReg()); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ZAMES_MSCT)) + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSCT)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp index a1ada1957c0ff7..d7ca0d2ff408d1 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp @@ -51,8 +51,8 @@ class RISCVExpandPseudoPreRAImpl final : public RISCVExpandPseudoImplBase { bool expandLoadTLSDescAddress(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; - bool expandZamesConfigPseudo(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI) const; + bool expandXSAIZamesConfigPseudo(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; }; class RISCVExpandPseudoPreRALegacy : public MachineFunctionPass { @@ -95,85 +95,85 @@ bool RISCVExpandPseudoPreRAImpl::expandMI( return expandLoadTLSGDAddress(MBB, MBBI); case RISCV::PseudoLA_TLSDESC: return expandLoadTLSDescAddress(MBB, MBBI); - case RISCV::PSEUDO_ZAMES_MLA_A_I8_PRE: - case RISCV::PSEUDO_ZAMES_MLB_B_I8_PRE: - case RISCV::PSEUDO_ZAMES_MLBT_B_I8_PRE: - case RISCV::PSEUDO_ZAMES_MLAT_A_I8_PRE: - case RISCV::PSEUDO_ZAMES_MLC_C_I32_PRE: - case RISCV::PSEUDO_ZAMES_MLCT_C_I32_PRE: - case RISCV::PSEUDO_ZAMES_MZERO_C_I32_PRE: - case RISCV::PSEUDO_ZAMES_MMACC_I8_I32_PRE: - case RISCV::PSEUDO_ZAMES_MSC_C_I32_PRE: - case RISCV::PSEUDO_ZAMES_MSA_A_I8_PRE: - case RISCV::PSEUDO_ZAMES_MSAT_A_I8_PRE: - case RISCV::PSEUDO_ZAMES_MSB_B_I8_PRE: - case RISCV::PSEUDO_ZAMES_MSBT_B_I8_PRE: - case RISCV::PSEUDO_ZAMES_MSCT_C_I32_PRE: - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MLA_A_F16_PRE: - case RISCV::PSEUDO_ZAMES_MLA_A_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MLAT_A_F16_PRE: - case RISCV::PSEUDO_ZAMES_MLAT_A_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MLB_B_F16_PRE: - case RISCV::PSEUDO_ZAMES_MLB_B_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MLBT_B_F16_PRE: - case RISCV::PSEUDO_ZAMES_MLBT_B_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MLC_C_F32_PRE: - case RISCV::PSEUDO_ZAMES_MLCT_C_F32_PRE: - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MSA_A_F16_PRE: - case RISCV::PSEUDO_ZAMES_MSA_A_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MSAT_A_F16_PRE: - case RISCV::PSEUDO_ZAMES_MSAT_A_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MSB_B_F16_PRE: - case RISCV::PSEUDO_ZAMES_MSB_B_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MSBT_B_F16_PRE: - case RISCV::PSEUDO_ZAMES_MSBT_B_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MSC_C_F32_PRE: - case RISCV::PSEUDO_ZAMES_MSCT_C_F32_PRE: - case RISCV::PSEUDO_ZAMES_MZERO_T_I8_PRE: - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4_PRE: - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5_PRE: - case RISCV::PSEUDO_ZAMES_MZERO_T_F16_PRE: - case RISCV::PSEUDO_ZAMES_MZERO_T_BF16_PRE: - case RISCV::PSEUDO_ZAMES_MZERO_C_F32_PRE: - case RISCV::PSEUDO_ZAMES_MMACC_I8_F32_PRE: - case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE: - case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE: - case RISCV::PSEUDO_ZAMES_MMACC_F16_F32_PRE: - case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32_PRE: - return expandZamesConfigPseudo(MBB, MBBI); + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE: + return expandXSAIZamesConfigPseudo(MBB, MBBI); } return false; } -static constexpr int64_t ZamesCfgI8 = 0x02; -static constexpr int64_t ZamesCfgI32 = 0x04; -static constexpr int64_t ZamesCfgF8E5 = 0x07; -static constexpr int64_t ZamesCfgF8E4 = 0x08; -static constexpr int64_t ZamesCfgF16 = 0x09; -static constexpr int64_t ZamesCfgBF16 = 0x0a; -static constexpr int64_t ZamesCfgF32 = 0x0c; - -static Register createZamesConfigPayload(const RISCVInstrInfo *TII, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - DebugLoc DL, int64_t Payload) { +static constexpr int64_t XSAIZamesCfgI8 = 0x02; +static constexpr int64_t XSAIZamesCfgI32 = 0x04; +static constexpr int64_t XSAIZamesCfgF8E5 = 0x07; +static constexpr int64_t XSAIZamesCfgF8E4 = 0x08; +static constexpr int64_t XSAIZamesCfgF16 = 0x09; +static constexpr int64_t XSAIZamesCfgBF16 = 0x0a; +static constexpr int64_t XSAIZamesCfgF32 = 0x0c; + +static Register createXSAIZamesConfigPayload(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + DebugLoc DL, int64_t Payload) { MachineFunction *MF = MBB.getParent(); Register ConfigReg = MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); @@ -186,13 +186,13 @@ static Register createZamesConfigPayload(const RISCVInstrInfo *TII, // Expand a single-config _PRE pseudo: materialize the payload GPR, rebuild // the second-level pseudo with all original operands plus the payload // register, and carry the memory operands over. -static void expandZamesPreRASingleConfig(const RISCVInstrInfo *TII, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineInstr &MI, unsigned NewOpc, - int64_t Payload) { +static void expandXSAIZamesPreRASingleConfig(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineInstr &MI, unsigned NewOpc, + int64_t Payload) { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, MI.getDebugLoc(), Payload); + createXSAIZamesConfigPayload(TII, MBB, MBBI, MI.getDebugLoc(), Payload); auto MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc)); for (const MachineOperand &MO : MI.operands()) MIB.add(MO); @@ -204,16 +204,16 @@ static void expandZamesPreRASingleConfig(const RISCVInstrInfo *TII, // payload GPRs and rebuild the second-level pseudo. The B and A matrices // always share the element type in every supported combination, so one // payload register feeds both config operands. -static void expandZamesPreRAMMAcc(const RISCVInstrInfo *TII, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineInstr &MI, unsigned NewOpc, - int64_t AccPayload, int64_t MatPayload) { +static void expandXSAIZamesPreRAMMAcc(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineInstr &MI, unsigned NewOpc, + int64_t AccPayload, int64_t MatPayload) { DebugLoc DL = MI.getDebugLoc(); Register AccConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, AccPayload); + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, AccPayload); Register MatConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, MatPayload); + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, MatPayload); BuildMI(MBB, MBBI, DL, TII->get(NewOpc)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) @@ -224,7 +224,7 @@ static void expandZamesPreRAMMAcc(const RISCVInstrInfo *TII, .addReg(MatConfigReg); } -bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( +bool RISCVExpandPseudoPreRAImpl::expandXSAIZamesConfigPseudo( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { MachineInstr &MI = *MBBI; DebugLoc DL = MI.getDebugLoc(); @@ -233,10 +233,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( switch (Opc) { default: llvm_unreachable("Unexpected ZAMES pre-RA pseudo opcode"); - case RISCV::PSEUDO_ZAMES_MLA_A_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLA_A_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -244,10 +244,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLB_B_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLB_B_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -255,10 +255,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLBT_B_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLBT_B_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -266,10 +266,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLAT_A_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLAT_A_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -277,10 +277,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLC_C_I32_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLC_C_I32)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -288,10 +288,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLCT_C_I32_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MLCT_C_I32)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -299,20 +299,20 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MZERO_C_I32_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MZERO_C_I32)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32)) .add(MI.getOperand(0)) .addReg(ConfigReg); break; } - case RISCV::PSEUDO_ZAMES_MMACC_I8_I32_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE: { Register AccConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); Register MatConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MMACC_I8_I32)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -322,10 +322,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .addReg(MatConfigReg); break; } - case RISCV::PSEUDO_ZAMES_MSC_C_I32_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSC_C_I32)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -333,10 +333,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSA_A_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSA_A_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -344,10 +344,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSAT_A_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSAT_A_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -355,10 +355,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSB_B_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSB_B_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -366,10 +366,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSBT_B_I8_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI8); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSBT_B_I8)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -377,10 +377,10 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MSCT_C_I32_PRE: { + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE: { Register ConfigReg = - createZamesConfigPayload(TII, MBB, MBBI, DL, ZamesCfgI32); - BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_ZAMES_MSCT_C_I32)) + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32)) .add(MI.getOperand(0)) .add(MI.getOperand(1)) .add(MI.getOperand(2)) @@ -388,196 +388,240 @@ bool RISCVExpandPseudoPreRAImpl::expandZamesConfigPseudo( .cloneMemRefs(MI); break; } - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E4_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLA_A_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MLA_A_FP8E5_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLA_A_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MLA_A_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLA_A_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MLA_A_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLA_A_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLAT_A_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLAT_A_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MLAT_A_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLAT_A_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MLAT_A_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLAT_A_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E4_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLB_B_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MLB_B_FP8E5_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLB_B_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MLB_B_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLB_B_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MLB_B_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLB_B_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLBT_B_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MLBT_B_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MLBT_B_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLBT_B_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MLBT_B_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLBT_B_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MLC_C_F32_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLC_C_F32, ZamesCfgF32); - break; - case RISCV::PSEUDO_ZAMES_MLCT_C_F32_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MLCT_C_F32, ZamesCfgF32); - break; - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E4_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSA_A_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MSA_A_FP8E5_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSA_A_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MSA_A_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSA_A_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MSA_A_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSA_A_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSAT_A_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSAT_A_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MSAT_A_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSAT_A_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MSAT_A_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSAT_A_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E4_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSB_B_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MSB_B_FP8E5_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSB_B_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MSB_B_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSB_B_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MSB_B_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSB_B_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSBT_B_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MSBT_B_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MSBT_B_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSBT_B_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MSBT_B_BF16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSBT_B_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MSC_C_F32_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSC_C_F32, ZamesCfgF32); - break; - case RISCV::PSEUDO_ZAMES_MSCT_C_F32_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MSCT_C_F32, ZamesCfgF32); - break; - case RISCV::PSEUDO_ZAMES_MZERO_T_I8_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MZERO_T_I8, ZamesCfgI8); - break; - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MZERO_T_FP8E4, ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MZERO_T_FP8E5, ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MZERO_T_F16_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MZERO_T_F16, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MZERO_T_BF16_PRE: - expandZamesPreRASingleConfig( - TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MZERO_T_BF16, ZamesCfgBF16); - break; - case RISCV::PSEUDO_ZAMES_MZERO_C_F32_PRE: - expandZamesPreRASingleConfig(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MZERO_C_F32, ZamesCfgF32); - break; - case RISCV::PSEUDO_ZAMES_MMACC_I8_F32_PRE: - expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MMACC_I8_F32, - ZamesCfgF32, ZamesCfgI8); - break; - case RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE: - expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MMACC_FP8E4_F32, ZamesCfgF32, - ZamesCfgF8E4); - break; - case RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE: - expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MMACC_FP8E5_F32, ZamesCfgF32, - ZamesCfgF8E5); - break; - case RISCV::PSEUDO_ZAMES_MMACC_F16_F32_PRE: - expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, RISCV::PSEUDO_ZAMES_MMACC_F16_F32, - ZamesCfgF32, ZamesCfgF16); - break; - case RISCV::PSEUDO_ZAMES_MMACC_BF16_F32_PRE: - expandZamesPreRAMMAcc(TII, MBB, MBBI, MI, - RISCV::PSEUDO_ZAMES_MMACC_BF16_F32, ZamesCfgF32, - ZamesCfgBF16); + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8, + XSAIZamesCfgI8); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32, + XSAIZamesCfgF32, XSAIZamesCfgI8); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32, + XSAIZamesCfgF32, XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32, + XSAIZamesCfgF32, XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32, + XSAIZamesCfgF32, XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32, + XSAIZamesCfgF32, XSAIZamesCfgBF16); break; } diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h index 0c34439597f6a4..d13d9d7b501693 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -47,8 +47,8 @@ class RISCVDAGToDAGISel : public SelectionDAGISel { if (!II) continue; StringRef Name = Intrinsic::getBaseName(II->getIntrinsicID()); - if (Name.starts_with("llvm.riscv.zames.") || - Name.starts_with("llvm.riscv.zmasync.")) { + if (Name.starts_with("llvm.riscv.xsai.zames.") || + Name.starts_with("llvm.riscv.xsai.zmasync.")) { F.getContext().diagnose(DiagnosticInfoUnsupported( F, "ZAMES/ZMASYNC code generation requires RV64", I.getDebugLoc())); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index db8def4774dd8a..231a163385a850 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -197,16 +197,24 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, } if (Subtarget.hasZames()) { - addRegisterClass(MVT::riscv_ame_tile_i8, &RISCV::ZamesTileRegRegClass); - addRegisterClass(MVT::riscv_ame_tile_i32, &RISCV::ZamesAccRegRegClass); - addRegisterClass(MVT::riscv_ame_tile_fp8e4, &RISCV::ZamesTileRegRegClass); - addRegisterClass(MVT::riscv_ame_tile_fp8e5, &RISCV::ZamesTileRegRegClass); - addRegisterClass(MVT::riscv_ame_tile_f16, &RISCV::ZamesTileRegRegClass); - addRegisterClass(MVT::riscv_ame_tile_bf16, &RISCV::ZamesTileRegRegClass); - addRegisterClass(MVT::riscv_ame_tile_f32, &RISCV::ZamesAccRegRegClass); + addRegisterClass(MVT::riscv_xsai_ame_tile_i8, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_ame_tile_i32, + &RISCV::XSAI_ZAMES_AccRegRegClass); + addRegisterClass(MVT::riscv_xsai_ame_tile_fp8e4, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_ame_tile_fp8e5, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_ame_tile_f16, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_ame_tile_bf16, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_ame_tile_f32, + &RISCV::XSAI_ZAMES_AccRegRegClass); } if (Subtarget.hasZmasync()) - addRegisterClass(MVT::riscv_ame_sync, &RISCV::SyncRRegClass); + addRegisterClass(MVT::riscv_xsai_ame_sync, + &RISCV::XSAI_ZMASYNC_SyncRRegClass); static const MVT::SimpleValueType BoolVecVTs[] = { MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1, @@ -2288,23 +2296,26 @@ void RISCVTargetLowering::getTgtMemIntrinsic( Infos.push_back(Info); }; - auto GetZamesMemoryVT = [&](bool IsStore) { + auto GetXSAIZamesMemoryVT = [&](bool IsStore) { Type *TileTy = IsStore ? I.getArgOperand(0)->getType() : I.getType(); auto *TargetTy = dyn_cast(TileTy); if (!TargetTy) return MVT(MVT::Other); return StringSwitch(TargetTy->getName()) - .Cases({"riscv.ame.tile.int8", "riscv.ame.tile.fp8e4", - "riscv.ame.tile.fp8e5"}, + .Cases({"riscv.xsai.ame.tile.int8", "riscv.xsai.ame.tile.fp8e4", + "riscv.xsai.ame.tile.fp8e5"}, MVT::i8) - .Cases({"riscv.ame.tile.fp16", "riscv.ame.tile.bf16"}, MVT::i16) - .Cases({"riscv.ame.tile.int32", "riscv.ame.tile.fp32"}, MVT::i32) + .Cases({"riscv.xsai.ame.tile.fp16", "riscv.xsai.ame.tile.bf16"}, + MVT::i16) + .Cases({"riscv.xsai.ame.tile.int32", "riscv.xsai.ame.tile.fp32"}, + MVT::i32) .Default(MVT::Other); }; - auto SetZamesMemoryInfo = [&](unsigned PtrOp, bool IsStore, bool UsePtrVal) { + auto SetXSAIZamesMemoryInfo = [&](unsigned PtrOp, bool IsStore, + bool UsePtrVal) { Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN; - Info.memVT = GetZamesMemoryVT(IsStore); + Info.memVT = GetXSAIZamesMemoryVT(IsStore); if (Info.memVT == MVT::Other) return; @@ -2651,41 +2662,41 @@ void RISCVTargetLowering::getTgtMemIntrinsic( Info.flags |= MachineMemOperand::MOStore; Infos.push_back(Info); return; - case Intrinsic::riscv_zames_mla: - case Intrinsic::riscv_zames_mlat: - case Intrinsic::riscv_zames_mlb: - case Intrinsic::riscv_zames_mlbt: - SetZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, - /*UsePtrVal*/ false); + case Intrinsic::riscv_xsai_zames_mla: + case Intrinsic::riscv_xsai_zames_mlat: + case Intrinsic::riscv_xsai_zames_mlb: + case Intrinsic::riscv_xsai_zames_mlbt: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ false); return; - case Intrinsic::riscv_zames_mlc: - case Intrinsic::riscv_zames_mlct: - SetZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, - /*UsePtrVal*/ false); + case Intrinsic::riscv_xsai_zames_mlc: + case Intrinsic::riscv_xsai_zames_mlct: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ false); return; - case Intrinsic::riscv_zames_msa: - case Intrinsic::riscv_zames_msat: - case Intrinsic::riscv_zames_msb: - case Intrinsic::riscv_zames_msbt: - SetZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, - /*UsePtrVal*/ false); + case Intrinsic::riscv_xsai_zames_msa: + case Intrinsic::riscv_xsai_zames_msat: + case Intrinsic::riscv_xsai_zames_msb: + case Intrinsic::riscv_xsai_zames_msbt: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ false); return; - case Intrinsic::riscv_zames_msc: - case Intrinsic::riscv_zames_msct: - SetZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, - /*UsePtrVal*/ false); + case Intrinsic::riscv_xsai_zames_msc: + case Intrinsic::riscv_xsai_zames_msct: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ false); return; - case Intrinsic::riscv_zames_mla_whole: - case Intrinsic::riscv_zames_mlb_whole: - case Intrinsic::riscv_zames_mlc_whole: - SetZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, - /*UsePtrVal*/ true); + case Intrinsic::riscv_xsai_zames_mla_whole: + case Intrinsic::riscv_xsai_zames_mlb_whole: + case Intrinsic::riscv_xsai_zames_mlc_whole: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ true); return; - case Intrinsic::riscv_zames_msa_whole: - case Intrinsic::riscv_zames_msb_whole: - case Intrinsic::riscv_zames_msc_whole: - SetZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, - /*UsePtrVal*/ true); + case Intrinsic::riscv_xsai_zames_msa_whole: + case Intrinsic::riscv_xsai_zames_msb_whole: + case Intrinsic::riscv_xsai_zames_msc_whole: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ true); return; } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index d641d2138fdfc0..663c0162872ba0 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -2529,8 +2529,8 @@ include "RISCVInstrInfoSFB.td" include "RISCVInstrInfoY.td" // ZAMES/ZMASYNC matrix extensions -include "RISCVInstrInfoZames.td" -include "RISCVInstrInfoZamesPatterns.td" +include "RISCVInstrInfoXSAIZames.td" +include "RISCVInstrInfoXSAIZamesPatterns.td" //===----------------------------------------------------------------------===// // Vendor extensions diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZames.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td similarity index 58% rename from llvm/lib/Target/RISCV/RISCVInstrInfoZames.td rename to llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td index 32f44ae3ce4228..f738bfef46e550 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZames.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td @@ -1,4 +1,4 @@ -//===-- RISCVInstrInfoZames.td - ZAMES/ZMASYNC instructions -*- tablegen -*-===// +//===-- RISCVInstrInfoXSAIZames.td - ZAMES/ZMASYNC instructions -*- tablegen -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,23 +11,23 @@ // //===----------------------------------------------------------------------===// -def zames_uimm10 : RISCVUImmLeafOp<10>; +def xsai_zames_uimm10 : RISCVUImmLeafOp<10>; // mcfg0..mcfg7 select the hardware matrix configuration register configured by // msetcfg/mgetcfg. The value is encoded as a 3-bit immediate. -def MCFGSelAsmOperand : AsmOperandClass { - let Name = "MCFGSel"; +def XSAIZamesMCFGSelAsmOperand : AsmOperandClass { + let Name = "XSAIZamesMCFGSel"; let RenderMethod = "addImmOperands"; - let ParserMethod = "parseMCFGSel"; + let ParserMethod = "parseXSAIZamesMCFGSel"; } -def mcfgsel : RISCVUImmLeafOp<3> { - let ParserMatchClass = MCFGSelAsmOperand; - let PrintMethod = "printMCFGSel"; +def xsai_zames_mcfgsel : RISCVUImmLeafOp<3> { + let ParserMatchClass = XSAIZamesMCFGSelAsmOperand; + let PrintMethod = "printXSAIZamesMCFGSel"; } -class ZAMESInstConfigImm func4, string opcodestr> - : RVInst<(outs), (ins zames_uimm10:$imm), opcodestr, "$imm", [], +class XSAI_ZAMES_InstConfigImm func4, string opcodestr> + : RVInst<(outs), (ins xsai_zames_uimm10:$imm), opcodestr, "$imm", [], InstFormatI> { bits<10> imm; @@ -40,7 +40,7 @@ class ZAMESInstConfigImm func4, string opcodestr> let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstConfigReg func4, string opcodestr> +class XSAI_ZAMES_InstConfigReg func4, string opcodestr> : RVInst<(outs), (ins GPR:$rs1), opcodestr, "$rs1", [], InstFormatI> { bits<5> rs1; @@ -55,8 +55,8 @@ class ZAMESInstConfigReg func4, string opcodestr> let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstSetCfg - : RVInst<(outs), (ins mcfgsel:$cfg, GPR:$rs1), opcodestr, "$cfg, $rs1", +class XSAI_ZAMES_InstSetCfg + : RVInst<(outs), (ins xsai_zames_mcfgsel:$cfg, GPR:$rs1), opcodestr, "$cfg, $rs1", [], InstFormatI> { bits<3> cfg; bits<5> rs1; @@ -72,8 +72,8 @@ class ZAMESInstSetCfg let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstGetCfg - : RVInst<(outs GPR:$rd), (ins mcfgsel:$cfg), opcodestr, "$rd, $cfg", +class XSAI_ZAMES_InstGetCfg + : RVInst<(outs GPR:$rd), (ins xsai_zames_mcfgsel:$cfg), opcodestr, "$rd, $cfg", [], InstFormatI> { bits<5> rd; bits<3> cfg; @@ -89,13 +89,13 @@ class ZAMESInstGetCfg let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstInit +class XSAI_ZAMES_InstInit : RVInst<(outs), (ins), opcodestr, "", [], InstFormatI> { let Inst{31-7} = 0; let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstLoadStore func4, bit ls, dag outs, dag ins, +class XSAI_ZAMES_InstLoadStore func4, bit ls, dag outs, dag ins, string opcodestr> : RVInst { @@ -113,7 +113,7 @@ class ZAMESInstLoadStore func4, bit ls, dag outs, dag ins, let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstWholeLoadStore func4, bit ls, dag outs, dag ins, +class XSAI_ZAMES_InstWholeLoadStore func4, bit ls, dag outs, dag ins, string opcodestr> : RVInst { bits<5> md; @@ -129,7 +129,7 @@ class ZAMESInstWholeLoadStore func4, bit ls, dag outs, dag ins, let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstMMAcc func4, dag outs, dag ins, +class XSAI_ZAMES_InstMMAcc func4, dag outs, dag ins, string opcodestr> : RVInst { @@ -146,7 +146,7 @@ class ZAMESInstMMAcc func4, dag outs, dag ins, let Inst{6-0} = OPC_CUSTOM_1.Value; } -class ZAMESInstMisc func4, dag outs, dag ins, +class XSAI_ZAMES_InstMisc func4, dag outs, dag ins, string opcodestr> : RVInst { bits<5> md; @@ -160,101 +160,101 @@ class ZAMESInstMisc func4, dag outs, dag ins, let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, mayStore = 0 in { - def ZAMES_MSETTILEMI : ZAMESInstConfigImm<0b0010, "msettilemi">; - def ZAMES_MSETTILENI : ZAMESInstConfigImm<0b0011, "msettileni">; - def ZAMES_MSETTILEKI : ZAMESInstConfigImm<0b0001, "msettileki">; + def XSAI_ZAMES_MSETTILEMI : XSAI_ZAMES_InstConfigImm<0b0010, "msettilemi">; + def XSAI_ZAMES_MSETTILENI : XSAI_ZAMES_InstConfigImm<0b0011, "msettileni">; + def XSAI_ZAMES_MSETTILEKI : XSAI_ZAMES_InstConfigImm<0b0001, "msettileki">; - def ZAMES_MSETTILEM : ZAMESInstConfigReg<0b0010, "msettilem">; - def ZAMES_MSETTILEN : ZAMESInstConfigReg<0b0011, "msettilen">; - def ZAMES_MSETTILEK : ZAMESInstConfigReg<0b0001, "msettilek">; + def XSAI_ZAMES_MSETTILEM : XSAI_ZAMES_InstConfigReg<0b0010, "msettilem">; + def XSAI_ZAMES_MSETTILEN : XSAI_ZAMES_InstConfigReg<0b0011, "msettilen">; + def XSAI_ZAMES_MSETTILEK : XSAI_ZAMES_InstConfigReg<0b0001, "msettilek">; - def ZAMES_MSETCFG : ZAMESInstSetCfg<"msetcfg">; - def ZAMES_MGETCFG : ZAMESInstGetCfg<"mgetcfg">; + def XSAI_ZAMES_MSETCFG : XSAI_ZAMES_InstSetCfg<"msetcfg">; + def XSAI_ZAMES_MGETCFG : XSAI_ZAMES_InstGetCfg<"mgetcfg">; - def ZAMES_MINIT : ZAMESInstInit<"minit">; + def XSAI_ZAMES_MINIT : XSAI_ZAMES_InstInit<"minit">; } let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 1, mayStore = 0 in { - def ZAMES_MLA : ZAMESInstLoadStore<0b0000, 0b0, (outs ZamesTileReg:$md), + def XSAI_ZAMES_MLA : XSAI_ZAMES_InstLoadStore<0b0000, 0b0, (outs XSAI_ZAMES_TileReg:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), "mla">; - def ZAMES_MLB : ZAMESInstLoadStore<0b0001, 0b0, (outs ZamesTileReg:$md), + def XSAI_ZAMES_MLB : XSAI_ZAMES_InstLoadStore<0b0001, 0b0, (outs XSAI_ZAMES_TileReg:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), "mlb">; - def ZAMES_MLC : ZAMESInstLoadStore<0b0010, 0b0, (outs ZamesAccReg:$md), + def XSAI_ZAMES_MLC : XSAI_ZAMES_InstLoadStore<0b0010, 0b0, (outs XSAI_ZAMES_AccReg:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), "mlc">; - def ZAMES_MLBT : ZAMESInstLoadStore<0b0101, 0b0, (outs ZamesTileReg:$md), + def XSAI_ZAMES_MLBT : XSAI_ZAMES_InstLoadStore<0b0101, 0b0, (outs XSAI_ZAMES_TileReg:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), "mlbt">; - def ZAMES_MLAT : ZAMESInstLoadStore<0b0100, 0b0, (outs ZamesTileReg:$md), + def XSAI_ZAMES_MLAT : XSAI_ZAMES_InstLoadStore<0b0100, 0b0, (outs XSAI_ZAMES_TileReg:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), "mlat">; - def ZAMES_MLCT : ZAMESInstLoadStore<0b0110, 0b0, (outs ZamesAccReg:$md), + def XSAI_ZAMES_MLCT : XSAI_ZAMES_InstLoadStore<0b0110, 0b0, (outs XSAI_ZAMES_AccReg:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), "mlct">; - def ZAMES_MLA_WHOLE - : ZAMESInstWholeLoadStore<0b1000, 0b0, (outs ZamesTileReg:$md), + def XSAI_ZAMES_MLA_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1000, 0b0, (outs XSAI_ZAMES_TileReg:$md), (ins GPRMemZeroOffset:$rs1), "mla.whole">; - def ZAMES_MLB_WHOLE - : ZAMESInstWholeLoadStore<0b1001, 0b0, (outs ZamesTileReg:$md), + def XSAI_ZAMES_MLB_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1001, 0b0, (outs XSAI_ZAMES_TileReg:$md), (ins GPRMemZeroOffset:$rs1), "mlb.whole">; - def ZAMES_MLC_WHOLE - : ZAMESInstWholeLoadStore<0b1010, 0b0, (outs ZamesAccReg:$md), + def XSAI_ZAMES_MLC_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1010, 0b0, (outs XSAI_ZAMES_AccReg:$md), (ins GPRMemZeroOffset:$rs1), "mlc.whole">; } let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, mayStore = 1 in { - def ZAMES_MSC : ZAMESInstLoadStore<0b0010, 0b1, (outs), - (ins ZamesAccReg:$md, + def XSAI_ZAMES_MSC : XSAI_ZAMES_InstLoadStore<0b0010, 0b1, (outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), "msc">; - def ZAMES_MSA : ZAMESInstLoadStore<0b0000, 0b1, (outs), - (ins ZamesTileReg:$md, + def XSAI_ZAMES_MSA : XSAI_ZAMES_InstLoadStore<0b0000, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), "msa">; - def ZAMES_MSB : ZAMESInstLoadStore<0b0001, 0b1, (outs), - (ins ZamesTileReg:$md, + def XSAI_ZAMES_MSB : XSAI_ZAMES_InstLoadStore<0b0001, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), "msb">; - def ZAMES_MSAT : ZAMESInstLoadStore<0b0100, 0b1, (outs), - (ins ZamesTileReg:$md, + def XSAI_ZAMES_MSAT : XSAI_ZAMES_InstLoadStore<0b0100, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), "msat">; - def ZAMES_MSBT : ZAMESInstLoadStore<0b0101, 0b1, (outs), - (ins ZamesTileReg:$md, + def XSAI_ZAMES_MSBT : XSAI_ZAMES_InstLoadStore<0b0101, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), "msbt">; - def ZAMES_MSCT : ZAMESInstLoadStore<0b0110, 0b1, (outs), - (ins ZamesAccReg:$md, + def XSAI_ZAMES_MSCT : XSAI_ZAMES_InstLoadStore<0b0110, 0b1, (outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), "msct">; - def ZAMES_MSA_WHOLE - : ZAMESInstWholeLoadStore<0b1000, 0b1, (outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1), + def XSAI_ZAMES_MSA_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1000, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1), "msa.whole">; - def ZAMES_MSB_WHOLE - : ZAMESInstWholeLoadStore<0b1001, 0b1, (outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1), + def XSAI_ZAMES_MSB_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1001, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1), "msb.whole">; - def ZAMES_MSC_WHOLE - : ZAMESInstWholeLoadStore<0b1010, 0b1, (outs), - (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1), + def XSAI_ZAMES_MSC_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1010, 0b1, (outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1), "msc.whole">; } let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, mayStore = 0 in { - def ZAMES_MZERO : ZAMESInstMisc<0b0000, (outs ZamesTileOrAccReg:$md), (ins), + def XSAI_ZAMES_MZERO : XSAI_ZAMES_InstMisc<0b0000, (outs XSAI_ZAMES_TileOrAccReg:$md), (ins), "mzero">; - def ZAMES_MMACC : ZAMESInstMMAcc<0b0000, (outs ZamesAccReg:$md_wb), - (ins ZamesAccReg:$md, ZamesTileReg:$ms2, - ZamesTileReg:$ms1), + def XSAI_ZAMES_MMACC : XSAI_ZAMES_InstMMAcc<0b0000, (outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, + XSAI_ZAMES_TileReg:$ms1), "mmacc"> { let Constraints = "$md_wb = $md"; } @@ -264,7 +264,7 @@ let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, // ZMASYNC Matrix Async Synchronization Instructions //===----------------------------------------------------------------------===// -class ZMASYNCInst func4, dag outs, dag ins, +class XSAI_ZMASYNC_Inst func4, dag outs, dag ins, string opcodestr, string argstr> : RVInst { bits<5> rs1; @@ -282,26 +282,26 @@ class ZMASYNCInst func4, dag outs, dag ins, let Predicates = [HasStdExtZmasync], hasSideEffects = 1, mayLoad = 0, mayStore = 0 in { - def ZMASYNC_MSYNCREGRESET - : ZMASYNCInst<0b1000, (outs SyncR:$sync), (ins), + def XSAI_ZMASYNC_MSYNCREGRESET + : XSAI_ZMASYNC_Inst<0b1000, (outs XSAI_ZMASYNC_SyncR:$sync), (ins), "msyncregreset", "$sync"> { let rs1 = 0b00000; } let Constraints = "$sync = $src" in { - def ZMASYNC_MRELEASE - : ZMASYNCInst<0b1001, (outs SyncR:$sync), (ins SyncR:$src), + def XSAI_ZMASYNC_MRELEASE + : XSAI_ZMASYNC_Inst<0b1001, (outs XSAI_ZMASYNC_SyncR:$sync), (ins XSAI_ZMASYNC_SyncR:$src), "mrelease", "$sync"> { let rs1 = 0b00000; } } - def ZMASYNC_MACQUIRE - : ZMASYNCInst<0b1010, (outs), (ins SyncR:$sync, GPR:$rs1), + def XSAI_ZMASYNC_MACQUIRE + : XSAI_ZMASYNC_Inst<0b1010, (outs), (ins XSAI_ZMASYNC_SyncR:$sync, GPR:$rs1), "macquire", "$sync, $rs1">; - def ZMASYNC_MFENCE - : ZMASYNCInst<0b1011, (outs), (ins), "mfence", ""> { + def XSAI_ZMASYNC_MFENCE + : XSAI_ZMASYNC_Inst<0b1011, (outs), (ins), "mfence", ""> { let rs1 = 0b00000; let sync = 0b00000; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td new file mode 100644 index 00000000000000..3c2bf200497ea8 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td @@ -0,0 +1,594 @@ +//===-- RISCVInstrInfoXSAIZamesPatterns.td - ZAMES/ZMASYNC pseudos -*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC codegen-only pseudos and SelectionDAG patterns. +// +// Ordinary load/store, mzero and mmacc go through a two-level pseudo: +// intrinsic -> PSEUDO_XSAI_ZAMES_*_PRE -> (pre-RA: materialize cfg payload GPR) +// -> PSEUDO_XSAI_ZAMES_* with cfg operands -> RA -> (post-RA: emit msetcfg by +// final TR/ACC encoding) -> real ZAMES instruction. +// +// msettile*, msetcfg/mgetcfg, minit, *.whole and the four ZMASYNC +// instructions are selected directly to real instructions. +// +// mmacc operand reorder: LLVM IR operands are A, B, C (the intrinsic +// overload suffix lists the C, A, B *type* sequence, not the operand order); +// the typed pseudo and real instruction use the machine order C, B, A. +// +//===----------------------------------------------------------------------===// + +// Defensive second-layer immediate constraints. The ImmArg/Range intrinsic +// attributes reject out-of-range constants in the IR verifier; these leaves +// additionally prevent ISel from silently truncating an out-of-range target +// constant into the 10-bit tile field or the 3-bit config selector when the +// verifier is disabled or a transform built an illegal DAG. +def xsai_zames_tuimm10 : TImmLeaf(Imm); }]>; +def xsai_zames_tuimm3 : TImmLeaf(Imm); }]>; + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 1, + mayStore = 0, isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + + def PSEUDO_XSAI_ZAMES_MLA_A_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLB_B_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLBT_B_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLAT_A_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLC_C_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLCT_C_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, + mayStore = 0, isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE : Pseudo<(outs XSAI_ZAMES_AccReg:$md), (ins), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1), []> { + let Constraints = "$md_wb = $md"; + let Size = 28; + } + + def PSEUDO_XSAI_ZAMES_MZERO_C_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), (ins GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MMACC_I8_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1, GPR:$cfg_c, + GPR:$cfg_b, GPR:$cfg_a), []> { + let Constraints = "$md_wb = $md"; + let Size = 16; + } +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, + mayStore = 1, isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + + def PSEUDO_XSAI_ZAMES_MSC_C_I32 + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSA_A_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSAT_A_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSB_B_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSBT_B_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSCT_C_I32 + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } +} + +class PatXSAI_ZAMES_SetTileReg + : Pat<(OpNode (XLenVT GPR:$rs1)), (Inst GPR:$rs1)>; + +// Shared shells for the typed ZAMES pseudo instructions. Concrete defs carry +// the PSEUDO_XSAI_ZAMES___ names so every instance stays +// reachable from the pre-RA and late C++ expansion switches. +class XSAI_ZAMES_LdPseudoPRE + : Pseudo<(outs RC:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; +} +class XSAI_ZAMES_LdPseudoCFG + : Pseudo<(outs RC:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; +} +class XSAI_ZAMES_StPseudoPRE + : Pseudo<(outs), (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; +} +class XSAI_ZAMES_StPseudoCFG + : Pseudo<(outs), + (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; +} +class XSAI_ZAMES_MZeroPseudoPRE + : Pseudo<(outs RC:$md), (ins), []> { + let Size = 12; +} +class XSAI_ZAMES_MZeroPseudoCFG + : Pseudo<(outs RC:$md), (ins GPR:$cfg), []> { + let Size = 8; +} +class XSAI_ZAMES_MMAccPseudoPRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1), []> { + let Constraints = "$md_wb = $md"; + let Size = 28; +} +class XSAI_ZAMES_MMAccPseudoCFG + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1, GPR:$cfg_c, + GPR:$cfg_b, GPR:$cfg_a), []> { + let Constraints = "$md_wb = $md"; + let Size = 16; +} + +// Typed load pseudo instances: A/B tiles use every supported input element; +// C accumulators use fp32. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 1, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLC_C_F32 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLCT_C_F32 : XSAI_ZAMES_LdPseudoCFG; +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 1, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSC_C_F32 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSCT_C_F32 : XSAI_ZAMES_StPseudoCFG; +} + +// Typed mzero pseudo instances preserve the input/output register partition. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_I8 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_F16 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_BF16 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_C_F32 : XSAI_ZAMES_MZeroPseudoCFG; +} + +// Typed mmacc pseudo instances for the six supported combinations. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_I8_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_F16_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_BF16_F32 : XSAI_ZAMES_MMAccPseudoCFG; +} + +class PatXSAI_ZAMES_Load + : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1, GPR:$rs2)), + (Inst GPRMemZeroOffset:$rs1, GPR:$rs2)>; + +class PatXSAI_ZAMES_Store + : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2), + (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2)>; + +class PatXSAI_ZAMES_WholeLoad + : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1)), + (Inst GPRMemZeroOffset:$rs1)>; + +class PatXSAI_ZAMES_WholeStore + : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1), + (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1)>; + +let Predicates = [HasStdExtZames, IsRV64] in { + def : Pat<(int_riscv_xsai_zames_msettilemi xsai_zames_tuimm10:$imm), + (XSAI_ZAMES_MSETTILEMI timm:$imm)>; + def : Pat<(int_riscv_xsai_zames_msettileni xsai_zames_tuimm10:$imm), + (XSAI_ZAMES_MSETTILENI timm:$imm)>; + def : Pat<(int_riscv_xsai_zames_msettileki xsai_zames_tuimm10:$imm), + (XSAI_ZAMES_MSETTILEKI timm:$imm)>; + + def : PatXSAI_ZAMES_SetTileReg; + def : PatXSAI_ZAMES_SetTileReg; + def : PatXSAI_ZAMES_SetTileReg; + + def : Pat<(int_riscv_xsai_zames_msetcfg xsai_zames_tuimm3:$cfg, (XLenVT GPR:$rs1)), + (XSAI_ZAMES_MSETCFG timm:$cfg, GPR:$rs1)>; + + def : Pat<(int_riscv_xsai_zames_minit), (XSAI_ZAMES_MINIT)>; + def : Pat<(XLenVT (int_riscv_xsai_zames_mgetcfg xsai_zames_tuimm3:$cfg)), + (XSAI_ZAMES_MGETCFG timm:$cfg)>; + + def : Pat<(xsai_mi32_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE)>; + + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + + def : Pat<(xsai_mi32_t (int_riscv_xsai_zames_mmacc (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mi32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE (xsai_mi32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1))>; + + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + + def : Pat<(xsai_mi8_t (int_riscv_xsai_zames_mzero)), (PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE)>; + def : Pat<(xsai_mf8e4_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE)>; + def : Pat<(xsai_mf8e5_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE)>; + def : Pat<(xsai_mf16_t (int_riscv_xsai_zames_mzero)), (PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE)>; + def : Pat<(xsai_mbf16_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE)>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mzero)), (PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE)>; + + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mf16_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf16_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs1))>; + + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; +} + +let Predicates = [HasStdExtZmasync, IsRV64] in { + def : Pat<(xsai_msync_t (int_riscv_xsai_zmasync_msyncregreset)), + (XSAI_ZMASYNC_MSYNCREGRESET)>; + def : Pat<(xsai_msync_t (int_riscv_xsai_zmasync_mrelease (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync))), + (XSAI_ZMASYNC_MRELEASE (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync))>; + def : Pat<(int_riscv_xsai_zmasync_macquire (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync), + (XLenVT GPR:$rs1)), + (XSAI_ZMASYNC_MACQUIRE (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync), (XLenVT GPR:$rs1))>; + def : Pat<(int_riscv_xsai_zmasync_mfence), (XSAI_ZMASYNC_MFENCE)>; +} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td deleted file mode 100644 index 26a10558dc5d18..00000000000000 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZamesPatterns.td +++ /dev/null @@ -1,595 +0,0 @@ -//===-- RISCVInstrInfoZamesPatterns.td - ZAMES/ZMASYNC pseudos -*- tablegen -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// ZAMES/ZMASYNC codegen-only pseudos and SelectionDAG patterns. -// -// Ordinary load/store, mzero and mmacc go through a two-level pseudo: -// intrinsic -> PSEUDO_ZAMES_*_PRE -> (pre-RA: materialize cfg payload GPR) -// -> PSEUDO_ZAMES_* with cfg operands -> RA -> (post-RA: emit msetcfg by -// final TR/ACC encoding) -> real ZAMES instruction. -// -// msettile*, msetcfg/mgetcfg, minit, *.whole and the four ZMASYNC -// instructions are selected directly to real instructions. -// -// mmacc operand reorder: LLVM IR operands are A, B, C (the intrinsic -// overload suffix lists the C, A, B *type* sequence, not the operand order); -// the typed pseudo and real instruction use the machine order C, B, A. -// -//===----------------------------------------------------------------------===// - -// Defensive second-layer immediate constraints. The ImmArg/Range intrinsic -// attributes reject out-of-range constants in the IR verifier; these leaves -// additionally prevent ISel from silently truncating an out-of-range target -// constant into the 10-bit tile field or the 3-bit config selector when the -// verifier is disabled or a transform built an illegal DAG. -def zames_tuimm10 : TImmLeaf(Imm); }]>; -def zames_tuimm3 : TImmLeaf(Imm); }]>; - -let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 1, - mayStore = 0, isCodeGenOnly = 1 in { - def PSEUDO_ZAMES_MLA_A_I8_PRE - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MLB_B_I8_PRE - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MLBT_B_I8_PRE - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MLAT_A_I8_PRE - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MLC_C_I32_PRE - : Pseudo<(outs ZamesAccReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MLCT_C_I32_PRE - : Pseudo<(outs ZamesAccReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - - def PSEUDO_ZAMES_MLA_A_I8 - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; - } - def PSEUDO_ZAMES_MLB_B_I8 - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; - } - def PSEUDO_ZAMES_MLBT_B_I8 - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; - } - def PSEUDO_ZAMES_MLAT_A_I8 - : Pseudo<(outs ZamesTileReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; - } - def PSEUDO_ZAMES_MLC_C_I32 - : Pseudo<(outs ZamesAccReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; - } - def PSEUDO_ZAMES_MLCT_C_I32 - : Pseudo<(outs ZamesAccReg:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; - } -} - -let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, - mayStore = 0, isCodeGenOnly = 1 in { - def PSEUDO_ZAMES_MZERO_C_I32_PRE : Pseudo<(outs ZamesAccReg:$md), (ins), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MMACC_I8_I32_PRE - : Pseudo<(outs ZamesAccReg:$md_wb), - (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1), []> { - let Constraints = "$md_wb = $md"; - let Size = 28; - } - - def PSEUDO_ZAMES_MZERO_C_I32 - : Pseudo<(outs ZamesAccReg:$md), (ins GPR:$cfg), []> { - let Size = 8; - } - def PSEUDO_ZAMES_MMACC_I8_I32 - : Pseudo<(outs ZamesAccReg:$md_wb), - (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1, GPR:$cfg_c, - GPR:$cfg_b, GPR:$cfg_a), []> { - let Constraints = "$md_wb = $md"; - let Size = 16; - } -} - -let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, - mayStore = 1, isCodeGenOnly = 1 in { - def PSEUDO_ZAMES_MSC_C_I32_PRE - : Pseudo<(outs), - (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MSA_A_I8_PRE - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MSAT_A_I8_PRE - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MSB_B_I8_PRE - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MSBT_B_I8_PRE - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - def PSEUDO_ZAMES_MSCT_C_I32_PRE - : Pseudo<(outs), - (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; - } - - def PSEUDO_ZAMES_MSC_C_I32 - : Pseudo<(outs), - (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), - []> { - let Size = 8; - } - def PSEUDO_ZAMES_MSA_A_I8 - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), - []> { - let Size = 8; - } - def PSEUDO_ZAMES_MSAT_A_I8 - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), - []> { - let Size = 8; - } - def PSEUDO_ZAMES_MSB_B_I8 - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), - []> { - let Size = 8; - } - def PSEUDO_ZAMES_MSBT_B_I8 - : Pseudo<(outs), - (ins ZamesTileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), - []> { - let Size = 8; - } - def PSEUDO_ZAMES_MSCT_C_I32 - : Pseudo<(outs), - (ins ZamesAccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), - []> { - let Size = 8; - } -} - -class PatZAMESSetTileReg - : Pat<(OpNode (XLenVT GPR:$rs1)), (Inst GPR:$rs1)>; - -// Shared shells for the typed ZAMES pseudo instructions. Concrete defs carry -// the PSEUDO_ZAMES___ names so every instance stays -// reachable from the pre-RA and late C++ expansion switches. -class ZAMESLdPseudoPRE - : Pseudo<(outs RC:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; -} -class ZAMESLdPseudoCFG - : Pseudo<(outs RC:$md), - (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; -} -class ZAMESStPseudoPRE - : Pseudo<(outs), (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { - let Size = 12; -} -class ZAMESStPseudoCFG - : Pseudo<(outs), - (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { - let Size = 8; -} -class ZAMESMZeroPseudoPRE - : Pseudo<(outs RC:$md), (ins), []> { - let Size = 12; -} -class ZAMESMZeroPseudoCFG - : Pseudo<(outs RC:$md), (ins GPR:$cfg), []> { - let Size = 8; -} -class ZAMESMMAccPseudoPRE - : Pseudo<(outs ZamesAccReg:$md_wb), - (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1), []> { - let Constraints = "$md_wb = $md"; - let Size = 28; -} -class ZAMESMMAccPseudoCFG - : Pseudo<(outs ZamesAccReg:$md_wb), - (ins ZamesAccReg:$md, ZamesTileReg:$ms2, ZamesTileReg:$ms1, GPR:$cfg_c, - GPR:$cfg_b, GPR:$cfg_a), []> { - let Constraints = "$md_wb = $md"; - let Size = 16; -} - -// Typed load pseudo instances: A/B tiles use every supported input element; -// C accumulators use fp32. -let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 1, mayStore = 0, - isCodeGenOnly = 1 in { - def PSEUDO_ZAMES_MLA_A_FP8E4_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLA_A_FP8E4 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLA_A_FP8E5_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLA_A_FP8E5 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLA_A_F16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLA_A_F16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLA_A_BF16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLA_A_BF16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLAT_A_FP8E4_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLAT_A_FP8E4 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLAT_A_FP8E5_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLAT_A_FP8E5 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLAT_A_F16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLAT_A_F16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLAT_A_BF16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLAT_A_BF16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLB_B_FP8E4_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLB_B_FP8E4 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLB_B_FP8E5_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLB_B_FP8E5 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLB_B_F16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLB_B_F16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLB_B_BF16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLB_B_BF16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLBT_B_FP8E4_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLBT_B_FP8E4 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLBT_B_FP8E5_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLBT_B_FP8E5 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLBT_B_F16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLBT_B_F16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLBT_B_BF16_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLBT_B_BF16 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLC_C_F32_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLC_C_F32 : ZAMESLdPseudoCFG; - def PSEUDO_ZAMES_MLCT_C_F32_PRE : ZAMESLdPseudoPRE; - def PSEUDO_ZAMES_MLCT_C_F32 : ZAMESLdPseudoCFG; -} - -let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 1, - isCodeGenOnly = 1 in { - def PSEUDO_ZAMES_MSA_A_FP8E4_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSA_A_FP8E4 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSA_A_FP8E5_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSA_A_FP8E5 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSA_A_F16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSA_A_F16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSA_A_BF16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSA_A_BF16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSAT_A_FP8E4_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSAT_A_FP8E4 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSAT_A_FP8E5_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSAT_A_FP8E5 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSAT_A_F16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSAT_A_F16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSAT_A_BF16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSAT_A_BF16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSB_B_FP8E4_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSB_B_FP8E4 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSB_B_FP8E5_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSB_B_FP8E5 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSB_B_F16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSB_B_F16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSB_B_BF16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSB_B_BF16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSBT_B_FP8E4_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSBT_B_FP8E4 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSBT_B_FP8E5_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSBT_B_FP8E5 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSBT_B_F16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSBT_B_F16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSBT_B_BF16_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSBT_B_BF16 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSC_C_F32_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSC_C_F32 : ZAMESStPseudoCFG; - def PSEUDO_ZAMES_MSCT_C_F32_PRE : ZAMESStPseudoPRE; - def PSEUDO_ZAMES_MSCT_C_F32 : ZAMESStPseudoCFG; -} - -// Typed mzero pseudo instances preserve the input/output register partition. -let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, - isCodeGenOnly = 1 in { - def PSEUDO_ZAMES_MZERO_T_I8_PRE : ZAMESMZeroPseudoPRE; - def PSEUDO_ZAMES_MZERO_T_I8 : ZAMESMZeroPseudoCFG; - def PSEUDO_ZAMES_MZERO_T_FP8E4_PRE : ZAMESMZeroPseudoPRE; - def PSEUDO_ZAMES_MZERO_T_FP8E4 : ZAMESMZeroPseudoCFG; - def PSEUDO_ZAMES_MZERO_T_FP8E5_PRE : ZAMESMZeroPseudoPRE; - def PSEUDO_ZAMES_MZERO_T_FP8E5 : ZAMESMZeroPseudoCFG; - def PSEUDO_ZAMES_MZERO_T_F16_PRE : ZAMESMZeroPseudoPRE; - def PSEUDO_ZAMES_MZERO_T_F16 : ZAMESMZeroPseudoCFG; - def PSEUDO_ZAMES_MZERO_T_BF16_PRE : ZAMESMZeroPseudoPRE; - def PSEUDO_ZAMES_MZERO_T_BF16 : ZAMESMZeroPseudoCFG; - def PSEUDO_ZAMES_MZERO_C_F32_PRE : ZAMESMZeroPseudoPRE; - def PSEUDO_ZAMES_MZERO_C_F32 : ZAMESMZeroPseudoCFG; -} - -// Typed mmacc pseudo instances for the six supported combinations. -let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, - isCodeGenOnly = 1 in { - def PSEUDO_ZAMES_MMACC_I8_F32_PRE : ZAMESMMAccPseudoPRE; - def PSEUDO_ZAMES_MMACC_I8_F32 : ZAMESMMAccPseudoCFG; - def PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE : ZAMESMMAccPseudoPRE; - def PSEUDO_ZAMES_MMACC_FP8E4_F32 : ZAMESMMAccPseudoCFG; - def PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE : ZAMESMMAccPseudoPRE; - def PSEUDO_ZAMES_MMACC_FP8E5_F32 : ZAMESMMAccPseudoCFG; - def PSEUDO_ZAMES_MMACC_F16_F32_PRE : ZAMESMMAccPseudoPRE; - def PSEUDO_ZAMES_MMACC_F16_F32 : ZAMESMMAccPseudoCFG; - def PSEUDO_ZAMES_MMACC_BF16_F32_PRE : ZAMESMMAccPseudoPRE; - def PSEUDO_ZAMES_MMACC_BF16_F32 : ZAMESMMAccPseudoCFG; -} - -class PatZAMESLoad - : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1, GPR:$rs2)), - (Inst GPRMemZeroOffset:$rs1, GPR:$rs2)>; - -class PatZAMESStore - : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2), - (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2)>; - -class PatZAMESWholeLoad - : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1)), - (Inst GPRMemZeroOffset:$rs1)>; - -class PatZAMESWholeStore - : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1), - (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1)>; - -let Predicates = [HasStdExtZames, IsRV64] in { - def : Pat<(int_riscv_zames_msettilemi zames_tuimm10:$imm), - (ZAMES_MSETTILEMI timm:$imm)>; - def : Pat<(int_riscv_zames_msettileni zames_tuimm10:$imm), - (ZAMES_MSETTILENI timm:$imm)>; - def : Pat<(int_riscv_zames_msettileki zames_tuimm10:$imm), - (ZAMES_MSETTILEKI timm:$imm)>; - - def : PatZAMESSetTileReg; - def : PatZAMESSetTileReg; - def : PatZAMESSetTileReg; - - def : Pat<(int_riscv_zames_msetcfg zames_tuimm3:$cfg, (XLenVT GPR:$rs1)), - (ZAMES_MSETCFG timm:$cfg, GPR:$rs1)>; - - def : Pat<(int_riscv_zames_minit), (ZAMES_MINIT)>; - def : Pat<(XLenVT (int_riscv_zames_mgetcfg zames_tuimm3:$cfg)), - (ZAMES_MGETCFG timm:$cfg)>; - - def : Pat<(mi32_t (int_riscv_zames_mzero)), - (PSEUDO_ZAMES_MZERO_C_I32_PRE)>; - - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - - def : Pat<(mi32_t (int_riscv_zames_mmacc (mi8_t ZamesTileReg:$rs1), - (mi8_t ZamesTileReg:$rs2), - (mi32_t ZamesAccReg:$rs3))), - (PSEUDO_ZAMES_MMACC_I8_I32_PRE (mi32_t ZamesAccReg:$rs3), - (mi8_t ZamesTileReg:$rs2), - (mi8_t ZamesTileReg:$rs1))>; - - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - def : PatZAMESLoad; - - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeLoad; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - def : PatZAMESWholeStore; - - def : Pat<(mi8_t (int_riscv_zames_mzero)), (PSEUDO_ZAMES_MZERO_T_I8_PRE)>; - def : Pat<(mf8e4_t (int_riscv_zames_mzero)), - (PSEUDO_ZAMES_MZERO_T_FP8E4_PRE)>; - def : Pat<(mf8e5_t (int_riscv_zames_mzero)), - (PSEUDO_ZAMES_MZERO_T_FP8E5_PRE)>; - def : Pat<(mf16_t (int_riscv_zames_mzero)), (PSEUDO_ZAMES_MZERO_T_F16_PRE)>; - def : Pat<(mbf16_t (int_riscv_zames_mzero)), - (PSEUDO_ZAMES_MZERO_T_BF16_PRE)>; - def : Pat<(mf32_t (int_riscv_zames_mzero)), (PSEUDO_ZAMES_MZERO_C_F32_PRE)>; - - def : Pat<(mf32_t (int_riscv_zames_mmacc (mi8_t ZamesTileReg:$rs1), - (mi8_t ZamesTileReg:$rs2), - (mf32_t ZamesAccReg:$rs3))), - (PSEUDO_ZAMES_MMACC_I8_F32_PRE (mf32_t ZamesAccReg:$rs3), - (mi8_t ZamesTileReg:$rs2), - (mi8_t ZamesTileReg:$rs1))>; - def : Pat<(mf32_t (int_riscv_zames_mmacc (mf8e4_t ZamesTileReg:$rs1), - (mf8e4_t ZamesTileReg:$rs2), - (mf32_t ZamesAccReg:$rs3))), - (PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE (mf32_t ZamesAccReg:$rs3), - (mf8e4_t ZamesTileReg:$rs2), - (mf8e4_t ZamesTileReg:$rs1))>; - def : Pat<(mf32_t (int_riscv_zames_mmacc (mf8e5_t ZamesTileReg:$rs1), - (mf8e5_t ZamesTileReg:$rs2), - (mf32_t ZamesAccReg:$rs3))), - (PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE (mf32_t ZamesAccReg:$rs3), - (mf8e5_t ZamesTileReg:$rs2), - (mf8e5_t ZamesTileReg:$rs1))>; - def : Pat<(mf32_t (int_riscv_zames_mmacc (mf16_t ZamesTileReg:$rs1), - (mf16_t ZamesTileReg:$rs2), - (mf32_t ZamesAccReg:$rs3))), - (PSEUDO_ZAMES_MMACC_F16_F32_PRE (mf32_t ZamesAccReg:$rs3), - (mf16_t ZamesTileReg:$rs2), - (mf16_t ZamesTileReg:$rs1))>; - def : Pat<(mf32_t (int_riscv_zames_mmacc (mbf16_t ZamesTileReg:$rs1), - (mbf16_t ZamesTileReg:$rs2), - (mf32_t ZamesAccReg:$rs3))), - (PSEUDO_ZAMES_MMACC_BF16_F32_PRE (mf32_t ZamesAccReg:$rs3), - (mbf16_t ZamesTileReg:$rs2), - (mbf16_t ZamesTileReg:$rs1))>; - - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; - def : PatZAMESStore; -} - -let Predicates = [HasStdExtZmasync, IsRV64] in { - def : Pat<(msync_t (int_riscv_zmasync_msyncregreset)), - (ZMASYNC_MSYNCREGRESET)>; - def : Pat<(msync_t (int_riscv_zmasync_mrelease (msync_t SyncR:$sync))), - (ZMASYNC_MRELEASE (msync_t SyncR:$sync))>; - def : Pat<(int_riscv_zmasync_macquire (msync_t SyncR:$sync), - (XLenVT GPR:$rs1)), - (ZMASYNC_MACQUIRE (msync_t SyncR:$sync), (XLenVT GPR:$rs1))>; - def : Pat<(int_riscv_zmasync_mfence), (ZMASYNC_MFENCE)>; -} - diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td new file mode 100644 index 00000000000000..c5cea4a329c3e0 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td @@ -0,0 +1,96 @@ +//===-- RISCVRegisterInfoXSAIZames.td - ZAMES/ZMASYNC registers -*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC register definitions. +// +// - 4 Tile Registers (tr0-tr3): input matrices A and B +// - 4 Accumulation Registers (acc0-acc3): output/accumulation matrix C +// - 16 Sync Registers (sync0-sync15): ZMASYNC synchronization +// +// RegisterClass Size values are placeholders; the real MLEN/AMUL ABI is not +// part of the MC-level migration. +// +//===----------------------------------------------------------------------===// + +let Namespace = "RISCV" in { + +// Base class for Tile Registers +class XSAI_ZAMES_TileRegBase Enc, string n> : Register { + let HWEncoding{2-0} = Enc; + let HWEncoding{4-3} = 0b00; +} + +// Keep ZAMES records distinct from BOSC AME's identically spelled registers. +def XSAI_ZAMES_TR0 : XSAI_ZAMES_TileRegBase<0, "tr0">; +def XSAI_ZAMES_TR1 : XSAI_ZAMES_TileRegBase<1, "tr1">; +def XSAI_ZAMES_TR2 : XSAI_ZAMES_TileRegBase<2, "tr2">; +def XSAI_ZAMES_TR3 : XSAI_ZAMES_TileRegBase<3, "tr3">; + +// Base class for Accumulation Registers +class XSAI_ZAMES_AccRegBase Enc, string n> : Register { + let HWEncoding{2-0} = Enc; + let HWEncoding{4-3} = 0b00; +} + +// Define 4 Accumulation Registers: acc0-acc3 occupy the hardware register +// slots 4-7. +def XSAI_ZAMES_ACC0 : XSAI_ZAMES_AccRegBase<4, "acc0">; +def XSAI_ZAMES_ACC1 : XSAI_ZAMES_AccRegBase<5, "acc1">; +def XSAI_ZAMES_ACC2 : XSAI_ZAMES_AccRegBase<6, "acc2">; +def XSAI_ZAMES_ACC3 : XSAI_ZAMES_AccRegBase<7, "acc3">; + +// ZMASYNC Sync Registers (sync0-sync15) +class XSAI_ZMASYNC_SyncReg Enc, string n> : Register { + let HWEncoding{4-0} = Enc; +} + +foreach Index = 0-15 in { + def XSAI_ZMASYNC_SYNC#Index : XSAI_ZMASYNC_SyncReg; +} + +} // End Namespace = "RISCV" + +defvar xsai_mi8_t = riscv_xsai_ame_tile_i8; +defvar xsai_mi32_t = riscv_xsai_ame_tile_i32; +defvar xsai_msync_t = riscv_xsai_ame_sync; +defvar xsai_mf8e4_t = riscv_xsai_ame_tile_fp8e4; +defvar xsai_mf8e5_t = riscv_xsai_ame_tile_fp8e5; +defvar xsai_mf16_t = riscv_xsai_ame_tile_f16; +defvar xsai_mbf16_t = riscv_xsai_ame_tile_bf16; +defvar xsai_mf32_t = riscv_xsai_ame_tile_f32; + +// Tile Register class (tr0-tr3) +// Note: Size is set to 256 as a placeholder; actual size depends on MLEN +def XSAI_ZAMES_TileReg + : RegisterClass<"RISCV", [xsai_mi8_t, xsai_mf8e4_t, xsai_mf8e5_t, xsai_mf16_t, xsai_mbf16_t], 256, + (add XSAI_ZAMES_TR0, XSAI_ZAMES_TR1, XSAI_ZAMES_TR2, XSAI_ZAMES_TR3)> { + let Size = 256; // Placeholder: actual MLEN is hardware-defined +} + +// Accumulation Register class (acc0-acc3) +def XSAI_ZAMES_AccReg + : RegisterClass<"RISCV", [xsai_mi32_t, xsai_mf32_t], 1024, + (add XSAI_ZAMES_ACC0, XSAI_ZAMES_ACC1, XSAI_ZAMES_ACC2, XSAI_ZAMES_ACC3)> { + let Size = 1024; // Placeholder: actual MLEN×AMUL is hardware-defined +} + +// Union of XSAI_ZAMES_TileReg and XSAI_ZAMES_AccReg used only as a machine-instruction +// operand class for real instructions whose encoding accepts both register +// files (e.g. mzero). Virtual registers are still allocated through +// XSAI_ZAMES_TileReg or XSAI_ZAMES_AccReg according to their MVT; this class never +// participates in register allocation. +def XSAI_ZAMES_TileOrAccReg : RegisterClass<"RISCV", + [xsai_mi8_t, xsai_mf8e4_t, xsai_mf8e5_t, xsai_mi32_t, xsai_mf16_t, + xsai_mbf16_t, xsai_mf32_t], 1024, + (add XSAI_ZAMES_TileReg, XSAI_ZAMES_AccReg)> { + let Size = 1024; // Placeholder, matches XSAI_ZAMES_AccReg +} + +// Sync register class used by ZMASYNC instructions. +def XSAI_ZMASYNC_SyncR : RegisterClass<"RISCV", [xsai_msync_t], 64, + (add (sequence "XSAI_ZMASYNC_SYNC%u", 0, 15))>; diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td b/llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td deleted file mode 100644 index 8b82d60fd0a7c7..00000000000000 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfoZames.td +++ /dev/null @@ -1,96 +0,0 @@ -//===-- RISCVRegisterInfoZames.td - ZAMES/ZMASYNC registers -*- tablegen -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// ZAMES/ZMASYNC register definitions. -// -// - 4 Tile Registers (tr0-tr3): input matrices A and B -// - 4 Accumulation Registers (acc0-acc3): output/accumulation matrix C -// - 16 Sync Registers (sync0-sync15): ZMASYNC synchronization -// -// RegisterClass Size values are placeholders; the real MLEN/AMUL ABI is not -// part of the MC-level migration. -// -//===----------------------------------------------------------------------===// - -let Namespace = "RISCV" in { - -// Base class for Tile Registers -class AMETileReg Enc, string n> : Register { - let HWEncoding{2-0} = Enc; - let HWEncoding{4-3} = 0b00; -} - -// Keep ZAMES records distinct from BOSC AME's identically spelled registers. -def ZAMES_TR0 : AMETileReg<0, "tr0">; -def ZAMES_TR1 : AMETileReg<1, "tr1">; -def ZAMES_TR2 : AMETileReg<2, "tr2">; -def ZAMES_TR3 : AMETileReg<3, "tr3">; - -// Base class for Accumulation Registers -class AMEAccReg Enc, string n> : Register { - let HWEncoding{2-0} = Enc; - let HWEncoding{4-3} = 0b00; -} - -// Define 4 Accumulation Registers: acc0-acc3 occupy the hardware register -// slots 4-7. -def ZAMES_ACC0 : AMEAccReg<4, "acc0">; -def ZAMES_ACC1 : AMEAccReg<5, "acc1">; -def ZAMES_ACC2 : AMEAccReg<6, "acc2">; -def ZAMES_ACC3 : AMEAccReg<7, "acc3">; - -// ZMASYNC Sync Registers (sync0-sync15) -class ZMASyncReg Enc, string n> : Register { - let HWEncoding{4-0} = Enc; -} - -foreach Index = 0-15 in { - def SYNC#Index : ZMASyncReg; -} - -} // End Namespace = "RISCV" - -defvar mi8_t = riscv_ame_tile_i8; -defvar mi32_t = riscv_ame_tile_i32; -defvar msync_t = riscv_ame_sync; -defvar mf8e4_t = riscv_ame_tile_fp8e4; -defvar mf8e5_t = riscv_ame_tile_fp8e5; -defvar mf16_t = riscv_ame_tile_f16; -defvar mbf16_t = riscv_ame_tile_bf16; -defvar mf32_t = riscv_ame_tile_f32; - -// Tile Register class (tr0-tr3) -// Note: Size is set to 256 as a placeholder; actual size depends on MLEN -def ZamesTileReg - : RegisterClass<"RISCV", [mi8_t, mf8e4_t, mf8e5_t, mf16_t, mbf16_t], 256, - (add ZAMES_TR0, ZAMES_TR1, ZAMES_TR2, ZAMES_TR3)> { - let Size = 256; // Placeholder: actual MLEN is hardware-defined -} - -// Accumulation Register class (acc0-acc3) -def ZamesAccReg - : RegisterClass<"RISCV", [mi32_t, mf32_t], 1024, - (add ZAMES_ACC0, ZAMES_ACC1, ZAMES_ACC2, ZAMES_ACC3)> { - let Size = 1024; // Placeholder: actual MLEN×AMUL is hardware-defined -} - -// Union of ZamesTileReg and ZamesAccReg used only as a machine-instruction -// operand class for real instructions whose encoding accepts both register -// files (e.g. mzero). Virtual registers are still allocated through -// ZamesTileReg or ZamesAccReg according to their MVT; this class never -// participates in register allocation. -def ZamesTileOrAccReg : RegisterClass<"RISCV", - [mi8_t, mf8e4_t, mf8e5_t, mi32_t, mf16_t, - mbf16_t, mf32_t], 1024, - (add ZamesTileReg, ZamesAccReg)> { - let Size = 1024; // Placeholder, matches ZamesAccReg -} - -// Sync register class used by ZMASYNC instructions. -def SyncR : RegisterClass<"RISCV", [msync_t], 64, - (add (sequence "SYNC%u", 0, 15))>; diff --git a/llvm/test/Assembler/riscv-zames-intrinsics.ll b/llvm/test/Assembler/riscv-zames-intrinsics.ll index 1673760fb3cfdc..12385aa5099632 100644 --- a/llvm/test/Assembler/riscv-zames-intrinsics.ll +++ b/llvm/test/Assembler/riscv-zames-intrinsics.ll @@ -8,123 +8,123 @@ ; --- Non-overloaded configuration intrinsics ---------------------------- ; CHECK-LABEL: define void @config( -; CHECK: call void @llvm.riscv.zames.msettilemi(i64 4) -; CHECK: call void @llvm.riscv.zames.msettileni(i64 8) -; CHECK: call void @llvm.riscv.zames.msettileki(i64 16) -; CHECK: call void @llvm.riscv.zames.msettilem(i64 %m) -; CHECK: call void @llvm.riscv.zames.msettilen(i64 %n) -; CHECK: call void @llvm.riscv.zames.msettilek(i64 %k) -; CHECK: call void @llvm.riscv.zames.msetcfg(i64 4, i64 %payload) -; CHECK: %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 4) -; CHECK: call void @llvm.riscv.zames.minit() +; CHECK: call void @llvm.riscv.xsai.zames.msettilemi(i64 4) +; CHECK: call void @llvm.riscv.xsai.zames.msettileni(i64 8) +; CHECK: call void @llvm.riscv.xsai.zames.msettileki(i64 16) +; CHECK: call void @llvm.riscv.xsai.zames.msettilem(i64 %m) +; CHECK: call void @llvm.riscv.xsai.zames.msettilen(i64 %n) +; CHECK: call void @llvm.riscv.xsai.zames.msettilek(i64 %k) +; CHECK: call void @llvm.riscv.xsai.zames.msetcfg(i64 4, i64 %payload) +; CHECK: %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 4) +; CHECK: call void @llvm.riscv.xsai.zames.minit() define void @config(i64 %m, i64 %n, i64 %k, i64 %payload) { - call void @llvm.riscv.zames.msettilemi(i64 4) - call void @llvm.riscv.zames.msettileni(i64 8) - call void @llvm.riscv.zames.msettileki(i64 16) - call void @llvm.riscv.zames.msettilem(i64 %m) - call void @llvm.riscv.zames.msettilen(i64 %n) - call void @llvm.riscv.zames.msettilek(i64 %k) - call void @llvm.riscv.zames.msetcfg(i64 4, i64 %payload) - %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 4) - call void @llvm.riscv.zames.minit() + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 8) + call void @llvm.riscv.xsai.zames.msettileki(i64 16) + call void @llvm.riscv.xsai.zames.msettilem(i64 %m) + call void @llvm.riscv.xsai.zames.msettilen(i64 %n) + call void @llvm.riscv.xsai.zames.msettilek(i64 %k) + call void @llvm.riscv.xsai.zames.msetcfg(i64 4, i64 %payload) + %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 4) + call void @llvm.riscv.xsai.zames.minit() ret void } ; --- Overloaded load/store on int8 tiles -------------------------------- ; CHECK-LABEL: define void @load_store_i8( -; CHECK: %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %pa, i64 %s) -; CHECK: %at = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %pa, i64 %s) -; CHECK: %aw = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %pa) -; CHECK: call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %pa, i64 %s) -; CHECK: call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %at, ptr %pa, i64 %s) -; CHECK: call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %aw, ptr %pa) +; CHECK: %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) +; CHECK: %at = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) +; CHECK: %aw = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %pa) +; CHECK: call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %at, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %aw, ptr %pa) define void @load_store_i8(ptr %pa, i64 %s) { - %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %pa, i64 %s) - %at = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %pa, i64 %s) - %aw = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %pa) - call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %pa, i64 %s) - call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %at, ptr %pa, i64 %s) - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %aw, ptr %pa) + %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) + %at = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) + %aw = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %pa) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %pa, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %at, ptr %pa, i64 %s) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %aw, ptr %pa) ret void } ; --- B/C tile class + transposed/whole on int32 accumulator ------------- ; CHECK-LABEL: define void @load_store_bc( -; CHECK: %b = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.triscv.ame.tile.int32t(ptr %pb, i64 %s) -; CHECK: %bt = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlbt.triscv.ame.tile.int32t(ptr %pb, i64 %s) -; CHECK: %bw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int32t(ptr %pb) -; CHECK: %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %pc, i64 %s) -; CHECK: %ct = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %pc, i64 %s) -; CHECK: %cw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %pc) -; CHECK: call void @llvm.riscv.zames.msb.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %b, ptr %pb, i64 %s) -; CHECK: call void @llvm.riscv.zames.msbt.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bt, ptr %pb, i64 %s) -; CHECK: call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bw, ptr %pb) -; CHECK: call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %pc, i64 %s) -; CHECK: call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %ct, ptr %pc, i64 %s) -; CHECK: call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %cw, ptr %pc) +; CHECK: %b = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bt = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int32t(ptr %pb) +; CHECK: %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) +; CHECK: %ct = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) +; CHECK: %cw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %pc) +; CHECK: call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %b, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bt, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bw, ptr %pb) +; CHECK: call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %ct, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %cw, ptr %pc) define void @load_store_bc(ptr %pb, ptr %pc, i64 %s) { - %b = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.triscv.ame.tile.int32t(ptr %pb, i64 %s) - %bt = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlbt.triscv.ame.tile.int32t(ptr %pb, i64 %s) - %bw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int32t(ptr %pb) - %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %pc, i64 %s) - %ct = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %pc, i64 %s) - %cw = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %pc) - call void @llvm.riscv.zames.msb.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %b, ptr %pb, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bt, ptr %pb, i64 %s) - call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %bw, ptr %pb) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %pc, i64 %s) - call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %ct, ptr %pc, i64 %s) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %cw, ptr %pc) + %b = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) + %bt = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) + %bw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int32t(ptr %pb) + %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) + %ct = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) + %cw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %pc) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %b, ptr %pb, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bt, ptr %pb, i64 %s) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bw, ptr %pb) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %pc, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %ct, ptr %pc, i64 %s) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %cw, ptr %pc) ret void } ; --- mzero / mmacc ------------------------------------------------------ -; CHECK-LABEL: define target("riscv.ame.tile.int32") @zero_acc() { -; CHECK: %z = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() -; CHECK: ret target("riscv.ame.tile.int32") %z -define target("riscv.ame.tile.int32") @zero_acc() { - %z = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - ret target("riscv.ame.tile.int32") %z +; CHECK-LABEL: define target("riscv.xsai.ame.tile.int32") @zero_acc() { +; CHECK: %z = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() +; CHECK: ret target("riscv.xsai.ame.tile.int32") %z +define target("riscv.xsai.ame.tile.int32") @zero_acc() { + %z = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + ret target("riscv.xsai.ame.tile.int32") %z } -; CHECK-LABEL: define target("riscv.ame.tile.int32") @matmul( -; CHECK: %r = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, target("riscv.ame.tile.int8") %b, target("riscv.ame.tile.int32") %c) -define target("riscv.ame.tile.int32") @matmul(target("riscv.ame.tile.int8") %a, target("riscv.ame.tile.int8") %b, target("riscv.ame.tile.int32") %c) { - %r = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, target("riscv.ame.tile.int8") %b, target("riscv.ame.tile.int32") %c) - ret target("riscv.ame.tile.int32") %r +; CHECK-LABEL: define target("riscv.xsai.ame.tile.int32") @matmul( +; CHECK: %r = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, target("riscv.xsai.ame.tile.int8") %b, target("riscv.xsai.ame.tile.int32") %c) +define target("riscv.xsai.ame.tile.int32") @matmul(target("riscv.xsai.ame.tile.int8") %a, target("riscv.xsai.ame.tile.int8") %b, target("riscv.xsai.ame.tile.int32") %c) { + %r = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, target("riscv.xsai.ame.tile.int8") %b, target("riscv.xsai.ame.tile.int32") %c) + ret target("riscv.xsai.ame.tile.int32") %r } ; --- Every floating tile type parses and prints ------------------------- ; CHECK-LABEL: define void @fp_types( -; CHECK: call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr %p, i64 %s) -; CHECK: call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr %p, i64 %s) -; CHECK: call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr %p, i64 %s) -; CHECK: call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr %p, i64 %s) -; CHECK: call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mla.triscv.ame.tile.fp32t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp32t(ptr %p, i64 %s) define void @fp_types(ptr %p, i64 %s) { - %e4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr %p, i64 %s) - %e5 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr %p, i64 %s) - %f16 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr %p, i64 %s) - %bf16 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr %p, i64 %s) - %f32 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mla.triscv.ame.tile.fp32t(ptr %p, i64 %s) + %e4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr %p, i64 %s) + %e5 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr %p, i64 %s) + %f16 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr %p, i64 %s) + %bf16 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr %p, i64 %s) + %f32 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp32t(ptr %p, i64 %s) ret void } ; --- ZMASYNC ------------------------------------------------------------ ; CHECK-LABEL: define void @sync_ops( -; CHECK: %s0 = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset.triscv.ame.synct() -; CHECK: %s1 = call target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease.triscv.ame.synct(target("riscv.ame.sync") %s0) -; CHECK: call void @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync") %s1, i64 %sem) -; CHECK: call void @llvm.riscv.zmasync.mfence() +; CHECK: %s0 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.ame.synct() +; CHECK: %s1 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s0) +; CHECK: call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s1, i64 %sem) +; CHECK: call void @llvm.riscv.xsai.zmasync.mfence() define void @sync_ops(i64 %sem) { - %s0 = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset.triscv.ame.synct() - %s1 = call target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease.triscv.ame.synct(target("riscv.ame.sync") %s0) - call void @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync") %s1, i64 %sem) - call void @llvm.riscv.zmasync.mfence() + %s0 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.ame.synct() + %s1 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s0) + call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s1, i64 %sem) + call void @llvm.riscv.xsai.zmasync.mfence() ret void } diff --git a/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll index 8700076394ed88..6326c55b424dea 100644 --- a/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll +++ b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll @@ -10,17 +10,17 @@ ; CHECK: error: {{.*}}ZAMES/ZMASYNC code generation requires RV64 ; CHECK-NOT: Cannot select -define target("riscv.ame.tile.int8") @rv32_zames(ptr %p, i64 %stride) { - %t = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %p, i64 %stride) - ret target("riscv.ame.tile.int8") %t +define target("riscv.xsai.ame.tile.int8") @rv32_zames(ptr %p, i64 %stride) { + %t = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %p, i64 %stride) + ret target("riscv.xsai.ame.tile.int8") %t } define void @rv32_zmasync() { - %s = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() - call void @llvm.riscv.zmasync.mfence() + %s = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() + call void @llvm.riscv.xsai.zmasync.mfence() ret void } -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() -declare void @llvm.riscv.zmasync.mfence() +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() +declare void @llvm.riscv.xsai.zmasync.mfence() diff --git a/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll index 681ca14c369d6e..f5f1170d9d30b0 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll @@ -12,9 +12,9 @@ define void @zames_settile_immediates() { ; CHECK-NEXT: msettileni 8 ; CHECK-NEXT: msettileki 16 ; CHECK-NEXT: ret - call void @llvm.riscv.zames.msettilemi(i64 4) - call void @llvm.riscv.zames.msettileni(i64 8) - call void @llvm.riscv.zames.msettileki(i64 16) + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 8) + call void @llvm.riscv.xsai.zames.msettileki(i64 16) ret void } @@ -27,9 +27,9 @@ define void @zames_settile_immediate_boundaries() { ; CHECK-NEXT: msettileni 1023 ; CHECK-NEXT: msettileki 0 ; CHECK-NEXT: ret - call void @llvm.riscv.zames.msettilemi(i64 0) - call void @llvm.riscv.zames.msettileni(i64 1023) - call void @llvm.riscv.zames.msettileki(i64 0) + call void @llvm.riscv.xsai.zames.msettilemi(i64 0) + call void @llvm.riscv.xsai.zames.msettileni(i64 1023) + call void @llvm.riscv.xsai.zames.msettileki(i64 0) ret void } @@ -43,9 +43,9 @@ define void @zames_settile_registers(i64 %m, i64 %n, i64 %k) { ; CHECK-OBJ: msettilem a0 ; CHECK-OBJ-NEXT: msettilen a1 ; CHECK-OBJ-NEXT: msettilek a2 - call void @llvm.riscv.zames.msettilem(i64 %m) - call void @llvm.riscv.zames.msettilen(i64 %n) - call void @llvm.riscv.zames.msettilek(i64 %k) + call void @llvm.riscv.xsai.zames.msettilem(i64 %m) + call void @llvm.riscv.xsai.zames.msettilen(i64 %n) + call void @llvm.riscv.xsai.zames.msettilek(i64 %k) ret void } @@ -56,7 +56,7 @@ define void @zames_minit() { ; CHECK-NEXT: ret ; CHECK-OBJ-LABEL: : ; CHECK-OBJ: minit{{$}} - call void @llvm.riscv.zames.minit() + call void @llvm.riscv.xsai.zames.minit() ret void } @@ -65,8 +65,8 @@ define i64 @zames_cfg_intrinsics(i64 %value) { ; CHECK: msetcfg mcfg3, a0 ; CHECK-NEXT: mgetcfg a0, mcfg3 ; CHECK-NEXT: ret - call void @llvm.riscv.zames.msetcfg(i64 3, i64 %value) - %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 3) + call void @llvm.riscv.xsai.zames.msetcfg(i64 3, i64 %value) + %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 3) ret i64 %cfg } @@ -75,8 +75,8 @@ define i64 @zames_cfg_immediate_boundaries(i64 %value) { ; CHECK: msetcfg mcfg0, a0 ; CHECK-NEXT: mgetcfg a0, mcfg7 ; CHECK-NEXT: ret - call void @llvm.riscv.zames.msetcfg(i64 0, i64 %value) - %cfg = call i64 @llvm.riscv.zames.mgetcfg(i64 7) + call void @llvm.riscv.xsai.zames.msetcfg(i64 0, i64 %value) + %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 7) ret i64 %cfg } @@ -100,17 +100,17 @@ define void @zames_straight_line(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %str ; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} ; CHECK-NEXT: msc acc[[#C-4]], (a2), a5 ; CHECK: ret - call void @llvm.riscv.zames.msettilemi(i64 4) - call void @llvm.riscv.zames.msettileni(i64 4) - call void @llvm.riscv.zames.msettileki(i64 8) - %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( - target("riscv.ame.tile.int8") %tile_a, - target("riscv.ame.tile.int8") %tile_b, - target("riscv.ame.tile.int32") %zero) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 4) + call void @llvm.riscv.xsai.zames.msettileki(i64 8) + %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( + target("riscv.xsai.ame.tile.int8") %tile_a, + target("riscv.xsai.ame.tile.int8") %tile_b, + target("riscv.xsai.ame.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -134,17 +134,17 @@ define void @zames_mlb_straight_line(ptr %a, ptr %b, ptr %c, ; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} ; CHECK-NEXT: msc acc[[#C-4]], (a2), a5 ; CHECK: ret - call void @llvm.riscv.zames.msettilemi(i64 4) - call void @llvm.riscv.zames.msettileni(i64 4) - call void @llvm.riscv.zames.msettileki(i64 8) - %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( - target("riscv.ame.tile.int8") %tile_a, - target("riscv.ame.tile.int8") %tile_b, - target("riscv.ame.tile.int32") %zero) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 4) + call void @llvm.riscv.xsai.zames.msettileki(i64 8) + %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( + target("riscv.xsai.ame.tile.int8") %tile_a, + target("riscv.xsai.ame.tile.int8") %tile_b, + target("riscv.xsai.ame.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -156,10 +156,10 @@ define void @zames_mlc_straight_line(ptr %c, i64 %stride_c) { ; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} ; CHECK-NEXT: mlc acc[[#C-4]], (a0), a1 ; CHECK: ret - call void @llvm.riscv.zames.msettilemi(i64 4) - call void @llvm.riscv.zames.msettileni(i64 4) - call void @llvm.riscv.zames.msettileki(i64 8) - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 4) + call void @llvm.riscv.xsai.zames.msettileki(i64 8) + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) ret void } @@ -174,10 +174,10 @@ define void @zames_transpose_load_store(ptr %a, ptr %c, i64 %stride_a, i64 %stri ; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} ; CHECK-NEXT: msct acc[[#C-4]], (a1), a3 ; CHECK: ret - %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) - %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) - call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %stride_a) - call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %stride_c) + %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) + %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %stride_a) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %stride_c) ret void } @@ -198,13 +198,13 @@ define void @zames_store_variants(ptr %a, ptr %b, ptr %c, i64 %stride) { ; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} ; CHECK-NEXT: msct acc[[#C-4]], (a2), a3 ; CHECK: ret - %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride) - %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride) - %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride) - call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %stride) - call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %stride) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %stride) - call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %stride) + %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride) + %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride) + %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %stride) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %stride) ret void } @@ -226,41 +226,41 @@ define void @zames_whole(ptr %src, ptr %dst) { ; CHECK-OBJ-NEXT: msa.whole tr0, (a1) ; CHECK-OBJ-NEXT: msb.whole tr1, (a1) ; CHECK-OBJ-NEXT: msc.whole acc0, (a1) - %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) - %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) - %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) + %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) ret void } -declare void @llvm.riscv.zames.msettilemi(i64) -declare void @llvm.riscv.zames.msettileni(i64) -declare void @llvm.riscv.zames.msettileki(i64) -declare void @llvm.riscv.zames.msettilem(i64) -declare void @llvm.riscv.zames.msettilen(i64) -declare void @llvm.riscv.zames.msettilek(i64) -declare void @llvm.riscv.zames.minit() -declare void @llvm.riscv.zames.msetcfg(i64, i64) -declare i64 @llvm.riscv.zames.mgetcfg(i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) -declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) +declare void @llvm.riscv.xsai.zames.msettilemi(i64) +declare void @llvm.riscv.xsai.zames.msettileni(i64) +declare void @llvm.riscv.xsai.zames.msettileki(i64) +declare void @llvm.riscv.xsai.zames.msettilem(i64) +declare void @llvm.riscv.xsai.zames.msettilen(i64) +declare void @llvm.riscv.xsai.zames.msettilek(i64) +declare void @llvm.riscv.xsai.zames.minit() +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) +declare i64 @llvm.riscv.xsai.zames.mgetcfg(i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir b/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir index b388c5207bea89..8a8efbc92810b4 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir @@ -15,40 +15,40 @@ name: zames_mir_cfg body: | bb.0: - liveins: $zames_acc2, $zames_tr3, $zames_tr1 + liveins: $xsai_zames_acc2, $xsai_zames_tr3, $xsai_zames_tr1 ; f16 x f16 -> f32 with C = acc2, B = tr3, A = tr1 $x10 = ADDI $x0, 12 $x11 = ADDI $x0, 9 - ; CHECK: ZAMES_MSETCFG 6, {{[^:]*}}$x10{{$}} - ; CHECK-NEXT: ZAMES_MSETCFG 3, {{[^:]*}}$x11{{$}} - ; CHECK-NEXT: ZAMES_MSETCFG 1, {{[^:]*}}$x11{{$}} - ; CHECK-NEXT: $zames_acc2 = ZAMES_MMACC {{[^:]*}}$zames_acc2, {{[^:]*}}$zames_tr3, {{[^:]*}}$zames_tr1{{$}} - $zames_acc2 = PSEUDO_ZAMES_MMACC_F16_F32 $zames_acc2, $zames_tr3, $zames_tr1, $x10, $x11, $x11 + ; CHECK: XSAI_ZAMES_MSETCFG 6, {{[^:]*}}$x10{{$}} + ; CHECK-NEXT: XSAI_ZAMES_MSETCFG 3, {{[^:]*}}$x11{{$}} + ; CHECK-NEXT: XSAI_ZAMES_MSETCFG 1, {{[^:]*}}$x11{{$}} + ; CHECK-NEXT: $xsai_zames_acc2 = XSAI_ZAMES_MMACC {{[^:]*}}$xsai_zames_acc2, {{[^:]*}}$xsai_zames_tr3, {{[^:]*}}$xsai_zames_tr1{{$}} + $xsai_zames_acc2 = PSEUDO_XSAI_ZAMES_MMACC_F16_F32 $xsai_zames_acc2, $xsai_zames_tr3, $xsai_zames_tr1, $x10, $x11, $x11 ; fp16 tile load into tr2 $x12 = ADDI $x0, 9 - ; CHECK: ZAMES_MSETCFG 2, {{[^:]*}}$x12{{$}} - ; CHECK-NEXT: $zames_tr2 = ZAMES_MLA - $zames_tr2 = PSEUDO_ZAMES_MLA_A_F16 $x10, $x11, $x12 + ; CHECK: XSAI_ZAMES_MSETCFG 2, {{[^:]*}}$x12{{$}} + ; CHECK-NEXT: $xsai_zames_tr2 = XSAI_ZAMES_MLA + $xsai_zames_tr2 = PSEUDO_XSAI_ZAMES_MLA_A_F16 $x10, $x11, $x12 ; bf16 tile store from tr3 $x13 = ADDI $x0, 10 - ; CHECK: ZAMES_MSETCFG 3, {{[^:]*}}$x13{{$}} - ; CHECK-NEXT: ZAMES_MSA - PSEUDO_ZAMES_MSA_A_BF16 $zames_tr3, $x10, $x11, $x13 + ; CHECK: XSAI_ZAMES_MSETCFG 3, {{[^:]*}}$x13{{$}} + ; CHECK-NEXT: XSAI_ZAMES_MSA + PSEUDO_XSAI_ZAMES_MSA_A_BF16 $xsai_zames_tr3, $x10, $x11, $x13 ; mzero into an fp16 tile register $x14 = ADDI $x0, 9 - ; CHECK: ZAMES_MSETCFG 1, {{[^:]*}}$x14{{$}} - ; CHECK-NEXT: $zames_tr1 = ZAMES_MZERO{{$}} - $zames_tr1 = PSEUDO_ZAMES_MZERO_T_F16 $x14 + ; CHECK: XSAI_ZAMES_MSETCFG 1, {{[^:]*}}$x14{{$}} + ; CHECK-NEXT: $xsai_zames_tr1 = XSAI_ZAMES_MZERO{{$}} + $xsai_zames_tr1 = PSEUDO_XSAI_ZAMES_MZERO_T_F16 $x14 ; mzero into accumulator acc2 $x15 = ADDI $x0, 12 - ; CHECK: ZAMES_MSETCFG 6, {{[^:]*}}$x15{{$}} - ; CHECK-NEXT: $zames_acc2 = ZAMES_MZERO{{$}} - $zames_acc2 = PSEUDO_ZAMES_MZERO_C_F32 $x15 + ; CHECK: XSAI_ZAMES_MSETCFG 6, {{[^:]*}}$x15{{$}} + ; CHECK-NEXT: $xsai_zames_acc2 = XSAI_ZAMES_MZERO{{$}} + $xsai_zames_acc2 = PSEUDO_XSAI_ZAMES_MZERO_C_F32 $x15 PseudoRET ... diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll index b83d1d03324a07..9b7676930b9965 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll @@ -4,18 +4,18 @@ ; FP8 inputs only accumulate into fp32. An i32 accumulator has no mmacc ; pattern and must fail selection explicitly. -; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.zames.mmacc +; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.xsai.zames.mmacc define void @bad_mmacc_acc(ptr %src, ptr %dst) { - %a4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) - %b4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %dst) - %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) - %r = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %a4, target("riscv.ame.tile.fp8e4") %b4, target("riscv.ame.tile.int32") %c) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %r, ptr %dst) + %a4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) + %b4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %dst) + %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) + %r = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %a4, target("riscv.xsai.ame.tile.fp8e4") %b4, target("riscv.xsai.ame.tile.int32") %c) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %r, ptr %dst) ret void } -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.int32")) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll index 593b383e613d61..e1be37fa3980d9 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll @@ -4,19 +4,19 @@ ; Mixed input element types have no mmacc pattern: fp8e4 x fp8e5 is not a ; supported combination and must fail selection explicitly. -; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.zames.mmacc +; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.xsai.zames.mmacc define void @bad_mmacc(ptr %src, ptr %dst) { - %a4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) - %a5 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr %src) - %c = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) - %r = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e4") %a4, target("riscv.ame.tile.fp8e5") %a5, target("riscv.ame.tile.fp32") %c) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r, ptr %dst) + %a4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) + %a5 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src) + %c = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) + %r = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e4") %a4, target("riscv.xsai.ame.tile.fp8e5") %a5, target("riscv.xsai.ame.tile.fp32") %c) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r, ptr %dst) ret void } -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr) -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp8e5"), target("riscv.ame.tile.fp32")) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr) +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp8e5"), target("riscv.xsai.ame.tile.fp32")) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll index 15a55ead9dae02..b2d26705c9ae07 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll @@ -20,104 +20,104 @@ define void @zames_fp_ab_ls(ptr %src, ptr %dst, i64 %s) { ; ISEL-LABEL: name: zames_fp_ab_ls -; ISEL: PSEUDO_ZAMES_MLA_A_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MSA_A_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MLAT_A_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MSAT_A_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MLB_B_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MSB_B_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MLBT_B_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MSBT_B_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MLA_A_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MSA_A_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MLAT_A_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MSAT_A_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MLB_B_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MSB_B_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MLBT_B_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MSBT_B_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE ; PRERA-LABEL: name: zames_fp_ab_ls ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4 {{.*}}, %[[C8]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_FP8E5 {{.*}}, %[[C7]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_FP8E5 {{.*}}, %[[C7]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5 {{.*}}, %[[C7]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5 {{.*}}, %[[C7]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_FP8E5 {{.*}}, %[[C7]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_FP8E5 {{.*}}, %[[C7]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5 {{.*}}, %[[C7]] :: ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5 {{.*}}, %[[C7]] :: ; POSTRA-LABEL: name: zames_fp_ab_ls ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size ; OBJ-LABEL: : ; OBJ: mla{{[ \t]}} ; OBJ: msa{{[ \t]}} @@ -135,165 +135,165 @@ define void @zames_fp_ab_ls(ptr %src, ptr %dst, i64 %s) { ; OBJ: msb{{[ \t]}} ; OBJ: mlbt{{[ \t]}} ; OBJ: msbt{{[ \t]}} - %t1 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t1, ptr %dst, i64 %s) - %t2 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t2, ptr %dst, i64 %s) - %t3 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t3, ptr %dst, i64 %s) - %t4 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %t4, ptr %dst, i64 %s) - %t5 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t5, ptr %dst, i64 %s) - %t6 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t6, ptr %dst, i64 %s) - %t7 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t7, ptr %dst, i64 %s) - %t8 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %t8, ptr %dst, i64 %s) + %t1 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t8, ptr %dst, i64 %s) ret void } define void @zames_fp_c_ls(ptr %src, ptr %dst, i64 %s) { ; ISEL-LABEL: name: zames_fp_c_ls -; ISEL: PSEUDO_ZAMES_MLC_C_F32_PRE -; ISEL: PSEUDO_ZAMES_MSC_C_F32_PRE -; ISEL: PSEUDO_ZAMES_MLCT_C_F32_PRE -; ISEL: PSEUDO_ZAMES_MSCT_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE ; PRERA-LABEL: name: zames_fp_c_ls ; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MLC_C_F32 {{.*}}, %[[C12]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLC_C_F32 {{.*}}, %[[C12]] :: ; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 -; PRERA-NEXT: PSEUDO_ZAMES_MSC_C_F32 {{.*}}, %[[C12]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_F32 {{.*}}, %[[C12]] :: ; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MLCT_C_F32 {{.*}}, %[[C12]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLCT_C_F32 {{.*}}, %[[C12]] :: ; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 -; PRERA-NEXT: PSEUDO_ZAMES_MSCT_C_F32 {{.*}}, %[[C12]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSCT_C_F32 {{.*}}, %[[C12]] :: ; POSTRA-LABEL: name: zames_fp_c_ls ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLC {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLC {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSC {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSC {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLCT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLCT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSCT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSCT {{.*}}:: (store unknown-size ; OBJ-LABEL: : ; OBJ: mlc{{[ \t]}} ; OBJ: msc{{[ \t]}} ; OBJ: mlct{{[ \t]}} ; OBJ: msct{{[ \t]}} - %c5 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.triscv.ame.tile.fp32t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msc.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %c5, ptr %dst, i64 %s) - %c6 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlct.triscv.ame.tile.fp32t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msct.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %c6, ptr %dst, i64 %s) + %c5 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %c5, ptr %dst, i64 %s) + %c6 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %c6, ptr %dst, i64 %s) ret void } define void @zames_fp16_bf16_ab_ls(ptr %src, ptr %dst, i64 %s) { ; ISEL-LABEL: name: zames_fp16_bf16_ab_ls -; ISEL: PSEUDO_ZAMES_MLA_A_F16_PRE -; ISEL: PSEUDO_ZAMES_MSA_A_F16_PRE -; ISEL: PSEUDO_ZAMES_MLAT_A_F16_PRE -; ISEL: PSEUDO_ZAMES_MSAT_A_F16_PRE -; ISEL: PSEUDO_ZAMES_MLB_B_F16_PRE -; ISEL: PSEUDO_ZAMES_MSB_B_F16_PRE -; ISEL: PSEUDO_ZAMES_MLBT_B_F16_PRE -; ISEL: PSEUDO_ZAMES_MSBT_B_F16_PRE -; ISEL: PSEUDO_ZAMES_MLA_A_BF16_PRE -; ISEL: PSEUDO_ZAMES_MSA_A_BF16_PRE -; ISEL: PSEUDO_ZAMES_MLAT_A_BF16_PRE -; ISEL: PSEUDO_ZAMES_MSAT_A_BF16_PRE -; ISEL: PSEUDO_ZAMES_MLB_B_BF16_PRE -; ISEL: PSEUDO_ZAMES_MSB_B_BF16_PRE -; ISEL: PSEUDO_ZAMES_MLBT_B_BF16_PRE -; ISEL: PSEUDO_ZAMES_MSBT_B_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE ; PRERA-LABEL: name: zames_fp16_bf16_ab_ls ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_F16 {{.*}}, %[[C9]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_F16 {{.*}}, %[[C9]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLA_A_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_BF16 {{.*}}, %[[C10]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: PSEUDO_ZAMES_MSA_A_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_BF16 {{.*}}, %[[C10]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLAT_A_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_BF16 {{.*}}, %[[C10]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: PSEUDO_ZAMES_MSAT_A_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_BF16 {{.*}}, %[[C10]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLB_B_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_BF16 {{.*}}, %[[C10]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: PSEUDO_ZAMES_MSB_B_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_BF16 {{.*}}, %[[C10]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MLBT_B_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_BF16 {{.*}}, %[[C10]] :: ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: PSEUDO_ZAMES_MSBT_B_BF16 {{.*}}, %[[C10]] :: +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_BF16 {{.*}}, %[[C10]] :: ; POSTRA-LABEL: name: zames_fp16_bf16_ab_ls ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: {{.*}} = ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size ; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 -; POSTRA: ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} -; POSTRA-NEXT: ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size ; OBJ-LABEL: : ; OBJ: mla{{[ \t]}} ; OBJ: msa{{[ \t]}} @@ -311,22 +311,22 @@ define void @zames_fp16_bf16_ab_ls(ptr %src, ptr %dst, i64 %s) { ; OBJ: msb{{[ \t]}} ; OBJ: mlbt{{[ \t]}} ; OBJ: msbt{{[ \t]}} - %t1 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msa.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t1, ptr %dst, i64 %s) - %t2 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlat.triscv.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msat.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t2, ptr %dst, i64 %s) - %t3 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.triscv.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msb.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t3, ptr %dst, i64 %s) - %t4 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %t4, ptr %dst, i64 %s) - %t5 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msa.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t5, ptr %dst, i64 %s) - %t6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlat.triscv.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msat.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t6, ptr %dst, i64 %s) - %t7 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.triscv.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msb.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t7, ptr %dst, i64 %s) - %t8 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlbt.triscv.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %t8, ptr %dst, i64 %s) + %t1 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t8, ptr %dst, i64 %s) ret void } @@ -334,16 +334,16 @@ define void @zames_fp16_bf16_ab_ls(ptr %src, ptr %dst, i64 %s) { ; does not read the type config, so no msetcfg is emitted for them. define void @zames_fp_whole(ptr %src, ptr %dst) { ; ISEL-LABEL: name: zames_fp_whole -; ISEL: ZAMES_MLA_WHOLE -; ISEL: ZAMES_MSA_WHOLE -; ISEL: ZAMES_MLB_WHOLE -; ISEL: ZAMES_MSB_WHOLE -; ISEL: ZAMES_MLA_WHOLE -; ISEL: ZAMES_MSA_WHOLE -; ISEL: ZAMES_MLB_WHOLE -; ISEL: ZAMES_MSB_WHOLE -; ISEL: ZAMES_MLC_WHOLE -; ISEL: ZAMES_MSC_WHOLE +; ISEL: XSAI_ZAMES_MLA_WHOLE +; ISEL: XSAI_ZAMES_MSA_WHOLE +; ISEL: XSAI_ZAMES_MLB_WHOLE +; ISEL: XSAI_ZAMES_MSB_WHOLE +; ISEL: XSAI_ZAMES_MLA_WHOLE +; ISEL: XSAI_ZAMES_MSA_WHOLE +; ISEL: XSAI_ZAMES_MLB_WHOLE +; ISEL: XSAI_ZAMES_MSB_WHOLE +; ISEL: XSAI_ZAMES_MLC_WHOLE +; ISEL: XSAI_ZAMES_MSC_WHOLE ; OBJ-LABEL: : ; OBJ-NOT: msetcfg ; OBJ: mla.whole{{[ \t]}} @@ -356,16 +356,16 @@ define void @zames_fp_whole(ptr %src, ptr %dst) { ; OBJ: msb.whole{{[ \t]}} ; OBJ: mlc.whole{{[ \t]}} ; OBJ: msc.whole{{[ \t]}} - %a = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %a, ptr %dst) - %b = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e5t(ptr %src) - call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %b, ptr %dst) - %a16 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp16t(ptr %src) - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %a16, ptr %dst) - %bb16 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.bf16t(ptr %src) - call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %bb16, ptr %dst) - %c = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %c, ptr %dst) + %a = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %a, ptr %dst) + %b = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %b, ptr %dst) + %a16 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp16t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %a16, ptr %dst) + %bb16 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.bf16t(ptr %src) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %bb16, ptr %dst) + %c = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %c, ptr %dst) ret void } @@ -373,43 +373,43 @@ define void @zames_fp_whole(ptr %src, ptr %dst) { ; bound to its payload config and register class. define void @zames_fp_mzero(ptr %dst) { ; ISEL-LABEL: name: zames_fp_mzero -; ISEL: PSEUDO_ZAMES_MZERO_T_I8_PRE -; ISEL: PSEUDO_ZAMES_MZERO_T_FP8E4_PRE -; ISEL: PSEUDO_ZAMES_MZERO_T_FP8E5_PRE -; ISEL: PSEUDO_ZAMES_MZERO_C_I32_PRE -; ISEL: PSEUDO_ZAMES_MZERO_T_F16_PRE -; ISEL: PSEUDO_ZAMES_MZERO_T_BF16_PRE -; ISEL: PSEUDO_ZAMES_MZERO_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE ; PRERA-LABEL: name: zames_fp_mzero ; PRERA: %[[C2:[0-9]+]]:gpr = ADDI $x0, 2 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_I8 %[[C2]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_I8 %[[C2]]{{$}} ; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_FP8E4 %[[C8]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4 %[[C8]]{{$}} ; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_FP8E5 %[[C7]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5 %[[C7]]{{$}} ; PRERA: %[[C4:[0-9]+]]:gpr = ADDI $x0, 4 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MZERO_C_I32 %[[C4]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_I32 %[[C4]]{{$}} ; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_F16 %[[C9]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_F16 %[[C9]]{{$}} ; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: %{{[0-9]+}}:zamestilereg = PSEUDO_ZAMES_MZERO_T_BF16 %[[C10]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_BF16 %[[C10]]{{$}} ; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MZERO_C_F32 %[[C12]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_F32 %[[C12]]{{$}} ; POSTRA-LABEL: name: zames_fp_mzero -; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} -; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} -; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MZERO{{$}} -; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} -; POSTRA: ZAMES_MSETCFG [[#T:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_tr[[#T]] = ZAMES_MZERO{{$}} -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MZERO{{$}} ; OBJ-LABEL: : ; OBJ: mzero{{[ \t]}}tr{{[0-3]}} ; OBJ: mzero{{[ \t]}}tr{{[0-3]}} @@ -418,20 +418,20 @@ define void @zames_fp_mzero(ptr %dst) { ; OBJ: mzero{{[ \t]}}tr{{[0-3]}} ; OBJ: mzero{{[ \t]}}tr{{[0-3]}} ; OBJ: mzero{{[ \t]}}acc{{[0-3]}} - %z1 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mzero.triscv.ame.tile.int8t() - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %z1, ptr %dst) - %z2 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e4t() - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %z2, ptr %dst) - %z3 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e5t() - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %z3, ptr %dst) - %z4 = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %z4, ptr %dst) - %z5 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mzero.triscv.ame.tile.fp16t() - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %z5, ptr %dst) - %z6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mzero.triscv.ame.tile.bf16t() - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %z6, ptr %dst) - %z7 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mzero.triscv.ame.tile.fp32t() - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %z7, ptr %dst) + %z1 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int8t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %z1, ptr %dst) + %z2 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e4t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %z2, ptr %dst) + %z3 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e5t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %z3, ptr %dst) + %z4 = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %z4, ptr %dst) + %z5 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp16t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %z5, ptr %dst) + %z6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.bf16t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %z6, ptr %dst) + %z7 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp32t() + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %z7, ptr %dst) ret void } @@ -441,74 +441,74 @@ define void @zames_fp_mzero(ptr %dst) { ; one msetcfg per physical matrix operand. define void @zames_fp_mmacc(ptr %src, ptr %src2, ptr %dst) { ; ISEL-LABEL: name: zames_fp_mmacc -; ISEL: %[[A1:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE -; ISEL: %[[B1:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE -; ISEL: %[[C1:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE -; ISEL: PSEUDO_ZAMES_MMACC_I8_I32_PRE %[[C1]], {{.*}}%[[B1]], {{.*}}%[[A1]]{{$}} -; ISEL: %[[A2:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE -; ISEL: %[[B2:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE -; ISEL: %[[C2:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE -; ISEL: PSEUDO_ZAMES_MMACC_I8_F32_PRE %[[C2]], {{.*}}%[[B2]], {{.*}}%[[A2]]{{$}} -; ISEL: %[[A3:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE -; ISEL: %[[B3:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE -; ISEL: %[[C3:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE -; ISEL: PSEUDO_ZAMES_MMACC_FP8E4_F32_PRE %[[C3]], {{.*}}%[[B3]], {{.*}}%[[A3]]{{$}} -; ISEL: %[[A4:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE -; ISEL: %[[B4:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE -; ISEL: %[[C4:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE -; ISEL: PSEUDO_ZAMES_MMACC_FP8E5_F32_PRE %[[C4]], {{.*}}%[[B4]], {{.*}}%[[A4]]{{$}} -; ISEL: %[[A5:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE -; ISEL: %[[B5:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE -; ISEL: %[[C5:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE -; ISEL: PSEUDO_ZAMES_MMACC_F16_F32_PRE %[[C5]], {{.*}}%[[B5]], {{.*}}%[[A5]]{{$}} -; ISEL: %[[A6:[0-9]+]]:zamestilereg = ZAMES_MLA_WHOLE -; ISEL: %[[B6:[0-9]+]]:zamestilereg = ZAMES_MLB_WHOLE -; ISEL: %[[C6:[0-9]+]]:zamesaccreg = ZAMES_MLC_WHOLE -; ISEL: PSEUDO_ZAMES_MMACC_BF16_F32_PRE %[[C6]], {{.*}}%[[B6]], {{.*}}%[[A6]]{{$}} +; ISEL: %[[A1:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B1:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C1:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE %[[C1]], {{.*}}%[[B1]], {{.*}}%[[A1]]{{$}} +; ISEL: %[[A2:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B2:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C2:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE %[[C2]], {{.*}}%[[B2]], {{.*}}%[[A2]]{{$}} +; ISEL: %[[A3:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B3:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C3:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE %[[C3]], {{.*}}%[[B3]], {{.*}}%[[A3]]{{$}} +; ISEL: %[[A4:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B4:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C4:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE %[[C4]], {{.*}}%[[B4]], {{.*}}%[[A4]]{{$}} +; ISEL: %[[A5:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B5:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C5:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE %[[C5]], {{.*}}%[[B5]], {{.*}}%[[A5]]{{$}} +; ISEL: %[[A6:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B6:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C6:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE %[[C6]], {{.*}}%[[B6]], {{.*}}%[[A6]]{{$}} ; PRERA-LABEL: name: zames_fp_mmacc ; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 4 ; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 2 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_I32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} ; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 ; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 2 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} ; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 ; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 8 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_FP8E4_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} ; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 ; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 7 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_FP8E5_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} ; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 ; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 9 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_F16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_F16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} ; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 ; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 10 -; PRERA-NEXT: %{{[0-9]+}}:zamesaccreg = PSEUDO_ZAMES_MMACC_BF16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_BF16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} ; POSTRA-LABEL: name: zames_fp_mmacc -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} -; POSTRA: ZAMES_MSETCFG [[#C:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#B:]], {{.*}} -; POSTRA-NEXT: ZAMES_MSETCFG [[#A:]], {{.*}} -; POSTRA-NEXT: {{.*}}$zames_acc[[#C-4]] = ZAMES_MMACC {{.*}}$zames_acc[[#C-4]], {{.*}}$zames_tr[[#B]], {{.*}}$zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} ; OBJ-LABEL: : ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} @@ -516,106 +516,106 @@ define void @zames_fp_mmacc(ptr %src, ptr %src2, ptr %dst) { ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} - %a1 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) - %b1 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src2) - %c1 = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) - %r1 = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a1, target("riscv.ame.tile.int8") %b1, target("riscv.ame.tile.int32") %c1) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %r1, ptr %dst) - %a2 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) - %b2 = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src2) - %c2 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) - %r2 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a2, target("riscv.ame.tile.int8") %b2, target("riscv.ame.tile.fp32") %c2) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r2, ptr %dst) - %a3 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr %src) - %b3 = call target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e4t(ptr %src2) - %c3 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) - %r3 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4") %a3, target("riscv.ame.tile.fp8e4") %b3, target("riscv.ame.tile.fp32") %c3) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r3, ptr %dst) - %a4 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr %src) - %b4 = call target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e5t(ptr %src2) - %c4 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) - %r4 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e5t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5") %a4, target("riscv.ame.tile.fp8e5") %b4, target("riscv.ame.tile.fp32") %c4) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r4, ptr %dst) - %a5 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp16t(ptr %src) - %b5 = call target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp16t(ptr %src2) - %c5 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) - %r5 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp16t.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16") %a5, target("riscv.ame.tile.fp16") %b5, target("riscv.ame.tile.fp32") %c5) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r5, ptr %dst) - %a6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.bf16t(ptr %src) - %b6 = call target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.bf16t(ptr %src2) - %c6 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr %src) - %r6 = call target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.bf16t.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16") %a6, target("riscv.ame.tile.bf16") %b6, target("riscv.ame.tile.fp32") %c6) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32") %r6, ptr %dst) + %a1 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %b1 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src2) + %c1 = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) + %r1 = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a1, target("riscv.xsai.ame.tile.int8") %b1, target("riscv.xsai.ame.tile.int32") %c1) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %r1, ptr %dst) + %a2 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %b2 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src2) + %c2 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) + %r2 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a2, target("riscv.xsai.ame.tile.int8") %b2, target("riscv.xsai.ame.tile.fp32") %c2) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r2, ptr %dst) + %a3 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) + %b3 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src2) + %c3 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) + %r3 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %a3, target("riscv.xsai.ame.tile.fp8e4") %b3, target("riscv.xsai.ame.tile.fp32") %c3) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r3, ptr %dst) + %a4 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src) + %b4 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src2) + %c4 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) + %r4 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e5t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %a4, target("riscv.xsai.ame.tile.fp8e5") %b4, target("riscv.xsai.ame.tile.fp32") %c4) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r4, ptr %dst) + %a5 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp16t(ptr %src) + %b5 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp16t(ptr %src2) + %c5 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) + %r5 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp16t.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %a5, target("riscv.xsai.ame.tile.fp16") %b5, target("riscv.xsai.ame.tile.fp32") %c5) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r5, ptr %dst) + %a6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.bf16t(ptr %src) + %b6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.bf16t(ptr %src2) + %c6 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) + %r6 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.bf16t.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %a6, target("riscv.xsai.ame.tile.bf16") %b6, target("riscv.xsai.ame.tile.fp32") %c6) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r6, ptr %dst) ret void } -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.triscv.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlat.triscv.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.triscv.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.triscv.ame.tile.fp16t(ptr, i64) -declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlat.triscv.ame.tile.fp16t(ptr, i64) -declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.triscv.ame.tile.fp16t(ptr, i64) -declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlbt.triscv.ame.tile.fp16t(ptr, i64) -declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.triscv.ame.tile.bf16t(ptr, i64) -declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlat.triscv.ame.tile.bf16t(ptr, i64) -declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.triscv.ame.tile.bf16t(ptr, i64) -declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlbt.triscv.ame.tile.bf16t(ptr, i64) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.triscv.ame.tile.fp32t(ptr, i64) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlct.triscv.ame.tile.fp32t(ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.zames.msc.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr, i64) -declare void @llvm.riscv.zames.msct.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr, i64) -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e4t(ptr) -declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp16t(ptr) -declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mla.whole.triscv.ame.tile.bf16t(ptr) -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e5t(ptr) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp8e4t(ptr) -declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.fp16t(ptr) -declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.bf16t(ptr) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.fp32t(ptr) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mla.whole.triscv.ame.tile.fp8e5t(ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr) -declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), ptr) -declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), ptr) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.fp32t(target("riscv.ame.tile.fp32"), ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mzero.triscv.ame.tile.int8t() -declare target("riscv.ame.tile.fp8e4") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e4t() -declare target("riscv.ame.tile.fp8e5") @llvm.riscv.zames.mzero.triscv.ame.tile.fp8e5t() -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() -declare target("riscv.ame.tile.fp16") @llvm.riscv.zames.mzero.triscv.ame.tile.fp16t() -declare target("riscv.ame.tile.bf16") @llvm.riscv.zames.mzero.triscv.ame.tile.bf16t() -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mzero.triscv.ame.tile.fp32t() -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.fp32")) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e4t.triscv.ame.tile.fp8e4t(target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp8e4"), target("riscv.ame.tile.fp32")) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp8e5t.triscv.ame.tile.fp8e5t(target("riscv.ame.tile.fp8e5"), target("riscv.ame.tile.fp8e5"), target("riscv.ame.tile.fp32")) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.fp16t.triscv.ame.tile.fp16t(target("riscv.ame.tile.fp16"), target("riscv.ame.tile.fp16"), target("riscv.ame.tile.fp32")) -declare target("riscv.ame.tile.fp32") @llvm.riscv.zames.mmacc.triscv.ame.tile.fp32t.triscv.ame.tile.bf16t.triscv.ame.tile.bf16t(target("riscv.ame.tile.bf16"), target("riscv.ame.tile.bf16"), target("riscv.ame.tile.fp32")) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp16t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp16t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp16t(ptr, i64) +declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr, i64) +declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.bf16t(ptr, i64) +declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.bf16t(ptr, i64) +declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.bf16t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.fp32t(ptr, i64) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.fp32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr, i64) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr) +declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp16t(ptr) +declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.bf16t(ptr) +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e5t(ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e4t(ptr) +declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp16t(ptr) +declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.bf16t(ptr) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int8t() +declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e4t() +declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e5t() +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() +declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp16t() +declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.bf16t() +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp32t() +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.fp32")) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp32")) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e5t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), target("riscv.xsai.ame.tile.fp8e5"), target("riscv.xsai.ame.tile.fp32")) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp16t.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), target("riscv.xsai.ame.tile.fp16"), target("riscv.xsai.ame.tile.fp32")) +declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.bf16t.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), target("riscv.xsai.ame.tile.bf16"), target("riscv.xsai.ame.tile.fp32")) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll b/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll index cd43377d0d909c..df9b8c17844808 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll @@ -10,46 +10,46 @@ ; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/mgetcfg-neg.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=GETCFG-NEG ;--- settilemi-neg.ll -declare void @llvm.riscv.zames.msettilemi(i64) +declare void @llvm.riscv.xsai.zames.msettilemi(i64) define void @f() { ; MI-NEG: Cannot select - call void @llvm.riscv.zames.msettilemi(i64 -1) + call void @llvm.riscv.xsai.zames.msettilemi(i64 -1) ret void } ;--- settileni-1024.ll -declare void @llvm.riscv.zames.msettileni(i64) +declare void @llvm.riscv.xsai.zames.msettileni(i64) define void @f() { ; NI-1024: Cannot select - call void @llvm.riscv.zames.msettileni(i64 1024) + call void @llvm.riscv.xsai.zames.msettileni(i64 1024) ret void } ;--- settileki-1024.ll -declare void @llvm.riscv.zames.msettileki(i64) +declare void @llvm.riscv.xsai.zames.msettileki(i64) define void @f() { ; KI-1024: Cannot select - call void @llvm.riscv.zames.msettileki(i64 1024) + call void @llvm.riscv.xsai.zames.msettileki(i64 1024) ret void } ;--- msetcfg-8.ll -declare void @llvm.riscv.zames.msetcfg(i64, i64) +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) define void @f(i64 %payload) { ; SETCFG-8: Cannot select - call void @llvm.riscv.zames.msetcfg(i64 8, i64 %payload) + call void @llvm.riscv.xsai.zames.msetcfg(i64 8, i64 %payload) ret void } ;--- mgetcfg-neg.ll -declare i64 @llvm.riscv.zames.mgetcfg(i64) +declare i64 @llvm.riscv.xsai.zames.mgetcfg(i64) define i64 @f() { ; GETCFG-NEG: Cannot select - %r = call i64 @llvm.riscv.zames.mgetcfg(i64 -1) + %r = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 -1) ret i64 %r } diff --git a/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll index 4261ce7f53dcc6..df12209354e3d3 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll @@ -5,19 +5,19 @@ define void @zames_selects_typed_pseudos(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %stride_b, i64 %stride_c) { ; CHECK-LABEL: name: zames_selects_typed_pseudos -; CHECK: PSEUDO_ZAMES_MLA_A_I8_PRE -; CHECK: PSEUDO_ZAMES_MLBT_B_I8_PRE -; CHECK: PSEUDO_ZAMES_MZERO_C_I32_PRE -; CHECK: PSEUDO_ZAMES_MMACC_I8_I32_PRE -; CHECK: PSEUDO_ZAMES_MSC_C_I32_PRE - %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( - target("riscv.ame.tile.int8") %tile_a, - target("riscv.ame.tile.int8") %tile_b, - target("riscv.ame.tile.int32") %zero) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) +; CHECK: PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE + %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( + target("riscv.xsai.ame.tile.int8") %tile_a, + target("riscv.xsai.ame.tile.int8") %tile_b, + target("riscv.xsai.ame.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -25,91 +25,91 @@ define void @zames_selects_mlb_pseudo(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %stride_b, i64 %stride_c) { ; CHECK-LABEL: name: zames_selects_mlb_pseudo -; CHECK: PSEUDO_ZAMES_MLA_A_I8_PRE -; CHECK: PSEUDO_ZAMES_MLB_B_I8_PRE -; CHECK: PSEUDO_ZAMES_MZERO_C_I32_PRE -; CHECK: PSEUDO_ZAMES_MMACC_I8_I32_PRE -; CHECK: PSEUDO_ZAMES_MSC_C_I32_PRE - %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( - target("riscv.ame.tile.int8") %tile_a, - target("riscv.ame.tile.int8") %tile_b, - target("riscv.ame.tile.int32") %zero) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) +; CHECK: PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE + %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( + target("riscv.xsai.ame.tile.int8") %tile_a, + target("riscv.xsai.ame.tile.int8") %tile_b, + target("riscv.xsai.ame.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } define void @zames_selects_mlc_pseudo(ptr %c, i64 %stride_c) { ; CHECK-LABEL: name: zames_selects_mlc_pseudo -; CHECK: PSEUDO_ZAMES_MLC_C_I32_PRE - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) +; CHECK: PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) ret void } define void @zames_selects_transpose_and_store_pseudos(ptr %a, ptr %c, i64 %sa, i64 %sc) { ; CHECK-LABEL: name: zames_selects_transpose_and_store_pseudos -; CHECK: PSEUDO_ZAMES_MLAT_A_I8_PRE -; CHECK: PSEUDO_ZAMES_MLCT_C_I32_PRE -; CHECK: PSEUDO_ZAMES_MSAT_A_I8_PRE -; CHECK: PSEUDO_ZAMES_MSCT_C_I32_PRE - %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %a, i64 %sa) - %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %c, i64 %sc) - call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %sa) - call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %sc) +; CHECK: PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE + %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %sc) ret void } define void @zames_selects_ab_store_pseudos(ptr %a, ptr %b, i64 %s) { ; CHECK-LABEL: name: zames_selects_ab_store_pseudos -; CHECK: PSEUDO_ZAMES_MSA_A_I8_PRE -; CHECK: PSEUDO_ZAMES_MSB_B_I8_PRE -; CHECK: PSEUDO_ZAMES_MSBT_B_I8_PRE - %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %s) - %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %s) - call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %s) - call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) +; CHECK: PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE + %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) ret void } define void @zames_selects_whole_direct(ptr %src, ptr %dst) { ; CHECK-LABEL: name: zames_selects_whole_direct ; CHECK-NOT: PSEUDO -; CHECK: ZAMES_MLA_WHOLE -; CHECK: ZAMES_MLB_WHOLE -; CHECK: ZAMES_MLC_WHOLE -; CHECK: ZAMES_MSA_WHOLE -; CHECK: ZAMES_MSB_WHOLE -; CHECK: ZAMES_MSC_WHOLE - %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) - %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) - %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) +; CHECK: XSAI_ZAMES_MLA_WHOLE +; CHECK: XSAI_ZAMES_MLB_WHOLE +; CHECK: XSAI_ZAMES_MLC_WHOLE +; CHECK: XSAI_ZAMES_MSA_WHOLE +; CHECK: XSAI_ZAMES_MSB_WHOLE +; CHECK: XSAI_ZAMES_MSC_WHOLE + %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) ret void } -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) -declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll index 8a19cb530be51d..e2af7d17dbc7e6 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll @@ -15,113 +15,113 @@ define void @zames_mem_ab_i8(ptr %src, ptr %dst, i64 %s) { ; ISEL-LABEL: name: zames_mem_ab_i8 -; ISEL: PSEUDO_ZAMES_MLA_A_I8_PRE {{.*}}:: (load unknown-size -; ISEL: PSEUDO_ZAMES_MLAT_A_I8_PRE {{.*}}:: (load unknown-size -; ISEL: PSEUDO_ZAMES_MLB_B_I8_PRE {{.*}}:: (load unknown-size -; ISEL: PSEUDO_ZAMES_MLBT_B_I8_PRE {{.*}}:: (load unknown-size -; ISEL: PSEUDO_ZAMES_MSA_A_I8_PRE {{.*}}:: (store unknown-size -; ISEL: PSEUDO_ZAMES_MSAT_A_I8_PRE {{.*}}:: (store unknown-size -; ISEL: PSEUDO_ZAMES_MSB_B_I8_PRE {{.*}}:: (store unknown-size -; ISEL: PSEUDO_ZAMES_MSBT_B_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE {{.*}}:: (store unknown-size ; PRERA-LABEL: name: zames_mem_ab_i8 -; PRERA: PSEUDO_ZAMES_MLA_A_I8 {{.*}}:: (load unknown-size -; PRERA: PSEUDO_ZAMES_MLAT_A_I8 {{.*}}:: (load unknown-size -; PRERA: PSEUDO_ZAMES_MLB_B_I8 {{.*}}:: (load unknown-size -; PRERA: PSEUDO_ZAMES_MLBT_B_I8 {{.*}}:: (load unknown-size -; PRERA: PSEUDO_ZAMES_MSA_A_I8 {{.*}}:: (store unknown-size -; PRERA: PSEUDO_ZAMES_MSAT_A_I8 {{.*}}:: (store unknown-size -; PRERA: PSEUDO_ZAMES_MSB_B_I8 {{.*}}:: (store unknown-size -; PRERA: PSEUDO_ZAMES_MSBT_B_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLA_A_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLAT_A_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLB_B_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLBT_B_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSA_A_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSAT_A_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSB_B_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSBT_B_I8 {{.*}}:: (store unknown-size ; POSTRA-LABEL: name: zames_mem_ab_i8 -; POSTRA: ZAMES_MSETCFG {{[^:]*$}} -; POSTRA: ZAMES_MLA {{.*}}:: (load unknown-size -; POSTRA: ZAMES_MLAT {{.*}}:: (load unknown-size -; POSTRA: ZAMES_MLB {{.*}}:: (load unknown-size -; POSTRA: ZAMES_MLBT {{.*}}:: (load unknown-size -; POSTRA: ZAMES_MSA {{.*}}:: (store unknown-size -; POSTRA: ZAMES_MSAT {{.*}}:: (store unknown-size -; POSTRA: ZAMES_MSB {{.*}}:: (store unknown-size -; POSTRA: ZAMES_MSBT {{.*}}:: (store unknown-size - %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %src, i64 %s) - %tat = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %src, i64 %s) - %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %src, i64 %s) - %tbt = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %dst, i64 %s) - call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tat, ptr %dst, i64 %s) - call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %dst, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tbt, ptr %dst, i64 %s) +; POSTRA: XSAI_ZAMES_MSETCFG {{[^:]*$}} +; POSTRA: XSAI_ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size + %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) + %tat = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) + %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) + %tbt = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tat, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tbt, ptr %dst, i64 %s) ret void } define void @zames_mem_c_i32(ptr %src, ptr %dst, i64 %s) { ; ISEL-LABEL: name: zames_mem_c_i32 -; ISEL: PSEUDO_ZAMES_MLC_C_I32_PRE {{.*}}:: (load unknown-size -; ISEL: PSEUDO_ZAMES_MLCT_C_I32_PRE {{.*}}:: (load unknown-size -; ISEL: PSEUDO_ZAMES_MSC_C_I32_PRE {{.*}}:: (store unknown-size -; ISEL: PSEUDO_ZAMES_MSCT_C_I32_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE {{.*}}:: (store unknown-size ; PRERA-LABEL: name: zames_mem_c_i32 -; PRERA: PSEUDO_ZAMES_MLC_C_I32 {{.*}}:: (load unknown-size -; PRERA: PSEUDO_ZAMES_MLCT_C_I32 {{.*}}:: (load unknown-size -; PRERA: PSEUDO_ZAMES_MSC_C_I32 {{.*}}:: (store unknown-size -; PRERA: PSEUDO_ZAMES_MSCT_C_I32 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLC_C_I32 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLCT_C_I32 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSC_C_I32 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSCT_C_I32 {{.*}}:: (store unknown-size ; POSTRA-LABEL: name: zames_mem_c_i32 -; POSTRA: ZAMES_MLC {{.*}}:: (load unknown-size -; POSTRA: ZAMES_MLCT {{.*}}:: (load unknown-size -; POSTRA: ZAMES_MSC {{.*}}:: (store unknown-size -; POSTRA: ZAMES_MSCT {{.*}}:: (store unknown-size - %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %src, i64 %s) - %tct = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %src, i64 %s) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %dst, i64 %s) - call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tct, ptr %dst, i64 %s) +; POSTRA: XSAI_ZAMES_MLC {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLCT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSC {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSCT {{.*}}:: (store unknown-size + %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %src, i64 %s) + %tct = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tct, ptr %dst, i64 %s) ret void } define void @zames_mem_whole(ptr %src, ptr %dst) { ; ISEL-LABEL: name: zames_mem_whole -; ISEL: ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src -; ISEL: ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src -; ISEL: ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src -; ISEL: ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst -; ISEL: ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst -; ISEL: ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; ISEL: XSAI_ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: XSAI_ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: XSAI_ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: XSAI_ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; ISEL: XSAI_ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; ISEL: XSAI_ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst ; PRERA-LABEL: name: zames_mem_whole -; PRERA: ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src -; PRERA: ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src -; PRERA: ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src -; PRERA: ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst -; PRERA: ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst -; PRERA: ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA: XSAI_ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: XSAI_ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: XSAI_ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: XSAI_ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA: XSAI_ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA: XSAI_ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst ; POSTRA-LABEL: name: zames_mem_whole -; POSTRA: ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src -; POSTRA: ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src -; POSTRA: ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src -; POSTRA: ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst -; POSTRA: ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst -; POSTRA: ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst - %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) - %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) - %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) +; POSTRA: XSAI_ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: XSAI_ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: XSAI_ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: XSAI_ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA: XSAI_ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA: XSAI_ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst + %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) ret void } -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll index d83c3a07db0b28..feb191e3696659 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll @@ -3,10 +3,10 @@ ; Without +zames, the target must not legally select ZAMES matrix intrinsics. define void @zames_requires_feature(ptr %c, i64 %stride) { -; CHECK: Cannot select: intrinsic %llvm.riscv.zames.msc - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t( - target("riscv.ame.tile.int32") undef, ptr %c, i64 %stride) +; CHECK: Cannot select: intrinsic %llvm.riscv.xsai.zames.msc + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t( + target("riscv.xsai.ame.tile.int32") undef, ptr %c, i64 %stride) ret void } -declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll index 90d8bf7804f835..3730510036beb9 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll @@ -5,24 +5,24 @@ define void @zames_prera_materializes_cfg(ptr %a, ptr %b, ptr %c, i64 %stride_c) { ; CHECK-LABEL: name: zames_prera_materializes_cfg ; CHECK: %[[CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_A]] +; CHECK-NEXT: %[[TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_A]] ; CHECK: %[[CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[TILE_B:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLBT_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_B]] +; CHECK-NEXT: %[[TILE_B:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_B]] ; CHECK: %[[CFG_ZERO:[0-9]+]]:gpr = ADDI $x0, 4 -; CHECK-NEXT: %[[ZERO:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MZERO_C_I32 %[[CFG_ZERO]] +; CHECK-NEXT: %[[ZERO:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_I32 %[[CFG_ZERO]] ; CHECK: %[[CFG_ACC:[0-9]+]]:gpr = ADDI $x0, 4 ; CHECK-NEXT: %[[CFG_MAT:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[ACC:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_I32 %[[ZERO]], %[[TILE_B]], %[[TILE_A]], %[[CFG_ACC]], %[[CFG_MAT]], %[[CFG_MAT]] +; CHECK-NEXT: %[[ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %[[ZERO]], %[[TILE_B]], %[[TILE_A]], %[[CFG_ACC]], %[[CFG_MAT]], %[[CFG_MAT]] ; CHECK: %[[CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 -; CHECK-NEXT: PSEUDO_ZAMES_MSC_C_I32 %[[ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_STORE]] - %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( - target("riscv.ame.tile.int8") %tile_a, - target("riscv.ame.tile.int8") %tile_b, - target("riscv.ame.tile.int32") %zero) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_I32 %[[ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_STORE]] + %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( + target("riscv.xsai.ame.tile.int8") %tile_a, + target("riscv.xsai.ame.tile.int8") %tile_b, + target("riscv.xsai.ame.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -32,32 +32,32 @@ define void @zames_prera_materializes_mlb_cfg(ptr %a, ptr %b, ptr %c, i64 %stride_c) { ; CHECK-LABEL: name: zames_prera_materializes_mlb_cfg ; CHECK: %[[MLB_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[MLB_TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_A]] +; CHECK-NEXT: %[[MLB_TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_A]] ; CHECK: %[[MLB_CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[MLB_TILE_B:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_B]] +; CHECK-NEXT: %[[MLB_TILE_B:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_B]] ; CHECK: %[[MLB_CFG_ZERO:[0-9]+]]:gpr = ADDI $x0, 4 -; CHECK-NEXT: %[[MLB_ZERO:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MZERO_C_I32 %[[MLB_CFG_ZERO]] +; CHECK-NEXT: %[[MLB_ZERO:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_I32 %[[MLB_CFG_ZERO]] ; CHECK: %[[MLB_CFG_ACC:[0-9]+]]:gpr = ADDI $x0, 4 ; CHECK-NEXT: %[[MLB_CFG_MAT:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[MLB_ACC:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MMACC_I8_I32 %[[MLB_ZERO]], %[[MLB_TILE_B]], %[[MLB_TILE_A]], %[[MLB_CFG_ACC]], %[[MLB_CFG_MAT]], %[[MLB_CFG_MAT]] +; CHECK-NEXT: %[[MLB_ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %[[MLB_ZERO]], %[[MLB_TILE_B]], %[[MLB_TILE_A]], %[[MLB_CFG_ACC]], %[[MLB_CFG_MAT]], %[[MLB_CFG_MAT]] ; CHECK: %[[MLB_CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 -; CHECK-NEXT: PSEUDO_ZAMES_MSC_C_I32 %[[MLB_ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_STORE]] - %tile_a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t( - target("riscv.ame.tile.int8") %tile_a, - target("riscv.ame.tile.int8") %tile_b, - target("riscv.ame.tile.int32") %zero) - call void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %acc, ptr %c, i64 %stride_c) +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_I32 %[[MLB_ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_STORE]] + %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( + target("riscv.xsai.ame.tile.int8") %tile_a, + target("riscv.xsai.ame.tile.int8") %tile_b, + target("riscv.xsai.ame.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } define void @zames_prera_materializes_mlc_cfg(ptr %c, i64 %stride_c) { ; CHECK-LABEL: name: zames_prera_materializes_mlc_cfg ; CHECK: %[[MLC_CFG:[0-9]+]]:gpr = ADDI $x0, 4 -; CHECK-NEXT: %[[MLC_ACC:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MLC_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLC_CFG]] - %acc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr %c, i64 %stride_c) +; CHECK-NEXT: %[[MLC_ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLC_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLC_CFG]] + %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) ret void } @@ -65,37 +65,37 @@ define void @zames_prera_transpose_store_cfg(ptr %a, ptr %c, i64 %sa, i64 %sc) { ; CHECK-LABEL: name: zames_prera_transpose_store_cfg ; CHECK: %[[T_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[T_TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLAT_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_A]] +; CHECK-NEXT: %[[T_TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_A]] ; CHECK: %[[T_CFG_C:[0-9]+]]:gpr = ADDI $x0, 4 -; CHECK-NEXT: %[[T_ACC_C:[0-9]+]]:zamesaccreg = PSEUDO_ZAMES_MLCT_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_C]] +; CHECK-NEXT: %[[T_ACC_C:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLCT_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_C]] ; CHECK: %[[T_CFG_SA:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: PSEUDO_ZAMES_MSAT_A_I8 %[[T_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SA]] +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_I8 %[[T_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SA]] ; CHECK: %[[T_CFG_SC:[0-9]+]]:gpr = ADDI $x0, 4 -; CHECK-NEXT: PSEUDO_ZAMES_MSCT_C_I32 %[[T_ACC_C]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SC]] - %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr %a, i64 %sa) - %tc = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr %c, i64 %sc) - call void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %sa) - call void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %tc, ptr %c, i64 %sc) +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSCT_C_I32 %[[T_ACC_C]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SC]] + %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %sc) ret void } define void @zames_prera_ab_store_cfg(ptr %a, ptr %b, i64 %s) { ; CHECK-LABEL: name: zames_prera_ab_store_cfg ; CHECK: %[[AB_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[AB_TILE_A:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_A]] +; CHECK-NEXT: %[[AB_TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_A]] ; CHECK: %[[AB_CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: %[[AB_TILE_B:[0-9]+]]:zamestilereg = PSEUDO_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_B]] +; CHECK-NEXT: %[[AB_TILE_B:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_B]] ; CHECK: %[[AB_CFG_SA:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: PSEUDO_ZAMES_MSA_A_I8 %[[AB_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SA]] +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_I8 %[[AB_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SA]] ; CHECK: %[[AB_CFG_SB:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: PSEUDO_ZAMES_MSB_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SB]] +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SB]] ; CHECK: %[[AB_CFG_ST:[0-9]+]]:gpr = ADDI $x0, 2 -; CHECK-NEXT: PSEUDO_ZAMES_MSBT_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_ST]] - %ta = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr %a, i64 %s) - %tb = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr %b, i64 %s) - call void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %ta, ptr %a, i64 %s) - call void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) - call void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %tb, ptr %b, i64 %s) +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_ST]] + %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) ret void } @@ -103,38 +103,38 @@ define void @zames_prera_whole_no_cfg(ptr %src, ptr %dst) { ; CHECK-LABEL: name: zames_prera_whole_no_cfg ; CHECK-NOT: ADDI ; CHECK-NOT: PSEUDO -; CHECK: ZAMES_MLA_WHOLE -; CHECK: ZAMES_MLB_WHOLE -; CHECK: ZAMES_MLC_WHOLE -; CHECK: ZAMES_MSA_WHOLE -; CHECK: ZAMES_MSB_WHOLE -; CHECK: ZAMES_MSC_WHOLE - %a = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %src) - %b = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr %src) - %c = call target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr %src) - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32") %c, ptr %dst) +; CHECK: XSAI_ZAMES_MLA_WHOLE +; CHECK: XSAI_ZAMES_MLB_WHOLE +; CHECK: XSAI_ZAMES_MLC_WHOLE +; CHECK: XSAI_ZAMES_MSA_WHOLE +; CHECK: XSAI_ZAMES_MSB_WHOLE +; CHECK: XSAI_ZAMES_MSC_WHOLE + %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) + %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) ret void } -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlbt.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.triscv.ame.tile.int32t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mzero.triscv.ame.tile.int32t() -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mmacc.triscv.ame.tile.int32t.triscv.ame.tile.int8t.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), target("riscv.ame.tile.int8"), target("riscv.ame.tile.int32")) -declare void @llvm.riscv.zames.msc.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlat.triscv.ame.tile.int8t(ptr, i64) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlct.triscv.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.zames.msa.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msb.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msat.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msbt.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.zames.msct.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr, i64) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mlb.whole.triscv.ame.tile.int8t(ptr) -declare target("riscv.ame.tile.int32") @llvm.riscv.zames.mlc.whole.triscv.ame.tile.int32t(ptr) -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msb.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr) -declare void @llvm.riscv.zames.msc.whole.triscv.ame.tile.int32t(target("riscv.ame.tile.int32"), ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) +declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll index 7d88ccd3586de1..489fc7fcb9c4b8 100644 --- a/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll +++ b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll @@ -7,14 +7,14 @@ define void @zmasync_intrinsics(i64 %target) { ; CHECK-NEXT: macquire sync0, a0 ; CHECK-NEXT: mfence ; CHECK-NEXT: ret - %sync0 = call target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() - %sync1 = call target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease(target("riscv.ame.sync") %sync0) - call void @llvm.riscv.zmasync.macquire(target("riscv.ame.sync") %sync1, i64 %target) - call void @llvm.riscv.zmasync.mfence() + %sync0 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() + %sync1 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.ame.sync") %sync0) + call void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.ame.sync") %sync1, i64 %target) + call void @llvm.riscv.xsai.zmasync.mfence() ret void } -declare target("riscv.ame.sync") @llvm.riscv.zmasync.msyncregreset() -declare target("riscv.ame.sync") @llvm.riscv.zmasync.mrelease(target("riscv.ame.sync")) -declare void @llvm.riscv.zmasync.macquire(target("riscv.ame.sync"), i64) -declare void @llvm.riscv.zmasync.mfence() +declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() +declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.ame.sync")) +declare void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.ame.sync"), i64) +declare void @llvm.riscv.xsai.zmasync.mfence() diff --git a/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll index 551b092b36069c..8a9c476eeabb2f 100644 --- a/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll +++ b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll @@ -18,52 +18,52 @@ ; RUN: not llvm-as %t/range-mgetcfg-neg.ll -o /dev/null ; Note: an overload suffix that disagrees with the argument types (e.g. -; msa.triscv.ame.tile.int8t taking an int32 tile) is NOT a verifier error — +; msa.triscv.xsai.ame.tile.int8t taking an int32 tile) is NOT a verifier error — ; the parser re-mangles the name from the argument types. Signature ; violations must therefore probe arity/type/immarg constraints. ;--- wrong-ret-settilem.ll ; msettilem returns void. -declare i64 @llvm.riscv.zames.msettilem(i64) +declare i64 @llvm.riscv.xsai.zames.msettilem(i64) define i64 @f(i64 %m) { - %r = call i64 @llvm.riscv.zames.msettilem(i64 %m) + %r = call i64 @llvm.riscv.xsai.zames.msettilem(i64 %m) ret i64 %r } ;--- whole-load-stride.ll ; .whole loads are pointer-only; a stride operand is not allowed. -declare target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr, i64) -define target("riscv.ame.tile.int8") @f(ptr %p, i64 %s) { - %r = call target("riscv.ame.tile.int8") @llvm.riscv.zames.mla.whole.triscv.ame.tile.int8t(ptr %p, i64 %s) - ret target("riscv.ame.tile.int8") %r +define target("riscv.xsai.ame.tile.int8") @f(ptr %p, i64 %s) { + %r = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %p, i64 %s) + ret target("riscv.xsai.ame.tile.int8") %r } ;--- acquire-wrong-ret.ll ; macquire returns void. -declare target("riscv.ame.sync") @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync"), i64) +declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync"), i64) -define target("riscv.ame.sync") @f(target("riscv.ame.sync") %s, i64 %sem) { - %r = call target("riscv.ame.sync") @llvm.riscv.zmasync.macquire.triscv.ame.synct(target("riscv.ame.sync") %s, i64 %sem) - ret target("riscv.ame.sync") %r +define target("riscv.xsai.ame.sync") @f(target("riscv.xsai.ame.sync") %s, i64 %sem) { + %r = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s, i64 %sem) + ret target("riscv.xsai.ame.sync") %r } ;--- whole-store-stride.ll ; .whole stores take tile + pointer only. -declare void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -define void @f(target("riscv.ame.tile.int8") %t, ptr %p, i64 %s) { - call void @llvm.riscv.zames.msa.whole.triscv.ame.tile.int8t(target("riscv.ame.tile.int8") %t, ptr %p, i64 %s) +define void @f(target("riscv.xsai.ame.tile.int8") %t, ptr %p, i64 %s) { + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %t, ptr %p, i64 %s) ret void } ;--- nonimm-cfg.ll ; The msetcfg selector operand is an ImmArg. -declare void @llvm.riscv.zames.msetcfg(i64, i64) +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) define void @f(i64 %sel, i64 %payload) { - call void @llvm.riscv.zames.msetcfg(i64 %sel, i64 %payload) + call void @llvm.riscv.xsai.zames.msetcfg(i64 %sel, i64 %payload) ret void } @@ -71,26 +71,26 @@ define void @f(i64 %sel, i64 %payload) { ; The immediate tile setters take a constant uimm10; a dynamic value is an ; ImmArg violation. Range selection (register form for >=1024) is done by ; the Buddy lowering, not the verifier. -declare void @llvm.riscv.zames.msettilemi(i64) +declare void @llvm.riscv.xsai.zames.msettilemi(i64) define void @f(i64 %m) { - call void @llvm.riscv.zames.msettilemi(i64 %m) + call void @llvm.riscv.xsai.zames.msettilemi(i64 %m) ret void } ;--- nonimm-settileni.ll -declare void @llvm.riscv.zames.msettileni(i64) +declare void @llvm.riscv.xsai.zames.msettileni(i64) define void @f(i64 %n) { - call void @llvm.riscv.zames.msettileni(i64 %n) + call void @llvm.riscv.xsai.zames.msettileni(i64 %n) ret void } ;--- nonimm-settileki.ll -declare void @llvm.riscv.zames.msettileki(i64) +declare void @llvm.riscv.xsai.zames.msettileki(i64) define void @f(i64 %k) { - call void @llvm.riscv.zames.msettileki(i64 %k) + call void @llvm.riscv.xsai.zames.msettileki(i64 %k) ret void } @@ -98,43 +98,43 @@ define void @f(i64 %k) { ; The immediate tile setters encode a uimm10 field: the LLVM contract is the ; half-open range [0, 1024). Out-of-range constants must be rejected even ; though they are immediates. -declare void @llvm.riscv.zames.msettilemi(i64) +declare void @llvm.riscv.xsai.zames.msettilemi(i64) define void @f() { - call void @llvm.riscv.zames.msettilemi(i64 -1) + call void @llvm.riscv.xsai.zames.msettilemi(i64 -1) ret void } ;--- range-settileni-1024.ll -declare void @llvm.riscv.zames.msettileni(i64) +declare void @llvm.riscv.xsai.zames.msettileni(i64) define void @f() { - call void @llvm.riscv.zames.msettileni(i64 1024) + call void @llvm.riscv.xsai.zames.msettileni(i64 1024) ret void } ;--- range-settileki-1024.ll -declare void @llvm.riscv.zames.msettileki(i64) +declare void @llvm.riscv.xsai.zames.msettileki(i64) define void @f() { - call void @llvm.riscv.zames.msettileki(i64 1024) + call void @llvm.riscv.xsai.zames.msettileki(i64 1024) ret void } ;--- range-msetcfg-8.ll ; The config selector is a 3-bit field: the LLVM contract is [0, 8). The ; payload argument is a full GPR value and is intentionally unconstrained. -declare void @llvm.riscv.zames.msetcfg(i64, i64) +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) define void @f(i64 %payload) { - call void @llvm.riscv.zames.msetcfg(i64 8, i64 %payload) + call void @llvm.riscv.xsai.zames.msetcfg(i64 8, i64 %payload) ret void } ;--- range-mgetcfg-neg.ll -declare i64 @llvm.riscv.zames.mgetcfg(i64) +declare i64 @llvm.riscv.xsai.zames.mgetcfg(i64) define i64 @f() { - %r = call i64 @llvm.riscv.zames.mgetcfg(i64 -1) + %r = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 -1) ret i64 %r } From 343a51a809153c7b6e9d28d79e955e05a9716111 Mon Sep 17 00:00:00 2001 From: jiaxiaoyu Date: Wed, 4 Nov 2026 17:36:44 +0800 Subject: [PATCH 3/6] [RISCV] Keep native XSAI TableGen banners within 80 columns Shorten the four native matrix TableGen file banners and put long backend filenames on a separate comment line. Preserve the existing license text. This is comment-only and does not change declarations, code generation, instruction encodings or the architectural interface. --- llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td | 2 +- llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td | 3 ++- llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td | 3 ++- llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td b/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td index 4f9493e7742d23..55223e0b2b5dfe 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td @@ -1,4 +1,4 @@ -//===- IntrinsicsRISCVXSAIZamesZmasync.td - Zames/Zmasync intrinsics -*- tablegen -*-===// +//===- IntrinsicsRISCVXSAIZamesZmasync.td - XSAI matrix -*- tablegen -*-===// // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td index f738bfef46e550..c669fb4c2ac26b 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td @@ -1,4 +1,5 @@ -//===-- RISCVInstrInfoXSAIZames.td - ZAMES/ZMASYNC instructions -*- tablegen -*-===// +//===-- XSAI Zames instructions ----------------------------*- tablegen -*-===// +// RISCVInstrInfoXSAIZames.td // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td index 3c2bf200497ea8..2ac553c2d8428c 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td @@ -1,4 +1,5 @@ -//===-- RISCVInstrInfoXSAIZamesPatterns.td - ZAMES/ZMASYNC pseudos -*- tablegen -*-===// +//===-- XSAI Zames patterns --------------------------------*- tablegen -*-===// +// RISCVInstrInfoXSAIZamesPatterns.td // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td index c5cea4a329c3e0..50288840e7bee8 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td @@ -1,4 +1,5 @@ -//===-- RISCVRegisterInfoXSAIZames.td - ZAMES/ZMASYNC registers -*- tablegen -*-===// +//===-- XSAI Zames registers -------------------------------*- tablegen -*-===// +// RISCVRegisterInfoXSAIZames.td // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. From e6b128e327f0c9f6e1e2aca86c1b06a05f0fc839 Mon Sep 17 00:00:00 2001 From: jiaxiaoyu Date: Wed, 4 Nov 2026 18:18:00 +0800 Subject: [PATCH 4/6] [RISCV] Separate native XSAI matrix and sync type namespaces Replace the mixed riscv.xsai.ame target type namespace with riscv.xsai.zames.tile.* for matrix tiles and riscv.xsai.zmasync.sync for synchronization values. Align the MVT definitions, native register aliases, type conversions, overload suffixes and fixtures. Do not retain compatibility aliases for the mixed namespace. Keep type storage properties, sync memory effects, architectural features, assembly spellings and instruction encodings unchanged. No numerical change is intended. Validation: rebuilt the affected LLVM/MLIR tools; 28 targeted tests passed. Three representative integer, floating-point and sync text sections are byte-identical. All eight native storage types, invalid matrix/sync type parameters and O2/O3 sync ordering and machine verification smokes passed. --- llvm/include/llvm/CodeGen/ValueTypes.td | 16 +- llvm/lib/CodeGen/ValueTypes.cpp | 96 +++--- llvm/lib/IR/Type.cpp | 31 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 24 +- .../RISCV/RISCVRegisterInfoXSAIZames.td | 16 +- llvm/test/Assembler/riscv-zames-intrinsics.ll | 126 +++---- .../CodeGen/RISCV/zames/rv32-unsupported.ll | 12 +- .../test/CodeGen/RISCV/zames/zames-codegen.ll | 108 +++--- .../zames/zames-fp-types-negative-acc.ll | 18 +- .../RISCV/zames/zames-fp-types-negative.ll | 20 +- .../CodeGen/RISCV/zames/zames-fp-types.ll | 320 +++++++++--------- .../CodeGen/RISCV/zames/zames-isel-pseudos.ll | 104 +++--- .../RISCV/zames/zames-memory-operands.ll | 72 ++-- .../CodeGen/RISCV/zames/zames-no-feature.ll | 6 +- .../RISCV/zames/zames-prera-cfg-pseudos.ll | 104 +++--- .../CodeGen/RISCV/zames/zmasync-codegen.ll | 12 +- .../riscv-zames-intrinsics-invalid.ll | 24 +- 17 files changed, 555 insertions(+), 554 deletions(-) diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td index 76eb92a55980ed..e08dff45137e28 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.td +++ b/llvm/include/llvm/CodeGen/ValueTypes.td @@ -409,14 +409,14 @@ def amdgpuBufferStridedPointer : ValueType<192>; def aarch64mfp8 : ValueType<8>; // 8-bit value in FPR (AArch64) -def riscv_xsai_ame_tile_i8 : ValueType<0>; // RISC-V AME/ZAMES tile with i8 elements -def riscv_xsai_ame_tile_i32 : ValueType<0>; // RISC-V AME/ZAMES tile with i32 elements -def riscv_xsai_ame_sync : ValueType<0>; // RISC-V Zmasync sync register -def riscv_xsai_ame_tile_fp8e4 : ValueType<0>; // RISC-V ZAMES tile with fp8e4m3 elements -def riscv_xsai_ame_tile_fp8e5 : ValueType<0>; // RISC-V ZAMES tile with fp8e5m2 elements -def riscv_xsai_ame_tile_f16 : ValueType<0>; // RISC-V ZAMES tile with fp16 elements -def riscv_xsai_ame_tile_bf16 : ValueType<0>; // RISC-V ZAMES tile with bf16 elements -def riscv_xsai_ame_tile_f32 : ValueType<0>; // RISC-V ZAMES tile with fp32 elements +def riscv_xsai_zames_tile_i8 : ValueType<0>; // XSAI ZAMES tile with i8 elements +def riscv_xsai_zames_tile_i32 : ValueType<0>; // XSAI ZAMES tile with i32 elements +def riscv_xsai_zmasync_sync : ValueType<0>; // XSAI ZMASYNC sync register +def riscv_xsai_zames_tile_fp8e4 : ValueType<0>; // XSAI ZAMES tile with fp8e4m3 elements +def riscv_xsai_zames_tile_fp8e5 : ValueType<0>; // XSAI ZAMES tile with fp8e5m2 elements +def riscv_xsai_zames_tile_f16 : ValueType<0>; // XSAI ZAMES tile with fp16 elements +def riscv_xsai_zames_tile_bf16 : ValueType<0>; // XSAI ZAMES tile with bf16 elements +def riscv_xsai_zames_tile_f32 : ValueType<0>; // XSAI ZAMES tile with fp32 elements // CHERI capabilities. Pointer-like values that carry additional metadata // for enforcing safety guarantees on CHERI-enabled targets. diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp index 046b239ff97b8e..8fbfbfac1b4727 100644 --- a/llvm/lib/CodeGen/ValueTypes.cpp +++ b/llvm/lib/CodeGen/ValueTypes.cpp @@ -242,29 +242,29 @@ void EVT::print(raw_ostream &OS) const { case MVT::aarch64mfp8: OS << "aarch64mfp8"; return; - case MVT::riscv_xsai_ame_tile_i8: - OS << "riscv_xsai_ame_tile_i8"; + case MVT::riscv_xsai_zames_tile_i8: + OS << "riscv_xsai_zames_tile_i8"; return; - case MVT::riscv_xsai_ame_tile_i32: - OS << "riscv_xsai_ame_tile_i32"; + case MVT::riscv_xsai_zames_tile_i32: + OS << "riscv_xsai_zames_tile_i32"; return; - case MVT::riscv_xsai_ame_sync: - OS << "riscv_xsai_ame_sync"; + case MVT::riscv_xsai_zmasync_sync: + OS << "riscv_xsai_zmasync_sync"; return; - case MVT::riscv_xsai_ame_tile_fp8e4: - OS << "riscv_xsai_ame_tile_fp8e4"; + case MVT::riscv_xsai_zames_tile_fp8e4: + OS << "riscv_xsai_zames_tile_fp8e4"; return; - case MVT::riscv_xsai_ame_tile_fp8e5: - OS << "riscv_xsai_ame_tile_fp8e5"; + case MVT::riscv_xsai_zames_tile_fp8e5: + OS << "riscv_xsai_zames_tile_fp8e5"; return; - case MVT::riscv_xsai_ame_tile_f16: - OS << "riscv_xsai_ame_tile_f16"; + case MVT::riscv_xsai_zames_tile_f16: + OS << "riscv_xsai_zames_tile_f16"; return; - case MVT::riscv_xsai_ame_tile_bf16: - OS << "riscv_xsai_ame_tile_bf16"; + case MVT::riscv_xsai_zames_tile_bf16: + OS << "riscv_xsai_zames_tile_bf16"; return; - case MVT::riscv_xsai_ame_tile_f32: - OS << "riscv_xsai_ame_tile_f32"; + case MVT::riscv_xsai_zames_tile_f32: + OS << "riscv_xsai_zames_tile_f32"; return; } } @@ -291,22 +291,22 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const { return TargetExtType::get(Context, "aarch64.svcount"); case MVT::aarch64mfp8: return FixedVectorType::get(IntegerType::get(Context, 8), 1); - case MVT::riscv_xsai_ame_tile_i8: - return TargetExtType::get(Context, "riscv.xsai.ame.tile.int8"); - case MVT::riscv_xsai_ame_tile_i32: - return TargetExtType::get(Context, "riscv.xsai.ame.tile.int32"); - case MVT::riscv_xsai_ame_sync: - return TargetExtType::get(Context, "riscv.xsai.ame.sync"); - case MVT::riscv_xsai_ame_tile_fp8e4: - return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp8e4"); - case MVT::riscv_xsai_ame_tile_fp8e5: - return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp8e5"); - case MVT::riscv_xsai_ame_tile_f16: - return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp16"); - case MVT::riscv_xsai_ame_tile_bf16: - return TargetExtType::get(Context, "riscv.xsai.ame.tile.bf16"); - case MVT::riscv_xsai_ame_tile_f32: - return TargetExtType::get(Context, "riscv.xsai.ame.tile.fp32"); + case MVT::riscv_xsai_zames_tile_i8: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.int8"); + case MVT::riscv_xsai_zames_tile_i32: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.int32"); + case MVT::riscv_xsai_zmasync_sync: + return TargetExtType::get(Context, "riscv.xsai.zmasync.sync"); + case MVT::riscv_xsai_zames_tile_fp8e4: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp8e4"); + case MVT::riscv_xsai_zames_tile_fp8e5: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp8e5"); + case MVT::riscv_xsai_zames_tile_f16: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp16"); + case MVT::riscv_xsai_zames_tile_bf16: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.bf16"); + case MVT::riscv_xsai_zames_tile_f32: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp32"); case MVT::x86amx: return Type::getX86_AMXTy(Context); case MVT::i64x8: return IntegerType::get(Context, 512); case MVT::amdgpuBufferFatPointer: return IntegerType::get(Context, 160); @@ -362,22 +362,22 @@ MVT MVT::getVT(Type *Ty, bool HandleUnknown){ return MVT::getRISCVVectorTupleVT(Sz * NF, NF); } - if (TargetExtTy->getName() == "riscv.xsai.ame.tile.int8") - return MVT(MVT::riscv_xsai_ame_tile_i8); - if (TargetExtTy->getName() == "riscv.xsai.ame.tile.int32") - return MVT(MVT::riscv_xsai_ame_tile_i32); - if (TargetExtTy->getName() == "riscv.xsai.ame.sync") - return MVT(MVT::riscv_xsai_ame_sync); - if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp8e4") - return MVT(MVT::riscv_xsai_ame_tile_fp8e4); - if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp8e5") - return MVT(MVT::riscv_xsai_ame_tile_fp8e5); - if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp16") - return MVT(MVT::riscv_xsai_ame_tile_f16); - if (TargetExtTy->getName() == "riscv.xsai.ame.tile.bf16") - return MVT(MVT::riscv_xsai_ame_tile_bf16); - if (TargetExtTy->getName() == "riscv.xsai.ame.tile.fp32") - return MVT(MVT::riscv_xsai_ame_tile_f32); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.int8") + return MVT(MVT::riscv_xsai_zames_tile_i8); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.int32") + return MVT(MVT::riscv_xsai_zames_tile_i32); + if (TargetExtTy->getName() == "riscv.xsai.zmasync.sync") + return MVT(MVT::riscv_xsai_zmasync_sync); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp8e4") + return MVT(MVT::riscv_xsai_zames_tile_fp8e4); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp8e5") + return MVT(MVT::riscv_xsai_zames_tile_fp8e5); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp16") + return MVT(MVT::riscv_xsai_zames_tile_f16); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.bf16") + return MVT(MVT::riscv_xsai_zames_tile_bf16); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp32") + return MVT(MVT::riscv_xsai_zames_tile_f32); if (HandleUnknown) return MVT(MVT::Other); llvm_unreachable("Unknown target ext type!"); diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 73f495161e2b84..aaba80b1deed5b 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -981,14 +981,14 @@ Expected TargetExtType::checkParams(TargetExtType *TTy) { "target extension type riscv.vector.tuple should have one " "type parameter and one integer parameter"); - if ((TTy->Name == "riscv.xsai.ame.tile.int8" || - TTy->Name == "riscv.xsai.ame.tile.int32" || - TTy->Name == "riscv.xsai.ame.tile.fp8e4" || - TTy->Name == "riscv.xsai.ame.tile.fp8e5" || - TTy->Name == "riscv.xsai.ame.tile.fp16" || - TTy->Name == "riscv.xsai.ame.tile.bf16" || - TTy->Name == "riscv.xsai.ame.tile.fp32" || - TTy->Name == "riscv.xsai.ame.sync") && + if ((TTy->Name == "riscv.xsai.zames.tile.int8" || + TTy->Name == "riscv.xsai.zames.tile.int32" || + TTy->Name == "riscv.xsai.zames.tile.fp8e4" || + TTy->Name == "riscv.xsai.zames.tile.fp8e5" || + TTy->Name == "riscv.xsai.zames.tile.fp16" || + TTy->Name == "riscv.xsai.zames.tile.bf16" || + TTy->Name == "riscv.xsai.zames.tile.fp32" || + TTy->Name == "riscv.xsai.zmasync.sync") && (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 0)) return createStringError("target extension type " + TTy->Name + " should have no parameters"); @@ -1089,17 +1089,18 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); } - if (Name == "riscv.xsai.ame.tile.int8" || - Name == "riscv.xsai.ame.tile.int32" || - Name == "riscv.xsai.ame.tile.fp8e4" || - Name == "riscv.xsai.ame.tile.fp8e5" || - Name == "riscv.xsai.ame.tile.fp16" || - Name == "riscv.xsai.ame.tile.bf16" || Name == "riscv.xsai.ame.tile.fp32") + if (Name == "riscv.xsai.zames.tile.int8" || + Name == "riscv.xsai.zames.tile.int32" || + Name == "riscv.xsai.zames.tile.fp8e4" || + Name == "riscv.xsai.zames.tile.fp8e5" || + Name == "riscv.xsai.zames.tile.fp16" || + Name == "riscv.xsai.zames.tile.bf16" || + Name == "riscv.xsai.zames.tile.fp32") return TargetTypeInfo(FixedVectorType::get(Type::getInt8Ty(C), 16), TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); - if (Name == "riscv.xsai.ame.sync") + if (Name == "riscv.xsai.zmasync.sync") return TargetTypeInfo(Type::getInt64Ty(C), TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 231a163385a850..a0ff91e81ad648 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -197,23 +197,23 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, } if (Subtarget.hasZames()) { - addRegisterClass(MVT::riscv_xsai_ame_tile_i8, + addRegisterClass(MVT::riscv_xsai_zames_tile_i8, &RISCV::XSAI_ZAMES_TileRegRegClass); - addRegisterClass(MVT::riscv_xsai_ame_tile_i32, + addRegisterClass(MVT::riscv_xsai_zames_tile_i32, &RISCV::XSAI_ZAMES_AccRegRegClass); - addRegisterClass(MVT::riscv_xsai_ame_tile_fp8e4, + addRegisterClass(MVT::riscv_xsai_zames_tile_fp8e4, &RISCV::XSAI_ZAMES_TileRegRegClass); - addRegisterClass(MVT::riscv_xsai_ame_tile_fp8e5, + addRegisterClass(MVT::riscv_xsai_zames_tile_fp8e5, &RISCV::XSAI_ZAMES_TileRegRegClass); - addRegisterClass(MVT::riscv_xsai_ame_tile_f16, + addRegisterClass(MVT::riscv_xsai_zames_tile_f16, &RISCV::XSAI_ZAMES_TileRegRegClass); - addRegisterClass(MVT::riscv_xsai_ame_tile_bf16, + addRegisterClass(MVT::riscv_xsai_zames_tile_bf16, &RISCV::XSAI_ZAMES_TileRegRegClass); - addRegisterClass(MVT::riscv_xsai_ame_tile_f32, + addRegisterClass(MVT::riscv_xsai_zames_tile_f32, &RISCV::XSAI_ZAMES_AccRegRegClass); } if (Subtarget.hasZmasync()) - addRegisterClass(MVT::riscv_xsai_ame_sync, + addRegisterClass(MVT::riscv_xsai_zmasync_sync, &RISCV::XSAI_ZMASYNC_SyncRRegClass); static const MVT::SimpleValueType BoolVecVTs[] = { @@ -2302,12 +2302,12 @@ void RISCVTargetLowering::getTgtMemIntrinsic( if (!TargetTy) return MVT(MVT::Other); return StringSwitch(TargetTy->getName()) - .Cases({"riscv.xsai.ame.tile.int8", "riscv.xsai.ame.tile.fp8e4", - "riscv.xsai.ame.tile.fp8e5"}, + .Cases({"riscv.xsai.zames.tile.int8", "riscv.xsai.zames.tile.fp8e4", + "riscv.xsai.zames.tile.fp8e5"}, MVT::i8) - .Cases({"riscv.xsai.ame.tile.fp16", "riscv.xsai.ame.tile.bf16"}, + .Cases({"riscv.xsai.zames.tile.fp16", "riscv.xsai.zames.tile.bf16"}, MVT::i16) - .Cases({"riscv.xsai.ame.tile.int32", "riscv.xsai.ame.tile.fp32"}, + .Cases({"riscv.xsai.zames.tile.int32", "riscv.xsai.zames.tile.fp32"}, MVT::i32) .Default(MVT::Other); }; diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td index 50288840e7bee8..e56c09a33e9505 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td @@ -56,14 +56,14 @@ foreach Index = 0-15 in { } // End Namespace = "RISCV" -defvar xsai_mi8_t = riscv_xsai_ame_tile_i8; -defvar xsai_mi32_t = riscv_xsai_ame_tile_i32; -defvar xsai_msync_t = riscv_xsai_ame_sync; -defvar xsai_mf8e4_t = riscv_xsai_ame_tile_fp8e4; -defvar xsai_mf8e5_t = riscv_xsai_ame_tile_fp8e5; -defvar xsai_mf16_t = riscv_xsai_ame_tile_f16; -defvar xsai_mbf16_t = riscv_xsai_ame_tile_bf16; -defvar xsai_mf32_t = riscv_xsai_ame_tile_f32; +defvar xsai_mi8_t = riscv_xsai_zames_tile_i8; +defvar xsai_mi32_t = riscv_xsai_zames_tile_i32; +defvar xsai_msync_t = riscv_xsai_zmasync_sync; +defvar xsai_mf8e4_t = riscv_xsai_zames_tile_fp8e4; +defvar xsai_mf8e5_t = riscv_xsai_zames_tile_fp8e5; +defvar xsai_mf16_t = riscv_xsai_zames_tile_f16; +defvar xsai_mbf16_t = riscv_xsai_zames_tile_bf16; +defvar xsai_mf32_t = riscv_xsai_zames_tile_f32; // Tile Register class (tr0-tr3) // Note: Size is set to 256 as a placeholder; actual size depends on MLEN diff --git a/llvm/test/Assembler/riscv-zames-intrinsics.ll b/llvm/test/Assembler/riscv-zames-intrinsics.ll index 12385aa5099632..9fc0eeffc31245 100644 --- a/llvm/test/Assembler/riscv-zames-intrinsics.ll +++ b/llvm/test/Assembler/riscv-zames-intrinsics.ll @@ -33,98 +33,98 @@ define void @config(i64 %m, i64 %n, i64 %k, i64 %payload) { ; --- Overloaded load/store on int8 tiles -------------------------------- ; CHECK-LABEL: define void @load_store_i8( -; CHECK: %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) -; CHECK: %at = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) -; CHECK: %aw = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %pa) -; CHECK: call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %pa, i64 %s) -; CHECK: call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %at, ptr %pa, i64 %s) -; CHECK: call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %aw, ptr %pa) +; CHECK: %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) +; CHECK: %at = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) +; CHECK: %aw = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %pa) +; CHECK: call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %at, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %aw, ptr %pa) define void @load_store_i8(ptr %pa, i64 %s) { - %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) - %at = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %pa, i64 %s) - %aw = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %pa) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %pa, i64 %s) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %at, ptr %pa, i64 %s) - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %aw, ptr %pa) + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) + %at = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) + %aw = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %pa) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %pa, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %at, ptr %pa, i64 %s) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %aw, ptr %pa) ret void } ; --- B/C tile class + transposed/whole on int32 accumulator ------------- ; CHECK-LABEL: define void @load_store_bc( -; CHECK: %b = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) -; CHECK: %bt = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) -; CHECK: %bw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int32t(ptr %pb) -; CHECK: %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) -; CHECK: %ct = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) -; CHECK: %cw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %pc) -; CHECK: call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %b, ptr %pb, i64 %s) -; CHECK: call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bt, ptr %pb, i64 %s) -; CHECK: call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bw, ptr %pb) -; CHECK: call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %pc, i64 %s) -; CHECK: call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %ct, ptr %pc, i64 %s) -; CHECK: call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %cw, ptr %pc) +; CHECK: %b = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bt = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int32t(ptr %pb) +; CHECK: %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) +; CHECK: %ct = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) +; CHECK: %cw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %pc) +; CHECK: call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %b, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bt, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bw, ptr %pb) +; CHECK: call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %ct, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %cw, ptr %pc) define void @load_store_bc(ptr %pb, ptr %pc, i64 %s) { - %b = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) - %bt = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int32t(ptr %pb, i64 %s) - %bw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int32t(ptr %pb) - %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) - %ct = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %pc, i64 %s) - %cw = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %pc) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %b, ptr %pb, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bt, ptr %pb, i64 %s) - call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %bw, ptr %pb) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %pc, i64 %s) - call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %ct, ptr %pc, i64 %s) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %cw, ptr %pc) + %b = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) + %bt = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) + %bw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int32t(ptr %pb) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) + %ct = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) + %cw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %pc) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %b, ptr %pb, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bt, ptr %pb, i64 %s) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bw, ptr %pb) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %pc, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %ct, ptr %pc, i64 %s) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %cw, ptr %pc) ret void } ; --- mzero / mmacc ------------------------------------------------------ -; CHECK-LABEL: define target("riscv.xsai.ame.tile.int32") @zero_acc() { -; CHECK: %z = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() -; CHECK: ret target("riscv.xsai.ame.tile.int32") %z -define target("riscv.xsai.ame.tile.int32") @zero_acc() { - %z = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - ret target("riscv.xsai.ame.tile.int32") %z +; CHECK-LABEL: define target("riscv.xsai.zames.tile.int32") @zero_acc() { +; CHECK: %z = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +; CHECK: ret target("riscv.xsai.zames.tile.int32") %z +define target("riscv.xsai.zames.tile.int32") @zero_acc() { + %z = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + ret target("riscv.xsai.zames.tile.int32") %z } -; CHECK-LABEL: define target("riscv.xsai.ame.tile.int32") @matmul( -; CHECK: %r = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, target("riscv.xsai.ame.tile.int8") %b, target("riscv.xsai.ame.tile.int32") %c) -define target("riscv.xsai.ame.tile.int32") @matmul(target("riscv.xsai.ame.tile.int8") %a, target("riscv.xsai.ame.tile.int8") %b, target("riscv.xsai.ame.tile.int32") %c) { - %r = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, target("riscv.xsai.ame.tile.int8") %b, target("riscv.xsai.ame.tile.int32") %c) - ret target("riscv.xsai.ame.tile.int32") %r +; CHECK-LABEL: define target("riscv.xsai.zames.tile.int32") @matmul( +; CHECK: %r = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b, target("riscv.xsai.zames.tile.int32") %c) +define target("riscv.xsai.zames.tile.int32") @matmul(target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b, target("riscv.xsai.zames.tile.int32") %c) { + %r = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b, target("riscv.xsai.zames.tile.int32") %c) + ret target("riscv.xsai.zames.tile.int32") %r } ; --- Every floating tile type parses and prints ------------------------- ; CHECK-LABEL: define void @fp_types( -; CHECK: call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr %p, i64 %s) -; CHECK: call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr %p, i64 %s) -; CHECK: call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr %p, i64 %s) -; CHECK: call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr %p, i64 %s) -; CHECK: call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp32t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp32t(ptr %p, i64 %s) define void @fp_types(ptr %p, i64 %s) { - %e4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr %p, i64 %s) - %e5 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr %p, i64 %s) - %f16 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr %p, i64 %s) - %bf16 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr %p, i64 %s) - %f32 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp32t(ptr %p, i64 %s) + %e4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr %p, i64 %s) + %e5 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr %p, i64 %s) + %f16 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr %p, i64 %s) + %bf16 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr %p, i64 %s) + %f32 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp32t(ptr %p, i64 %s) ret void } ; --- ZMASYNC ------------------------------------------------------------ ; CHECK-LABEL: define void @sync_ops( -; CHECK: %s0 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.ame.synct() -; CHECK: %s1 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s0) -; CHECK: call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s1, i64 %sem) +; CHECK: %s0 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.zmasync.synct() +; CHECK: %s1 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s0) +; CHECK: call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s1, i64 %sem) ; CHECK: call void @llvm.riscv.xsai.zmasync.mfence() define void @sync_ops(i64 %sem) { - %s0 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.ame.synct() - %s1 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s0) - call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s1, i64 %sem) + %s0 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.zmasync.synct() + %s1 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s0) + call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s1, i64 %sem) call void @llvm.riscv.xsai.zmasync.mfence() ret void } diff --git a/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll index 6326c55b424dea..1984917312b34e 100644 --- a/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll +++ b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll @@ -10,17 +10,17 @@ ; CHECK: error: {{.*}}ZAMES/ZMASYNC code generation requires RV64 ; CHECK-NOT: Cannot select -define target("riscv.xsai.ame.tile.int8") @rv32_zames(ptr %p, i64 %stride) { - %t = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %p, i64 %stride) - ret target("riscv.xsai.ame.tile.int8") %t +define target("riscv.xsai.zames.tile.int8") @rv32_zames(ptr %p, i64 %stride) { + %t = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %p, i64 %stride) + ret target("riscv.xsai.zames.tile.int8") %t } define void @rv32_zmasync() { - %s = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() + %s = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() call void @llvm.riscv.xsai.zmasync.mfence() ret void } -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() declare void @llvm.riscv.xsai.zmasync.mfence() diff --git a/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll index f5f1170d9d30b0..e2bcdfe7c91621 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll @@ -103,14 +103,14 @@ define void @zames_straight_line(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %str call void @llvm.riscv.xsai.zames.msettilemi(i64 4) call void @llvm.riscv.xsai.zames.msettileni(i64 4) call void @llvm.riscv.xsai.zames.msettileki(i64 8) - %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( - target("riscv.xsai.ame.tile.int8") %tile_a, - target("riscv.xsai.ame.tile.int8") %tile_b, - target("riscv.xsai.ame.tile.int32") %zero) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -137,14 +137,14 @@ define void @zames_mlb_straight_line(ptr %a, ptr %b, ptr %c, call void @llvm.riscv.xsai.zames.msettilemi(i64 4) call void @llvm.riscv.xsai.zames.msettileni(i64 4) call void @llvm.riscv.xsai.zames.msettileki(i64 8) - %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( - target("riscv.xsai.ame.tile.int8") %tile_a, - target("riscv.xsai.ame.tile.int8") %tile_b, - target("riscv.xsai.ame.tile.int32") %zero) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -159,7 +159,7 @@ define void @zames_mlc_straight_line(ptr %c, i64 %stride_c) { call void @llvm.riscv.xsai.zames.msettilemi(i64 4) call void @llvm.riscv.xsai.zames.msettileni(i64 4) call void @llvm.riscv.xsai.zames.msettileki(i64 8) - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) ret void } @@ -174,10 +174,10 @@ define void @zames_transpose_load_store(ptr %a, ptr %c, i64 %stride_a, i64 %stri ; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} ; CHECK-NEXT: msct acc[[#C-4]], (a1), a3 ; CHECK: ret - %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) - %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %stride_a) - call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %stride_c) + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %stride_a) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %stride_c) ret void } @@ -198,13 +198,13 @@ define void @zames_store_variants(ptr %a, ptr %b, ptr %c, i64 %stride) { ; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} ; CHECK-NEXT: msct acc[[#C-4]], (a2), a3 ; CHECK: ret - %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride) - %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride) - %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %stride) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %stride) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %stride) - call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %stride) + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %stride) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %stride) ret void } @@ -226,12 +226,12 @@ define void @zames_whole(ptr %src, ptr %dst) { ; CHECK-OBJ-NEXT: msa.whole tr0, (a1) ; CHECK-OBJ-NEXT: msb.whole tr1, (a1) ; CHECK-OBJ-NEXT: msc.whole acc0, (a1) - %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) ret void } @@ -244,23 +244,23 @@ declare void @llvm.riscv.xsai.zames.msettilek(i64) declare void @llvm.riscv.xsai.zames.minit() declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) declare i64 @llvm.riscv.xsai.zames.mgetcfg(i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) -declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll index 9b7676930b9965..2042d37e40a40f 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll @@ -7,15 +7,15 @@ ; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.xsai.zames.mmacc define void @bad_mmacc_acc(ptr %src, ptr %dst) { - %a4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) - %b4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %dst) - %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) - %r = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %a4, target("riscv.xsai.ame.tile.fp8e4") %b4, target("riscv.xsai.ame.tile.int32") %c) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %r, ptr %dst) + %a4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + %b4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %dst) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + %r = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %a4, target("riscv.xsai.zames.tile.fp8e4") %b4, target("riscv.xsai.zames.tile.int32") %c) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %r, ptr %dst) ret void } -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.int32")) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll index e1be37fa3980d9..a6d2bab08daf0d 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll @@ -7,16 +7,16 @@ ; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.xsai.zames.mmacc define void @bad_mmacc(ptr %src, ptr %dst) { - %a4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) - %a5 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src) - %c = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) - %r = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e4") %a4, target("riscv.xsai.ame.tile.fp8e5") %a5, target("riscv.xsai.ame.tile.fp32") %c) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r, ptr %dst) + %a4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + %a5 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src) + %c = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e4") %a4, target("riscv.xsai.zames.tile.fp8e5") %a5, target("riscv.xsai.zames.tile.fp32") %c) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r, ptr %dst) ret void } -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr) -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp8e5"), target("riscv.xsai.ame.tile.fp32")) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp8e5"), target("riscv.xsai.zames.tile.fp32")) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll index b2d26705c9ae07..22f31c35832917 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll @@ -135,22 +135,22 @@ define void @zames_fp_ab_ls(ptr %src, ptr %dst, i64 %s) { ; OBJ: msb{{[ \t]}} ; OBJ: mlbt{{[ \t]}} ; OBJ: msbt{{[ \t]}} - %t1 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t1, ptr %dst, i64 %s) - %t2 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t2, ptr %dst, i64 %s) - %t3 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t3, ptr %dst, i64 %s) - %t4 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e4t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %t4, ptr %dst, i64 %s) - %t5 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t5, ptr %dst, i64 %s) - %t6 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t6, ptr %dst, i64 %s) - %t7 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t7, ptr %dst, i64 %s) - %t8 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e5t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %t8, ptr %dst, i64 %s) + %t1 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t8, ptr %dst, i64 %s) ret void } @@ -187,10 +187,10 @@ define void @zames_fp_c_ls(ptr %src, ptr %dst, i64 %s) { ; OBJ: msc{{[ \t]}} ; OBJ: mlct{{[ \t]}} ; OBJ: msct{{[ \t]}} - %c5 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.fp32t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %c5, ptr %dst, i64 %s) - %c6 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.fp32t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %c6, ptr %dst, i64 %s) + %c5 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %c5, ptr %dst, i64 %s) + %c6 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %c6, ptr %dst, i64 %s) ret void } @@ -311,22 +311,22 @@ define void @zames_fp16_bf16_ab_ls(ptr %src, ptr %dst, i64 %s) { ; OBJ: msb{{[ \t]}} ; OBJ: mlbt{{[ \t]}} ; OBJ: msbt{{[ \t]}} - %t1 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t1, ptr %dst, i64 %s) - %t2 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t2, ptr %dst, i64 %s) - %t3 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t3, ptr %dst, i64 %s) - %t4 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %t4, ptr %dst, i64 %s) - %t5 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t5, ptr %dst, i64 %s) - %t6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t6, ptr %dst, i64 %s) - %t7 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t7, ptr %dst, i64 %s) - %t8 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.bf16t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %t8, ptr %dst, i64 %s) + %t1 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t8, ptr %dst, i64 %s) ret void } @@ -356,16 +356,16 @@ define void @zames_fp_whole(ptr %src, ptr %dst) { ; OBJ: msb.whole{{[ \t]}} ; OBJ: mlc.whole{{[ \t]}} ; OBJ: msc.whole{{[ \t]}} - %a = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %a, ptr %dst) - %b = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src) - call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %b, ptr %dst) - %a16 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp16t(ptr %src) - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %a16, ptr %dst) - %bb16 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.bf16t(ptr %src) - call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %bb16, ptr %dst) - %c = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %c, ptr %dst) + %a = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %a, ptr %dst) + %b = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %b, ptr %dst) + %a16 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp16t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %a16, ptr %dst) + %bb16 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.bf16t(ptr %src) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %bb16, ptr %dst) + %c = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %c, ptr %dst) ret void } @@ -418,20 +418,20 @@ define void @zames_fp_mzero(ptr %dst) { ; OBJ: mzero{{[ \t]}}tr{{[0-3]}} ; OBJ: mzero{{[ \t]}}tr{{[0-3]}} ; OBJ: mzero{{[ \t]}}acc{{[0-3]}} - %z1 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int8t() - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %z1, ptr %dst) - %z2 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e4t() - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %z2, ptr %dst) - %z3 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e5t() - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %z3, ptr %dst) - %z4 = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %z4, ptr %dst) - %z5 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp16t() - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %z5, ptr %dst) - %z6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.bf16t() - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %z6, ptr %dst) - %z7 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp32t() - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %z7, ptr %dst) + %z1 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int8t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %z1, ptr %dst) + %z2 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e4t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %z2, ptr %dst) + %z3 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e5t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %z3, ptr %dst) + %z4 = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %z4, ptr %dst) + %z5 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp16t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %z5, ptr %dst) + %z6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.bf16t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %z6, ptr %dst) + %z7 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp32t() + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %z7, ptr %dst) ret void } @@ -516,106 +516,106 @@ define void @zames_fp_mmacc(ptr %src, ptr %src2, ptr %dst) { ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} ; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} - %a1 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %b1 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src2) - %c1 = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) - %r1 = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a1, target("riscv.xsai.ame.tile.int8") %b1, target("riscv.xsai.ame.tile.int32") %c1) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %r1, ptr %dst) - %a2 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %b2 = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src2) - %c2 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) - %r2 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a2, target("riscv.xsai.ame.tile.int8") %b2, target("riscv.xsai.ame.tile.fp32") %c2) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r2, ptr %dst) - %a3 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src) - %b3 = call target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e4t(ptr %src2) - %c3 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) - %r3 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4") %a3, target("riscv.xsai.ame.tile.fp8e4") %b3, target("riscv.xsai.ame.tile.fp32") %c3) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r3, ptr %dst) - %a4 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src) - %b4 = call target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e5t(ptr %src2) - %c4 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) - %r4 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e5t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5") %a4, target("riscv.xsai.ame.tile.fp8e5") %b4, target("riscv.xsai.ame.tile.fp32") %c4) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r4, ptr %dst) - %a5 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp16t(ptr %src) - %b5 = call target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp16t(ptr %src2) - %c5 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) - %r5 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp16t.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16") %a5, target("riscv.xsai.ame.tile.fp16") %b5, target("riscv.xsai.ame.tile.fp32") %c5) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r5, ptr %dst) - %a6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.bf16t(ptr %src) - %b6 = call target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.bf16t(ptr %src2) - %c6 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr %src) - %r6 = call target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.bf16t.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16") %a6, target("riscv.xsai.ame.tile.bf16") %b6, target("riscv.xsai.ame.tile.fp32") %c6) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32") %r6, ptr %dst) + %a1 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b1 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src2) + %c1 = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + %r1 = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a1, target("riscv.xsai.zames.tile.int8") %b1, target("riscv.xsai.zames.tile.int32") %c1) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %r1, ptr %dst) + %a2 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b2 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src2) + %c2 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r2 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a2, target("riscv.xsai.zames.tile.int8") %b2, target("riscv.xsai.zames.tile.fp32") %c2) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r2, ptr %dst) + %a3 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + %b3 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src2) + %c3 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r3 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %a3, target("riscv.xsai.zames.tile.fp8e4") %b3, target("riscv.xsai.zames.tile.fp32") %c3) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r3, ptr %dst) + %a4 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src) + %b4 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src2) + %c4 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r4 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e5t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %a4, target("riscv.xsai.zames.tile.fp8e5") %b4, target("riscv.xsai.zames.tile.fp32") %c4) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r4, ptr %dst) + %a5 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp16t(ptr %src) + %b5 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp16t(ptr %src2) + %c5 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r5 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp16t.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %a5, target("riscv.xsai.zames.tile.fp16") %b5, target("riscv.xsai.zames.tile.fp32") %c5) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r5, ptr %dst) + %a6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.bf16t(ptr %src) + %b6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.bf16t(ptr %src2) + %c6 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r6 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.bf16t.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %a6, target("riscv.xsai.zames.tile.bf16") %b6, target("riscv.xsai.zames.tile.fp32") %c6) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r6, ptr %dst) ret void } -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e4t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp8e5t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.fp16t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.fp16t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.fp16t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.fp16t(ptr, i64) -declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.bf16t(ptr, i64) -declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.bf16t(ptr, i64) -declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.bf16t(ptr, i64) -declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.bf16t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.fp32t(ptr, i64) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.fp32t(ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr, i64) -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e4t(ptr) -declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp16t(ptr) -declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.bf16t(ptr) -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e5t(ptr) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp8e4t(ptr) -declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.fp16t(ptr) -declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.bf16t(ptr) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.fp32t(ptr) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.fp8e5t(ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr) -declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), ptr) -declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), ptr) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.fp32t(target("riscv.xsai.ame.tile.fp32"), ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int8t() -declare target("riscv.xsai.ame.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e4t() -declare target("riscv.xsai.ame.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp8e5t() -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() -declare target("riscv.xsai.ame.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp16t() -declare target("riscv.xsai.ame.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.bf16t() -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.fp32t() -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.fp32")) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e4t.triscv.xsai.ame.tile.fp8e4t(target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp8e4"), target("riscv.xsai.ame.tile.fp32")) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp8e5t.triscv.xsai.ame.tile.fp8e5t(target("riscv.xsai.ame.tile.fp8e5"), target("riscv.xsai.ame.tile.fp8e5"), target("riscv.xsai.ame.tile.fp32")) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.fp16t.triscv.xsai.ame.tile.fp16t(target("riscv.xsai.ame.tile.fp16"), target("riscv.xsai.ame.tile.fp16"), target("riscv.xsai.ame.tile.fp32")) -declare target("riscv.xsai.ame.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.fp32t.triscv.xsai.ame.tile.bf16t.triscv.xsai.ame.tile.bf16t(target("riscv.xsai.ame.tile.bf16"), target("riscv.xsai.ame.tile.bf16"), target("riscv.xsai.ame.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.fp32t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.fp32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp16t(ptr) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.bf16t(ptr) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e5t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp16t(ptr) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.bf16t(ptr) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int8t() +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e4t() +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e5t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp16t() +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.bf16t() +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e5t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), target("riscv.xsai.zames.tile.fp8e5"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp16t.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), target("riscv.xsai.zames.tile.fp16"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.bf16t.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), target("riscv.xsai.zames.tile.bf16"), target("riscv.xsai.zames.tile.fp32")) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll index df12209354e3d3..5065ab2fa6d3de 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll @@ -10,14 +10,14 @@ define void @zames_selects_typed_pseudos(ptr %a, ptr %b, ptr %c, i64 %stride_a, ; CHECK: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE - %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( - target("riscv.xsai.ame.tile.int8") %tile_a, - target("riscv.xsai.ame.tile.int8") %tile_b, - target("riscv.xsai.ame.tile.int32") %zero) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -30,21 +30,21 @@ define void @zames_selects_mlb_pseudo(ptr %a, ptr %b, ptr %c, ; CHECK: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE - %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( - target("riscv.xsai.ame.tile.int8") %tile_a, - target("riscv.xsai.ame.tile.int8") %tile_b, - target("riscv.xsai.ame.tile.int32") %zero) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } define void @zames_selects_mlc_pseudo(ptr %c, i64 %stride_c) { ; CHECK-LABEL: name: zames_selects_mlc_pseudo ; CHECK: PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) ret void } @@ -55,10 +55,10 @@ define void @zames_selects_transpose_and_store_pseudos(ptr %a, ptr %c, ; CHECK: PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE - %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %a, i64 %sa) - %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %c, i64 %sc) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %sa) - call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %sc) + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %sc) ret void } @@ -67,11 +67,11 @@ define void @zames_selects_ab_store_pseudos(ptr %a, ptr %b, i64 %s) { ; CHECK: PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE ; CHECK: PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE - %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %s) - %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %s) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %s) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) ret void } @@ -84,32 +84,32 @@ define void @zames_selects_whole_direct(ptr %src, ptr %dst) { ; CHECK: XSAI_ZAMES_MSA_WHOLE ; CHECK: XSAI_ZAMES_MSB_WHOLE ; CHECK: XSAI_ZAMES_MSC_WHOLE - %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) ret void } -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) -declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll index e2af7d17dbc7e6..0470863467cc18 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll @@ -42,14 +42,14 @@ define void @zames_mem_ab_i8(ptr %src, ptr %dst, i64 %s) { ; POSTRA: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size ; POSTRA: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size ; POSTRA: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size - %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) - %tat = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) - %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) - %tbt = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %dst, i64 %s) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tat, ptr %dst, i64 %s) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %dst, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tbt, ptr %dst, i64 %s) + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + %tat = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + %tbt = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tat, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tbt, ptr %dst, i64 %s) ret void } @@ -69,10 +69,10 @@ define void @zames_mem_c_i32(ptr %src, ptr %dst, i64 %s) { ; POSTRA: XSAI_ZAMES_MLCT {{.*}}:: (load unknown-size ; POSTRA: XSAI_ZAMES_MSC {{.*}}:: (store unknown-size ; POSTRA: XSAI_ZAMES_MSCT {{.*}}:: (store unknown-size - %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %src, i64 %s) - %tct = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %src, i64 %s) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %dst, i64 %s) - call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tct, ptr %dst, i64 %s) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %src, i64 %s) + %tct = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tct, ptr %dst, i64 %s) ret void } @@ -98,30 +98,30 @@ define void @zames_mem_whole(ptr %src, ptr %dst) { ; POSTRA: XSAI_ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst ; POSTRA: XSAI_ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst ; POSTRA: XSAI_ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst - %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) ret void } -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll index feb191e3696659..d4c9bca5e23a68 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll @@ -4,9 +4,9 @@ define void @zames_requires_feature(ptr %c, i64 %stride) { ; CHECK: Cannot select: intrinsic %llvm.riscv.xsai.zames.msc - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t( - target("riscv.xsai.ame.tile.int32") undef, ptr %c, i64 %stride) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t( + target("riscv.xsai.zames.tile.int32") undef, ptr %c, i64 %stride) ret void } -declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll index 3730510036beb9..65ec31f658e6a1 100644 --- a/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll +++ b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll @@ -15,14 +15,14 @@ define void @zames_prera_materializes_cfg(ptr %a, ptr %b, ptr %c, ; CHECK-NEXT: %[[ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %[[ZERO]], %[[TILE_B]], %[[TILE_A]], %[[CFG_ACC]], %[[CFG_MAT]], %[[CFG_MAT]] ; CHECK: %[[CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 ; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_I32 %[[ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_STORE]] - %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( - target("riscv.xsai.ame.tile.int8") %tile_a, - target("riscv.xsai.ame.tile.int8") %tile_b, - target("riscv.xsai.ame.tile.int32") %zero) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -42,14 +42,14 @@ define void @zames_prera_materializes_mlb_cfg(ptr %a, ptr %b, ptr %c, ; CHECK-NEXT: %[[MLB_ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %[[MLB_ZERO]], %[[MLB_TILE_B]], %[[MLB_TILE_A]], %[[MLB_CFG_ACC]], %[[MLB_CFG_MAT]], %[[MLB_CFG_MAT]] ; CHECK: %[[MLB_CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 ; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_I32 %[[MLB_ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_STORE]] - %tile_a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %stride_a) - %tile_b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %stride_b) - %zero = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t( - target("riscv.xsai.ame.tile.int8") %tile_a, - target("riscv.xsai.ame.tile.int8") %tile_b, - target("riscv.xsai.ame.tile.int32") %zero) - call void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %acc, ptr %c, i64 %stride_c) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) ret void } @@ -57,7 +57,7 @@ define void @zames_prera_materializes_mlc_cfg(ptr %c, i64 %stride_c) { ; CHECK-LABEL: name: zames_prera_materializes_mlc_cfg ; CHECK: %[[MLC_CFG:[0-9]+]]:gpr = ADDI $x0, 4 ; CHECK-NEXT: %[[MLC_ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLC_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLC_CFG]] - %acc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr %c, i64 %stride_c) + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) ret void } @@ -72,10 +72,10 @@ define void @zames_prera_transpose_store_cfg(ptr %a, ptr %c, ; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_I8 %[[T_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SA]] ; CHECK: %[[T_CFG_SC:[0-9]+]]:gpr = ADDI $x0, 4 ; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSCT_C_I32 %[[T_ACC_C]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SC]] - %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr %a, i64 %sa) - %tc = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr %c, i64 %sc) - call void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %sa) - call void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %tc, ptr %c, i64 %sc) + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %sc) ret void } @@ -91,11 +91,11 @@ define void @zames_prera_ab_store_cfg(ptr %a, ptr %b, i64 %s) { ; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SB]] ; CHECK: %[[AB_CFG_ST:[0-9]+]]:gpr = ADDI $x0, 2 ; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_ST]] - %ta = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr %a, i64 %s) - %tb = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr %b, i64 %s) - call void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %ta, ptr %a, i64 %s) - call void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) - call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %tb, ptr %b, i64 %s) + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) ret void } @@ -109,32 +109,32 @@ define void @zames_prera_whole_no_cfg(ptr %src, ptr %dst) { ; CHECK: XSAI_ZAMES_MSA_WHOLE ; CHECK: XSAI_ZAMES_MSB_WHOLE ; CHECK: XSAI_ZAMES_MSC_WHOLE - %a = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %b = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr %src) - %c = call target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr %src) - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %a, ptr %dst) - call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %b, ptr %dst) - call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32") %c, ptr %dst) + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) ret void } -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.ame.tile.int32t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.ame.tile.int32t() -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.ame.tile.int32t.triscv.xsai.ame.tile.int8t.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int8"), target("riscv.xsai.ame.tile.int32")) -declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.ame.tile.int8t(ptr, i64) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.ame.tile.int32t(ptr, i64) -declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) -declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr, i64) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.ame.tile.int8t(ptr) -declare target("riscv.xsai.ame.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.ame.tile.int32t(ptr) -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr) -declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.ame.tile.int32t(target("riscv.xsai.ame.tile.int32"), ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll index 489fc7fcb9c4b8..e013d630295a7d 100644 --- a/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll +++ b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll @@ -7,14 +7,14 @@ define void @zmasync_intrinsics(i64 %target) { ; CHECK-NEXT: macquire sync0, a0 ; CHECK-NEXT: mfence ; CHECK-NEXT: ret - %sync0 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() - %sync1 = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.ame.sync") %sync0) - call void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.ame.sync") %sync1, i64 %target) + %sync0 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() + %sync1 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.zmasync.sync") %sync0) + call void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.zmasync.sync") %sync1, i64 %target) call void @llvm.riscv.xsai.zmasync.mfence() ret void } -declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.msyncregreset() -declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.ame.sync")) -declare void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.ame.sync"), i64) +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.zmasync.sync")) +declare void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.zmasync.sync"), i64) declare void @llvm.riscv.xsai.zmasync.mfence() diff --git a/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll index 8a9c476eeabb2f..1724b25a13f020 100644 --- a/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll +++ b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll @@ -18,7 +18,7 @@ ; RUN: not llvm-as %t/range-mgetcfg-neg.ll -o /dev/null ; Note: an overload suffix that disagrees with the argument types (e.g. -; msa.triscv.xsai.ame.tile.int8t taking an int32 tile) is NOT a verifier error — +; msa.triscv.xsai.zames.tile.int8t taking an int32 tile) is NOT a verifier error — ; the parser re-mangles the name from the argument types. Signature ; violations must therefore probe arity/type/immarg constraints. @@ -33,28 +33,28 @@ define i64 @f(i64 %m) { ;--- whole-load-stride.ll ; .whole loads are pointer-only; a stride operand is not allowed. -declare target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr, i64) -define target("riscv.xsai.ame.tile.int8") @f(ptr %p, i64 %s) { - %r = call target("riscv.xsai.ame.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.ame.tile.int8t(ptr %p, i64 %s) - ret target("riscv.xsai.ame.tile.int8") %r +define target("riscv.xsai.zames.tile.int8") @f(ptr %p, i64 %s) { + %r = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %p, i64 %s) + ret target("riscv.xsai.zames.tile.int8") %r } ;--- acquire-wrong-ret.ll ; macquire returns void. -declare target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync"), i64) +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync"), i64) -define target("riscv.xsai.ame.sync") @f(target("riscv.xsai.ame.sync") %s, i64 %sem) { - %r = call target("riscv.xsai.ame.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.ame.synct(target("riscv.xsai.ame.sync") %s, i64 %sem) - ret target("riscv.xsai.ame.sync") %r +define target("riscv.xsai.zmasync.sync") @f(target("riscv.xsai.zmasync.sync") %s, i64 %sem) { + %r = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s, i64 %sem) + ret target("riscv.xsai.zmasync.sync") %r } ;--- whole-store-stride.ll ; .whole stores take tile + pointer only. -declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) -define void @f(target("riscv.xsai.ame.tile.int8") %t, ptr %p, i64 %s) { - call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.ame.tile.int8t(target("riscv.xsai.ame.tile.int8") %t, ptr %p, i64 %s) +define void @f(target("riscv.xsai.zames.tile.int8") %t, ptr %p, i64 %s) { + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %t, ptr %p, i64 %s) ret void } From c10ab12230f5dbfeffcbfc68ba850353cfd51e72 Mon Sep 17 00:00:00 2001 From: jiaxiaoyu Date: Thu, 5 Nov 2026 15:15:20 +0800 Subject: [PATCH 5/6] [RISCV] Remove local storage and zero-init properties from XSAI types Native XSAI tile and synchronization values are register resources. Drop CanBeLocal and HasZeroInit, which were reserved for possible frontend support that is not currently planned. Keep explicit mzero and msyncregreset initialization and the existing native SSA interface. Add assembler regressions rejecting alloca, byval and zero constants. The remaining sized-layout limitation is unchanged: ordinary load/store/GEP can still pass generic verification and are not supported native uses. Validation: the property-only state passed 30 targeted LLVM tests, 27 native ZAMES tests and all 24 storage/zero rejection cases. Representative integer, floating-point and synchronization machine code is byte-identical; supported native computations have no numerical change. --- llvm/lib/IR/Type.cpp | 9 ++-- .../Assembler/riscv-xsai-type-properties.ll | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 llvm/test/Assembler/riscv-xsai-type-properties.ll diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index aaba80b1deed5b..f7eb33ba5c33cb 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -1089,6 +1089,8 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); } + // Native XSAI values represent register resources. The sized layouts are + // placeholders, not matrix memory representations or a spilling ABI. if (Name == "riscv.xsai.zames.tile.int8" || Name == "riscv.xsai.zames.tile.int32" || Name == "riscv.xsai.zames.tile.fp8e4" || @@ -1096,13 +1098,10 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { Name == "riscv.xsai.zames.tile.fp16" || Name == "riscv.xsai.zames.tile.bf16" || Name == "riscv.xsai.zames.tile.fp32") - return TargetTypeInfo(FixedVectorType::get(Type::getInt8Ty(C), 16), - TargetExtType::CanBeLocal, - TargetExtType::HasZeroInit); + return TargetTypeInfo(FixedVectorType::get(Type::getInt8Ty(C), 16)); if (Name == "riscv.xsai.zmasync.sync") - return TargetTypeInfo(Type::getInt64Ty(C), TargetExtType::CanBeLocal, - TargetExtType::HasZeroInit); + return TargetTypeInfo(Type::getInt64Ty(C)); // DirectX resources if (Name == "dx.Padding") diff --git a/llvm/test/Assembler/riscv-xsai-type-properties.ll b/llvm/test/Assembler/riscv-xsai-type-properties.ll new file mode 100644 index 00000000000000..2d769ed9e888b9 --- /dev/null +++ b/llvm/test/Assembler/riscv-xsai-type-properties.ll @@ -0,0 +1,46 @@ +; RUN: split-file %s %t +; RUN: not llvm-as %t/tile-alloca.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-ALLOCA +; RUN: not llvm-as %t/sync-alloca.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-ALLOCA +; RUN: not llvm-as %t/tile-byval.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-BYVAL +; RUN: not llvm-as %t/sync-byval.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-BYVAL +; RUN: not llvm-as %t/tile-zero.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-ZERO +; RUN: not llvm-as %t/sync-zero.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-ZERO +; +; Sized placeholder layouts do not grant stack allocation or zero constants +; to the native matrix and synchronization register types. + +;--- tile-alloca.ll +; TILE-ALLOCA: Alloca has illegal target extension type +; TILE-ALLOCA-NEXT: %value = alloca target("riscv.xsai.zames.tile.int8") +define void @tile_alloca() { + %value = alloca target("riscv.xsai.zames.tile.int8") + ret void +} + +;--- sync-alloca.ll +; SYNC-ALLOCA: Alloca has illegal target extension type +; SYNC-ALLOCA-NEXT: %value = alloca target("riscv.xsai.zmasync.sync") +define void @sync_alloca() { + %value = alloca target("riscv.xsai.zmasync.sync") + ret void +} + +;--- tile-byval.ll +; TILE-BYVAL: 'byval' argument has illegal target extension type +declare void @tile_byval(ptr byval(target("riscv.xsai.zames.tile.int32"))) + +;--- sync-byval.ll +; SYNC-BYVAL: 'byval' argument has illegal target extension type +declare void @sync_byval(ptr byval(target("riscv.xsai.zmasync.sync"))) + +;--- tile-zero.ll +; TILE-ZERO: error: invalid type for null constant +define target("riscv.xsai.zames.tile.fp32") @tile_zero() { + ret target("riscv.xsai.zames.tile.fp32") zeroinitializer +} + +;--- sync-zero.ll +; SYNC-ZERO: error: invalid type for null constant +define target("riscv.xsai.zmasync.sync") @sync_zero() { + ret target("riscv.xsai.zmasync.sync") zeroinitializer +} From 5469e4b3dc4df774e6625b23264bad034920d2c2 Mon Sep 17 00:00:00 2001 From: jiaxiaoyu Date: Thu, 5 Nov 2026 15:15:55 +0800 Subject: [PATCH 6/6] [RISCV] Make native XSAI register types unsized Replace the sized tile and synchronization layout placeholders with void. The target extension identities remain intact, but ordinary LLVM memory objects and address arithmetic no longer acquire fictional sizes or strides. Extend assembler coverage to reject ordinary loads, stores and GEP, including constant-expression GEP, while retaining legal SSA phi/select carriers. Native select instruction selection and cross-function ABI support are unchanged. Validation: the final state passed 30 targeted LLVM tests, 27 native ZAMES tests, 64 negative storage/address cases and O2/O3 SSA optimization smoke. Representative integer, floating-point and synchronization machine code is byte-identical to the sized layout; supported native computations have no numerical change. Native tile select instruction selection remains an independently confirmed preexisting limitation. --- llvm/lib/IR/Type.cpp | 7 +- .../Assembler/riscv-xsai-type-properties.ll | 102 ++++++++++++++++-- 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index f7eb33ba5c33cb..e3bfe9ff5d5084 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -1089,8 +1089,7 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); } - // Native XSAI values represent register resources. The sized layouts are - // placeholders, not matrix memory representations or a spilling ABI. + // Native XSAI values are register-only SSA resources with no memory layout. if (Name == "riscv.xsai.zames.tile.int8" || Name == "riscv.xsai.zames.tile.int32" || Name == "riscv.xsai.zames.tile.fp8e4" || @@ -1098,10 +1097,10 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { Name == "riscv.xsai.zames.tile.fp16" || Name == "riscv.xsai.zames.tile.bf16" || Name == "riscv.xsai.zames.tile.fp32") - return TargetTypeInfo(FixedVectorType::get(Type::getInt8Ty(C), 16)); + return TargetTypeInfo(Type::getVoidTy(C)); if (Name == "riscv.xsai.zmasync.sync") - return TargetTypeInfo(Type::getInt64Ty(C)); + return TargetTypeInfo(Type::getVoidTy(C)); // DirectX resources if (Name == "dx.Padding") diff --git a/llvm/test/Assembler/riscv-xsai-type-properties.ll b/llvm/test/Assembler/riscv-xsai-type-properties.ll index 2d769ed9e888b9..5d2760b045c51b 100644 --- a/llvm/test/Assembler/riscv-xsai-type-properties.ll +++ b/llvm/test/Assembler/riscv-xsai-type-properties.ll @@ -5,32 +5,39 @@ ; RUN: not llvm-as %t/sync-byval.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-BYVAL ; RUN: not llvm-as %t/tile-zero.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-ZERO ; RUN: not llvm-as %t/sync-zero.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-ZERO +; RUN: not llvm-as %t/tile-load.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-LOAD +; RUN: not llvm-as %t/sync-load.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-LOAD +; RUN: not llvm-as %t/tile-store.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-STORE +; RUN: not llvm-as %t/sync-store.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-STORE +; RUN: not llvm-as %t/tile-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-GEP +; RUN: not llvm-as %t/sync-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-GEP +; RUN: not llvm-as %t/tile-constant-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-CONSTANT-GEP +; RUN: not llvm-as %t/sync-constant-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-CONSTANT-GEP +; RUN: opt -passes=verify -disable-output %t/ssa.ll ; -; Sized placeholder layouts do not grant stack allocation or zero constants -; to the native matrix and synchronization register types. +; Native matrix and synchronization values are unsized register resources. +; Ordinary memory objects and zero constants are invalid; SSA carriers remain valid. ;--- tile-alloca.ll -; TILE-ALLOCA: Alloca has illegal target extension type -; TILE-ALLOCA-NEXT: %value = alloca target("riscv.xsai.zames.tile.int8") +; TILE-ALLOCA: Cannot allocate unsized type define void @tile_alloca() { %value = alloca target("riscv.xsai.zames.tile.int8") ret void } ;--- sync-alloca.ll -; SYNC-ALLOCA: Alloca has illegal target extension type -; SYNC-ALLOCA-NEXT: %value = alloca target("riscv.xsai.zmasync.sync") +; SYNC-ALLOCA: Cannot allocate unsized type define void @sync_alloca() { %value = alloca target("riscv.xsai.zmasync.sync") ret void } ;--- tile-byval.ll -; TILE-BYVAL: 'byval' argument has illegal target extension type +; TILE-BYVAL: Attribute 'byval' does not support unsized types! declare void @tile_byval(ptr byval(target("riscv.xsai.zames.tile.int32"))) ;--- sync-byval.ll -; SYNC-BYVAL: 'byval' argument has illegal target extension type +; SYNC-BYVAL: Attribute 'byval' does not support unsized types! declare void @sync_byval(ptr byval(target("riscv.xsai.zmasync.sync"))) ;--- tile-zero.ll @@ -44,3 +51,82 @@ define target("riscv.xsai.zames.tile.fp32") @tile_zero() { define target("riscv.xsai.zmasync.sync") @sync_zero() { ret target("riscv.xsai.zmasync.sync") zeroinitializer } + +;--- tile-load.ll +; TILE-LOAD: loading unsized types is not allowed +define target("riscv.xsai.zames.tile.int8") @tile_load(ptr %p) { + %value = load target("riscv.xsai.zames.tile.int8"), ptr %p, align 1 + ret target("riscv.xsai.zames.tile.int8") %value +} + +;--- sync-load.ll +; SYNC-LOAD: loading unsized types is not allowed +define target("riscv.xsai.zmasync.sync") @sync_load(ptr %p) { + %value = load target("riscv.xsai.zmasync.sync"), ptr %p, align 1 + ret target("riscv.xsai.zmasync.sync") %value +} + +;--- tile-store.ll +; TILE-STORE: storing unsized types is not allowed +define void @tile_store(ptr %p, target("riscv.xsai.zames.tile.fp16") %value) { + store target("riscv.xsai.zames.tile.fp16") %value, ptr %p, align 1 + ret void +} + +;--- sync-store.ll +; SYNC-STORE: storing unsized types is not allowed +define void @sync_store(ptr %p, target("riscv.xsai.zmasync.sync") %value) { + store target("riscv.xsai.zmasync.sync") %value, ptr %p, align 1 + ret void +} + +;--- tile-gep.ll +; TILE-GEP: base element of getelementptr must be sized +define ptr @tile_gep(ptr %p) { + %next = getelementptr target("riscv.xsai.zames.tile.int32"), ptr %p, i64 1 + ret ptr %next +} + +;--- sync-gep.ll +; SYNC-GEP: base element of getelementptr must be sized +define ptr @sync_gep(ptr %p) { + %next = getelementptr target("riscv.xsai.zmasync.sync"), ptr %p, i64 1 + ret ptr %next +} + +;--- tile-constant-gep.ll +; TILE-CONSTANT-GEP: base element of getelementptr must be sized +define ptr @tile_constant_gep() { + ret ptr getelementptr (target("riscv.xsai.zames.tile.bf16"), ptr null, i64 1) +} + +;--- sync-constant-gep.ll +; SYNC-CONSTANT-GEP: base element of getelementptr must be sized +define ptr @sync_constant_gep() { + ret ptr getelementptr (target("riscv.xsai.zmasync.sync"), ptr null, i64 1) +} + +;--- ssa.ll +define target("riscv.xsai.zames.tile.int8") @tile_ssa(i1 %condition, target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b) { + br i1 %condition, label %left, label %right +left: + br label %merge +right: + br label %merge +merge: + %value = phi target("riscv.xsai.zames.tile.int8") [ %a, %left ], [ %b, %right ] + %result = select i1 %condition, target("riscv.xsai.zames.tile.int8") %value, target("riscv.xsai.zames.tile.int8") %b + ret target("riscv.xsai.zames.tile.int8") %result +} + +define target("riscv.xsai.zmasync.sync") @sync_ssa(i1 %condition, target("riscv.xsai.zmasync.sync") %a, target("riscv.xsai.zmasync.sync") %b) { + br i1 %condition, label %left, label %right +left: + br label %merge +right: + br label %merge +merge: + %value = phi target("riscv.xsai.zmasync.sync") [ %a, %left ], [ %b, %right ] + %result = select i1 %condition, target("riscv.xsai.zmasync.sync") %value, target("riscv.xsai.zmasync.sync") %b + ret target("riscv.xsai.zmasync.sync") %result +}