diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c index 30c45aa6e4bdb3..7b526f8055d40a 100644 --- a/clang/test/Driver/print-supported-extensions-riscv.c +++ b/clang/test/Driver/print-supported-extensions-riscv.c @@ -35,6 +35,7 @@ // CHECK-NEXT: zihpm 2.0 'Zihpm' (Hardware Performance Counters) // CHECK-NEXT: zilsd 1.0 'Zilsd' (Load/Store Pair Instructions) // CHECK-NEXT: zimop 1.0 'Zimop' (May-Be-Operations) +// CHECK: zmasync{{ +}}1.0{{( +|$)}} // CHECK-NEXT: zmmul 1.0 'Zmmul' (Integer Multiplication) // CHECK-NEXT: za128rs 1.0 'Za128rs' (Reservation Set Size of at Most 128 Bytes) // CHECK-NEXT: za64rs 1.0 'Za64rs' (Reservation Set Size of at Most 64 Bytes) @@ -44,6 +45,7 @@ // CHECK-NEXT: zalasr 1.0 'Zalasr' (Load-Acquire and Store-Release Instructions) // CHECK-NEXT: zalrsc 1.0 'Zalrsc' (Load-Reserved/Store-Conditional) // CHECK-NEXT: zama16b 1.0 'Zama16b' (Atomic 16-byte misaligned loads, stores and AMOs) +// CHECK: zames{{ +}}1.0{{( +|$)}} // CHECK-NEXT: zawrs 1.0 'Zawrs' (Wait on Reservation Set) // CHECK-NEXT: zfa 1.0 'Zfa' (Additional Floating-Point) // CHECK-NEXT: zfbfmin 1.0 'Zfbfmin' (Scalar BF16 Converts) diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td index b5ac120f92509e..e08dff45137e28 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.td +++ b/llvm/include/llvm/CodeGen/ValueTypes.td @@ -409,6 +409,15 @@ def amdgpuBufferStridedPointer : ValueType<192>; def aarch64mfp8 : ValueType<8>; // 8-bit value in FPR (AArch64) +def riscv_xsai_zames_tile_i8 : ValueType<0>; // XSAI ZAMES tile with i8 elements +def riscv_xsai_zames_tile_i32 : ValueType<0>; // XSAI ZAMES tile with i32 elements +def riscv_xsai_zmasync_sync : ValueType<0>; // XSAI ZMASYNC sync register +def riscv_xsai_zames_tile_fp8e4 : ValueType<0>; // XSAI ZAMES tile with fp8e4m3 elements +def riscv_xsai_zames_tile_fp8e5 : ValueType<0>; // XSAI ZAMES tile with fp8e5m2 elements +def riscv_xsai_zames_tile_f16 : ValueType<0>; // XSAI ZAMES tile with fp16 elements +def riscv_xsai_zames_tile_bf16 : ValueType<0>; // XSAI ZAMES tile with bf16 elements +def riscv_xsai_zames_tile_f32 : ValueType<0>; // XSAI ZAMES tile with fp32 elements + // CHERI capabilities. Pointer-like values that carry additional metadata // for enforcing safety guarantees on CHERI-enabled targets. def c64 : VTCheriCapability<64>; // 64-bit CHERI capability value diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 840b0568f942e6..89425d0435ac6e 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2498,3 +2498,7 @@ include "llvm/IR/IntrinsicsRISCVExt.td" include "llvm/IR/IntrinsicsRISCVBOSCExt.td" include "llvm/IR/IntrinsicsRISCVXTExt.td" include "llvm/IR/IntrinsicsRISCVBuckyballExt.td" + +// XSAI ZAMES/ZMASYNC extensions +//===----------------------------------------------------------------------===// +include "llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td" diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td b/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td new file mode 100644 index 00000000000000..55223e0b2b5dfe --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td @@ -0,0 +1,107 @@ +//===- IntrinsicsRISCVXSAIZamesZmasync.td - XSAI matrix -*- tablegen -*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in { + class XSAI_ZAMES_SetCfgIntrinsic + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 8>]>; + class XSAI_ZAMES_GetCfgIntrinsic + : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 8>]>; + class XSAI_ZAMES_SetTileIntrinsic + : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>, + Range, 0, 1024>]>; + class XSAI_ZAMES_SetTileRegIntrinsic + : Intrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZAMES_InitIntrinsic + : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; + + class XSAI_ZAMES_MZeroIntrinsic + : Intrinsic<[llvm_any_ty], [], [IntrNoMem]>; + class XSAI_ZAMES_LoadIntrinsic + : Intrinsic<[llvm_any_ty], [llvm_ptr_ty, llvm_i64_ty], + [NoCapture>, IntrReadMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_StoreIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty, llvm_i64_ty], + [NoCapture>, IntrWriteMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_WholeLoadIntrinsic + : Intrinsic<[llvm_any_ty], [llvm_ptr_ty], + [NoCapture>, IntrReadMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_WholeStoreIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty], + [NoCapture>, IntrWriteMem], "", + [SDNPMemOperand]>; + class XSAI_ZAMES_MMAccIntrinsic + : Intrinsic<[llvm_any_ty], + [llvm_any_ty, llvm_any_ty, LLVMMatchType<0>], + [IntrNoMem]>; + + def int_riscv_xsai_zames_msetcfg : XSAI_ZAMES_SetCfgIntrinsic; + def int_riscv_xsai_zames_mgetcfg : XSAI_ZAMES_GetCfgIntrinsic; + def int_riscv_xsai_zames_msettilemi : XSAI_ZAMES_SetTileIntrinsic; + def int_riscv_xsai_zames_msettileni : XSAI_ZAMES_SetTileIntrinsic; + def int_riscv_xsai_zames_msettileki : XSAI_ZAMES_SetTileIntrinsic; + def int_riscv_xsai_zames_msettilem : XSAI_ZAMES_SetTileRegIntrinsic; + def int_riscv_xsai_zames_msettilen : XSAI_ZAMES_SetTileRegIntrinsic; + def int_riscv_xsai_zames_msettilek : XSAI_ZAMES_SetTileRegIntrinsic; + + def int_riscv_xsai_zames_minit : XSAI_ZAMES_InitIntrinsic; + + def int_riscv_xsai_zames_mzero : XSAI_ZAMES_MZeroIntrinsic; + def int_riscv_xsai_zames_mmacc : XSAI_ZAMES_MMAccIntrinsic; + + def int_riscv_xsai_zames_mla : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlb : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlc : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlat : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlbt : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mlct : XSAI_ZAMES_LoadIntrinsic; + def int_riscv_xsai_zames_mla_whole : XSAI_ZAMES_WholeLoadIntrinsic; + def int_riscv_xsai_zames_mlb_whole : XSAI_ZAMES_WholeLoadIntrinsic; + def int_riscv_xsai_zames_mlc_whole : XSAI_ZAMES_WholeLoadIntrinsic; + + def int_riscv_xsai_zames_msa : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msb : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msc : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msat : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msbt : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msct : XSAI_ZAMES_StoreIntrinsic; + def int_riscv_xsai_zames_msa_whole : XSAI_ZAMES_WholeStoreIntrinsic; + def int_riscv_xsai_zames_msb_whole : XSAI_ZAMES_WholeStoreIntrinsic; + def int_riscv_xsai_zames_msc_whole : XSAI_ZAMES_WholeStoreIntrinsic; + + class XSAI_ZMASYNC_SyncResetIntrinsic + : Intrinsic<[llvm_any_ty], [], [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZMASYNC_ReleaseIntrinsic + : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], + [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZMASYNC_AcquireIntrinsic + : Intrinsic<[], [llvm_any_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + class XSAI_ZMASYNC_FenceIntrinsic + : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; + + def int_riscv_xsai_zmasync_msyncregreset : XSAI_ZMASYNC_SyncResetIntrinsic; + def int_riscv_xsai_zmasync_mrelease : XSAI_ZMASYNC_ReleaseIntrinsic; + def int_riscv_xsai_zmasync_macquire : XSAI_ZMASYNC_AcquireIntrinsic; + def int_riscv_xsai_zmasync_mfence : XSAI_ZMASYNC_FenceIntrinsic; +} diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp index 3bba70cd670016..8fbfbfac1b4727 100644 --- a/llvm/lib/CodeGen/ValueTypes.cpp +++ b/llvm/lib/CodeGen/ValueTypes.cpp @@ -158,7 +158,7 @@ TypeSize EVT::getExtendedSizeInBits() const { llvm_unreachable("Unrecognized extended type!"); } -/// getEVTString - This function returns value type as a string, e.g. "i32". +/// Print the value type, e.g. "i32". void EVT::print(raw_ostream &OS) const { switch (V.SimpleTy) { default: @@ -242,6 +242,30 @@ void EVT::print(raw_ostream &OS) const { case MVT::aarch64mfp8: OS << "aarch64mfp8"; return; + case MVT::riscv_xsai_zames_tile_i8: + OS << "riscv_xsai_zames_tile_i8"; + return; + case MVT::riscv_xsai_zames_tile_i32: + OS << "riscv_xsai_zames_tile_i32"; + return; + case MVT::riscv_xsai_zmasync_sync: + OS << "riscv_xsai_zmasync_sync"; + return; + case MVT::riscv_xsai_zames_tile_fp8e4: + OS << "riscv_xsai_zames_tile_fp8e4"; + return; + case MVT::riscv_xsai_zames_tile_fp8e5: + OS << "riscv_xsai_zames_tile_fp8e5"; + return; + case MVT::riscv_xsai_zames_tile_f16: + OS << "riscv_xsai_zames_tile_f16"; + return; + case MVT::riscv_xsai_zames_tile_bf16: + OS << "riscv_xsai_zames_tile_bf16"; + return; + case MVT::riscv_xsai_zames_tile_f32: + OS << "riscv_xsai_zames_tile_f32"; + return; } } @@ -267,6 +291,22 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const { return TargetExtType::get(Context, "aarch64.svcount"); case MVT::aarch64mfp8: return FixedVectorType::get(IntegerType::get(Context, 8), 1); + case MVT::riscv_xsai_zames_tile_i8: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.int8"); + case MVT::riscv_xsai_zames_tile_i32: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.int32"); + case MVT::riscv_xsai_zmasync_sync: + return TargetExtType::get(Context, "riscv.xsai.zmasync.sync"); + case MVT::riscv_xsai_zames_tile_fp8e4: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp8e4"); + case MVT::riscv_xsai_zames_tile_fp8e5: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp8e5"); + case MVT::riscv_xsai_zames_tile_f16: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp16"); + case MVT::riscv_xsai_zames_tile_bf16: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.bf16"); + case MVT::riscv_xsai_zames_tile_f32: + return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp32"); case MVT::x86amx: return Type::getX86_AMXTy(Context); case MVT::i64x8: return IntegerType::get(Context, 512); case MVT::amdgpuBufferFatPointer: return IntegerType::get(Context, 160); @@ -322,6 +362,22 @@ MVT MVT::getVT(Type *Ty, bool HandleUnknown){ return MVT::getRISCVVectorTupleVT(Sz * NF, NF); } + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.int8") + return MVT(MVT::riscv_xsai_zames_tile_i8); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.int32") + return MVT(MVT::riscv_xsai_zames_tile_i32); + if (TargetExtTy->getName() == "riscv.xsai.zmasync.sync") + return MVT(MVT::riscv_xsai_zmasync_sync); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp8e4") + return MVT(MVT::riscv_xsai_zames_tile_fp8e4); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp8e5") + return MVT(MVT::riscv_xsai_zames_tile_fp8e5); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp16") + return MVT(MVT::riscv_xsai_zames_tile_f16); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.bf16") + return MVT(MVT::riscv_xsai_zames_tile_bf16); + if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp32") + return MVT(MVT::riscv_xsai_zames_tile_f32); if (HandleUnknown) return MVT(MVT::Other); llvm_unreachable("Unknown target ext type!"); diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 7af638499cd868..e3bfe9ff5d5084 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -981,6 +981,18 @@ Expected TargetExtType::checkParams(TargetExtType *TTy) { "target extension type riscv.vector.tuple should have one " "type parameter and one integer parameter"); + if ((TTy->Name == "riscv.xsai.zames.tile.int8" || + TTy->Name == "riscv.xsai.zames.tile.int32" || + TTy->Name == "riscv.xsai.zames.tile.fp8e4" || + TTy->Name == "riscv.xsai.zames.tile.fp8e5" || + TTy->Name == "riscv.xsai.zames.tile.fp16" || + TTy->Name == "riscv.xsai.zames.tile.bf16" || + TTy->Name == "riscv.xsai.zames.tile.fp32" || + TTy->Name == "riscv.xsai.zmasync.sync") && + (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 0)) + return createStringError("target extension type " + TTy->Name + + " should have no parameters"); + // Opaque types in the AMDGPU name space. if (TTy->Name == "amdgcn.named.barrier" && (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 1)) { @@ -1077,6 +1089,19 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { TargetExtType::CanBeLocal, TargetExtType::HasZeroInit); } + // Native XSAI values are register-only SSA resources with no memory layout. + if (Name == "riscv.xsai.zames.tile.int8" || + Name == "riscv.xsai.zames.tile.int32" || + Name == "riscv.xsai.zames.tile.fp8e4" || + Name == "riscv.xsai.zames.tile.fp8e5" || + Name == "riscv.xsai.zames.tile.fp16" || + Name == "riscv.xsai.zames.tile.bf16" || + Name == "riscv.xsai.zames.tile.fp32") + return TargetTypeInfo(Type::getVoidTy(C)); + + if (Name == "riscv.xsai.zmasync.sync") + return TargetTypeInfo(Type::getVoidTy(C)); + // DirectX resources if (Name == "dx.Padding") return TargetTypeInfo( diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 0cb8e2800b29cf..a83183b1abc222 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -240,6 +240,7 @@ class RISCVAsmParser : public MCTargetAsmParser { } ParseStatus parseRegReg(OperandVector &Operands); + ParseStatus parseXSAIZamesMCFGSel(OperandVector &Operands); ParseStatus parseXSfmmVType(OperandVector &Operands); ParseStatus parseZcmpStackAdj(OperandVector &Operands, bool ExpectNegative = false); @@ -761,6 +762,8 @@ struct RISCVOperand final : public MCParsedAsmOperand { return IsConstantImm && isUInt(Imm); } + bool isXSAIZamesMCFGSel() const { return isUImm<3>(); } + template bool isUImmShifted() const { int64_t Imm; if (!isExpr()) @@ -1428,6 +1431,19 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp, getRISCVMCRegisterClass(RISCV::FPR64CRegClassID).contains(Reg); bool IsRegVR = getRISCVMCRegisterClass(RISCV::VRRegClassID).contains(Reg); + // ZAMES shares assembly names with BOSC AME, but has distinct register + // records and accumulator encodings. Coerce only for a ZAMES operand class. + if ((Kind == MCK_XSAI_ZAMES_TileReg || Kind == MCK_XSAI_ZAMES_TileOrAccReg) && + Reg >= RISCV::TR0 && Reg <= RISCV::TR3) { + Op.Reg.Reg = RISCV::XSAI_ZAMES_TR0 + (Reg - RISCV::TR0); + return Match_Success; + } + if ((Kind == MCK_XSAI_ZAMES_AccReg || Kind == MCK_XSAI_ZAMES_TileOrAccReg) && + Reg >= RISCV::ACC0 && Reg <= RISCV::ACC3) { + Op.Reg.Reg = RISCV::XSAI_ZAMES_ACC0 + (Reg - RISCV::ACC0); + return Match_Success; + } + if (Op.isGPR() && Kind == MCK_YGPR) { // GPR and capability GPR use the same register names, convert if required. Op.Reg.Reg = convertGPRToYGPR(Reg); @@ -1887,6 +1903,12 @@ MCRegister RISCVAsmParser::matchRegisterNameHelper(StringRef Name) const { static_assert(RISCV::F0_D < RISCV::F0_H, "FPR matching must be updated"); static_assert(RISCV::F0_D < RISCV::F0_F, "FPR matching must be updated"); static_assert(RISCV::F0_D < RISCV::F0_Q, "FPR matching must be updated"); + // Shared matrix names initially resolve to BOSC AME; operand validation + // converts them to ZAMES registers only when the instruction requires it. + static_assert(RISCV::TR0 < RISCV::XSAI_ZAMES_TR0, + "Matrix register matching must be updated"); + static_assert(RISCV::ACC0 < RISCV::XSAI_ZAMES_ACC0, + "Matrix register matching must be updated"); if (!Reg) Reg = MatchRegisterAltName(Name); if (isRVE() && Reg >= RISCV::X16 && Reg <= RISCV::X31) @@ -2573,6 +2595,27 @@ bool RISCVAsmParser::generateVTypeError(SMLoc ErrorLoc) { "e[8|8alt|16|16alt|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]"); } +ParseStatus RISCVAsmParser::parseXSAIZamesMCFGSel(OperandVector &Operands) { + const AsmToken &Tok = getLexer().getTok(); + if (!Tok.is(AsmToken::Identifier)) + return ParseStatus::NoMatch; + + StringRef Name = Tok.getIdentifier(); + if (!Name.consume_front("mcfg")) + return ParseStatus::NoMatch; + + unsigned Cfg = 0; + if (Name.consumeInteger(10, Cfg) || !Name.empty() || Cfg > 7) + return ParseStatus::NoMatch; + + SMLoc S = getLoc(); + SMLoc E = Tok.getEndLoc(); + Operands.push_back(RISCVOperand::createExpr( + MCConstantExpr::create(Cfg, getContext()), S, E, isRV64())); + Lex(); + return ParseStatus::Success; +} + ParseStatus RISCVAsmParser::parseXSfmmVType(OperandVector &Operands) { SMLoc S = getLoc(); diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp index 3853cefd59517b..1a96878d4f1e63 100644 --- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp +++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp @@ -318,6 +318,57 @@ static DecodeStatus DecodeYBNDSWImm(MCInst &Inst, uint64_t Imm, int64_t Address, return MCDisassembler::Success; } +static DecodeStatus +DecodeXSAI_ZAMES_TileRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo >= 4) + return MCDisassembler::Fail; + + MCRegister Reg = RISCV::XSAI_ZAMES_TR0 + RegNo; + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + +static DecodeStatus +DecodeXSAI_ZAMES_AccRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo < 4 || RegNo >= 8) + return MCDisassembler::Fail; + + MCRegister Reg = RISCV::XSAI_ZAMES_ACC0 + (RegNo - 4); + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + +// Union of XSAI_ZAMES_TileReg (0-3) and XSAI_ZAMES_AccReg (4-7) for real +// instructions whose md field accepts either register file (e.g. mzero). +static DecodeStatus +DecodeXSAI_ZAMES_TileOrAccRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo >= 8) + return MCDisassembler::Fail; + + MCRegister Reg = RegNo < 4 ? RISCV::XSAI_ZAMES_TR0 + RegNo + : RISCV::XSAI_ZAMES_ACC0 + (RegNo - 4); + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + +static DecodeStatus +DecodeXSAI_ZMASYNC_SyncRRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo >= 16) + return MCDisassembler::Fail; + + MCRegister Reg = RISCV::XSAI_ZMASYNC_SYNC0 + RegNo; + Inst.addOperand(MCOperand::createReg(Reg)); + return MCDisassembler::Success; +} + static DecodeStatus decodeVMaskReg(MCInst &Inst, uint32_t RegNo, uint64_t Address, const MCDisassembler *Decoder) { diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp index 5d941ff8205ae6..ae4f3aa437f0d3 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp @@ -259,6 +259,14 @@ void RISCVInstPrinter::printXSfmmVType(const MCInst *MI, unsigned OpNo, O << ", w" << Widen; } +void RISCVInstPrinter::printXSAIZamesMCFGSel(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, + raw_ostream &O) { + unsigned Imm = MI->getOperand(OpNo).getImm(); + assert(Imm < 8 && "Invalid MCFG selector immediate"); + markup(O, Markup::Register) << "mcfg" << Imm; +} + // Print a Zcmp RList. If we are printing architectural register names rather // than ABI register names, we need to print "{x1, x8-x9, x18-x27}" for all // registers. Otherwise, we print "{ra, s0-s11}". diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h index 324e131e50089c..6e75a49d0bd41e 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h @@ -54,6 +54,8 @@ class RISCVInstPrinter : public MCInstPrinter { raw_ostream &O); void printXSfmmVType(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); + void printXSAIZamesMCFGSel(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O); void printVMaskReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); void printMatrixReg(const MCInst *MI, unsigned OpNo, diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index 786120abb207d7..cfaad3cc603636 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -31,6 +31,7 @@ include "RISCVSystemOperands.td" //===----------------------------------------------------------------------===// include "RISCVRegisterInfo.td" +include "RISCVRegisterInfoXSAIZames.td" include "RISCVSchedule.td" include "RISCVCallingConv.td" include "RISCVInstrInfo.td" diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp index fb96ae371c73e5..9f9ca9bd2054c6 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreEmit.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/Support/ErrorHandling.h" using namespace llvm; @@ -62,6 +63,9 @@ class RISCVExpandPseudoPreEmitImpl final : public RISCVExpandPseudoImplBase { bool expandBOSCAMEOrderedTile(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; + + bool expandXSAIZamesMatrixPseudo(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; }; class RISCVExpandPseudoPreEmitLegacy : public MachineFunctionPass { @@ -177,6 +181,68 @@ bool RISCVExpandPseudoPreEmitImpl::expandMI( return expandPseudoReadVLENBViaVSETVLIX0(MBB, MBBI); case RISCV::PseudoClearFPR64: return expandPseudoClearFPR64(MBB, MBBI); + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32: + return expandXSAIZamesMatrixPseudo(MBB, MBBI); } return false; @@ -274,6 +340,243 @@ bool RISCVExpandPseudoPreEmitImpl::expandBOSCAMEOrderedTile( return true; } +static unsigned getXSAIZamesMcfgSelector(Register Reg) { + switch (Reg) { + case RISCV::XSAI_ZAMES_TR0: + return 0; + case RISCV::XSAI_ZAMES_TR1: + return 1; + case RISCV::XSAI_ZAMES_TR2: + return 2; + case RISCV::XSAI_ZAMES_TR3: + return 3; + case RISCV::XSAI_ZAMES_ACC0: + return 4; + case RISCV::XSAI_ZAMES_ACC1: + return 5; + case RISCV::XSAI_ZAMES_ACC2: + return 6; + case RISCV::XSAI_ZAMES_ACC3: + return 7; + default: + report_fatal_error( + "ZAMES msetcfg pseudo expansion requires tr0-tr3 or acc0-acc3"); + } +} + +static void emitXSAIZamesConfig(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, DebugLoc DL, + Register MatrixReg, Register ConfigReg) { + unsigned Selector = getXSAIZamesMcfgSelector(MatrixReg); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSETCFG)) + .addImm(Selector) + .addReg(ConfigReg); +} + +bool RISCVExpandPseudoPreEmitImpl::expandXSAIZamesMatrixPseudo( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + unsigned Opc = MI.getOpcode(); + + switch (Opc) { + default: + llvm_unreachable("Unexpected ZAMES pseudo opcode"); + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLA)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLB)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLBT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLAT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLC)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MLCT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(1).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MZERO)) + .add(MI.getOperand(0)); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(1).getReg(), + MI.getOperand(4).getReg()); + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(2).getReg(), + MI.getOperand(5).getReg()); + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(3).getReg(), + MI.getOperand(6).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MMACC)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(3)); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSC)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSA)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSAT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSB)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSBT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32: { + emitXSAIZamesConfig(TII, MBB, MBBI, DL, MI.getOperand(0).getReg(), + MI.getOperand(3).getReg()); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::XSAI_ZAMES_MSCT)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .cloneMemRefs(MI); + break; + } + } + + MI.eraseFromParent(); + return true; +} + bool RISCVExpandPseudoPreEmitImpl::expandCCOp( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NextMBBI) const { diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp index d64605fd27feb0..d7ca0d2ff408d1 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoPreRA.cpp @@ -17,7 +17,9 @@ #include "RISCVSubtarget.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/MC/MCContext.h" +#include "llvm/Support/ErrorHandling.h" using namespace llvm; @@ -48,6 +50,9 @@ class RISCVExpandPseudoPreRAImpl final : public RISCVExpandPseudoImplBase { bool expandLoadTLSDescAddress(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; + + bool expandXSAIZamesConfigPseudo(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const; }; class RISCVExpandPseudoPreRALegacy : public MachineFunctionPass { @@ -90,11 +95,540 @@ bool RISCVExpandPseudoPreRAImpl::expandMI( return expandLoadTLSGDAddress(MBB, MBBI); case RISCV::PseudoLA_TLSDESC: return expandLoadTLSDescAddress(MBB, MBBI); + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE: + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE: + return expandXSAIZamesConfigPseudo(MBB, MBBI); } return false; } +static constexpr int64_t XSAIZamesCfgI8 = 0x02; +static constexpr int64_t XSAIZamesCfgI32 = 0x04; +static constexpr int64_t XSAIZamesCfgF8E5 = 0x07; +static constexpr int64_t XSAIZamesCfgF8E4 = 0x08; +static constexpr int64_t XSAIZamesCfgF16 = 0x09; +static constexpr int64_t XSAIZamesCfgBF16 = 0x0a; +static constexpr int64_t XSAIZamesCfgF32 = 0x0c; + +static Register createXSAIZamesConfigPayload(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + DebugLoc DL, int64_t Payload) { + MachineFunction *MF = MBB.getParent(); + Register ConfigReg = + MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), ConfigReg) + .addReg(RISCV::X0) + .addImm(Payload); + return ConfigReg; +} + +// Expand a single-config _PRE pseudo: materialize the payload GPR, rebuild +// the second-level pseudo with all original operands plus the payload +// register, and carry the memory operands over. +static void expandXSAIZamesPreRASingleConfig(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineInstr &MI, unsigned NewOpc, + int64_t Payload) { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, MI.getDebugLoc(), Payload); + auto MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc)); + for (const MachineOperand &MO : MI.operands()) + MIB.add(MO); + MIB.addReg(ConfigReg); + MIB.cloneMemRefs(MI); +} + +// Expand an mmacc _PRE pseudo: materialize the accumulator and matrix +// payload GPRs and rebuild the second-level pseudo. The B and A matrices +// always share the element type in every supported combination, so one +// payload register feeds both config operands. +static void expandXSAIZamesPreRAMMAcc(const RISCVInstrInfo *TII, + MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineInstr &MI, unsigned NewOpc, + int64_t AccPayload, int64_t MatPayload) { + DebugLoc DL = MI.getDebugLoc(); + Register AccConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, AccPayload); + Register MatConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, MatPayload); + BuildMI(MBB, MBBI, DL, TII->get(NewOpc)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(3)) + .addReg(AccConfigReg) + .addReg(MatConfigReg) + .addReg(MatConfigReg); +} + +bool RISCVExpandPseudoPreRAImpl::expandXSAIZamesConfigPseudo( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + unsigned Opc = MI.getOpcode(); + + switch (Opc) { + default: + llvm_unreachable("Unexpected ZAMES pre-RA pseudo opcode"); + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLA_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLB_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLC_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_I32)) + .add(MI.getOperand(0)) + .addReg(ConfigReg); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE: { + Register AccConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + Register MatConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .add(MI.getOperand(3)) + .addReg(AccConfigReg) + .addReg(MatConfigReg) + .addReg(MatConfigReg); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSC_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSA_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSB_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI8); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_I8)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE: { + Register ConfigReg = + createXSAIZamesConfigPayload(TII, MBB, MBBI, DL, XSAIZamesCfgI32); + BuildMI(MBB, MBBI, DL, TII->get(RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_I32)) + .add(MI.getOperand(0)) + .add(MI.getOperand(1)) + .add(MI.getOperand(2)) + .addReg(ConfigReg) + .cloneMemRefs(MI); + break; + } + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLA_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLAT_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLB_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLBT_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLC_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MLCT_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSA_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSAT_A_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSB_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSBT_B_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSC_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MSCT_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_I8, + XSAIZamesCfgI8); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4, + XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5, + XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_F16, + XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_T_BF16, + XSAIZamesCfgBF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE: + expandXSAIZamesPreRASingleConfig(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MZERO_C_F32, + XSAIZamesCfgF32); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_I8_F32, + XSAIZamesCfgF32, XSAIZamesCfgI8); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32, + XSAIZamesCfgF32, XSAIZamesCfgF8E4); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32, + XSAIZamesCfgF32, XSAIZamesCfgF8E5); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_F16_F32, + XSAIZamesCfgF32, XSAIZamesCfgF16); + break; + case RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE: + expandXSAIZamesPreRAMMAcc(TII, MBB, MBBI, MI, + RISCV::PSEUDO_XSAI_ZAMES_MMACC_BF16_F32, + XSAIZamesCfgF32, XSAIZamesCfgBF16); + break; + } + + MI.eraseFromParent(); + return true; +} + bool RISCVExpandPseudoPreRAImpl::expandAuipcInstPair( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned FlagsHi, unsigned SecondOpcode) const { diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index c4c5115a973028..91a40b85072bc8 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1008,6 +1008,27 @@ def HasStdExtZvfbdota32f AssemblerPredicate<(all_of FeatureStdExtZvfbdota32f), "'Zvfbdota32f' (FP32 batched dot-product extension)">; +// Buddy matrix extensions + +def FeatureStdExtZames + : RISCVExtension<1, 0, "Buddy simplified matrix extension", [], + "HasZames">; + +def FeatureStdExtZmasync + : RISCVExtension<1, 0, "Buddy matrix async sync extension", + [FeatureStdExtZames], + "HasZmasync">; + +def HasStdExtZames + : Predicate<"Subtarget->hasZames()">, + AssemblerPredicate<(all_of FeatureStdExtZames), + "'ZAMES' (simplified matrix extension)">; + +def HasStdExtZmasync + : Predicate<"Subtarget->hasZmasync()">, + AssemblerPredicate<(all_of FeatureStdExtZmasync), + "'ZMASYNC' (matrix async sync extension)">; + // Vector instruction predicates def HasVInstructions : Predicate<"Subtarget->hasVInstructions()">, diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h index 7ee67ae085e22d..d13d9d7b501693 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -16,6 +16,9 @@ #include "RISCV.h" #include "RISCVTargetMachine.h" #include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/KnownBits.h" // RISC-V specific code to select RISC-V machine instructions for @@ -33,6 +36,33 @@ class RISCVDAGToDAGISel : public SelectionDAGISel { bool runOnMachineFunction(MachineFunction &MF) override { Subtarget = &MF.getSubtarget(); + // ZAMES/ZMASYNC intrinsics use i64 scalar interfaces and their + // SelectionDAG patterns and pseudos are constrained to RV64. Diagnose + // them before instruction selection on RV32 instead of failing with an + // uncontrolled "Cannot select" or a crash. + if (!Subtarget->is64Bit()) { + Function &F = MF.getFunction(); + for (Instruction &I : instructions(F)) { + auto *II = dyn_cast(&I); + if (!II) + continue; + StringRef Name = Intrinsic::getBaseName(II->getIntrinsicID()); + if (Name.starts_with("llvm.riscv.xsai.zames.") || + Name.starts_with("llvm.riscv.xsai.zmasync.")) { + F.getContext().diagnose(DiagnosticInfoUnsupported( + F, "ZAMES/ZMASYNC code generation requires RV64", + I.getDebugLoc())); + // Leave a minimal, valid function body so later passes do not + // trip over the unselected machine function; compilation still + // fails with the error emitted above. + MachineBasicBlock *Entry = MF.CreateMachineBasicBlock(); + MF.push_back(Entry); + BuildMI(Entry, DebugLoc(), + Subtarget->getInstrInfo()->get(RISCV::PseudoRET)); + return true; + } + } + } return SelectionDAGISel::runOnMachineFunction(MF); } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 189ae725cf5738..a0ff91e81ad648 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -196,6 +196,26 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, addRegisterClass(VT, &RISCV::MatrixRegRegClass); } + if (Subtarget.hasZames()) { + addRegisterClass(MVT::riscv_xsai_zames_tile_i8, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_zames_tile_i32, + &RISCV::XSAI_ZAMES_AccRegRegClass); + addRegisterClass(MVT::riscv_xsai_zames_tile_fp8e4, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_zames_tile_fp8e5, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_zames_tile_f16, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_zames_tile_bf16, + &RISCV::XSAI_ZAMES_TileRegRegClass); + addRegisterClass(MVT::riscv_xsai_zames_tile_f32, + &RISCV::XSAI_ZAMES_AccRegRegClass); + } + if (Subtarget.hasZmasync()) + addRegisterClass(MVT::riscv_xsai_zmasync_sync, + &RISCV::XSAI_ZMASYNC_SyncRRegClass); + static const MVT::SimpleValueType BoolVecVTs[] = { MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1, MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1}; @@ -2276,6 +2296,44 @@ void RISCVTargetLowering::getTgtMemIntrinsic( Infos.push_back(Info); }; + auto GetXSAIZamesMemoryVT = [&](bool IsStore) { + Type *TileTy = IsStore ? I.getArgOperand(0)->getType() : I.getType(); + auto *TargetTy = dyn_cast(TileTy); + if (!TargetTy) + return MVT(MVT::Other); + return StringSwitch(TargetTy->getName()) + .Cases({"riscv.xsai.zames.tile.int8", "riscv.xsai.zames.tile.fp8e4", + "riscv.xsai.zames.tile.fp8e5"}, + MVT::i8) + .Cases({"riscv.xsai.zames.tile.fp16", "riscv.xsai.zames.tile.bf16"}, + MVT::i16) + .Cases({"riscv.xsai.zames.tile.int32", "riscv.xsai.zames.tile.fp32"}, + MVT::i32) + .Default(MVT::Other); + }; + + auto SetXSAIZamesMemoryInfo = [&](unsigned PtrOp, bool IsStore, + bool UsePtrVal) { + Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN; + Info.memVT = GetXSAIZamesMemoryVT(IsStore); + if (Info.memVT == MVT::Other) + return; + + const Value *Ptr = I.getArgOperand(PtrOp); + if (UsePtrVal) + Info.ptrVal = Ptr; + else + Info.fallbackAddressSpace = Ptr->getType()->getPointerAddressSpace(); + + Info.offset = 0; + Info.size = MemoryLocation::UnknownSize; + if (MaybeAlign Alignment = I.getParamAlign(PtrOp)) + Info.align = Alignment; + Info.flags |= + IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad; + Infos.push_back(Info); + }; + if (I.hasMetadata(LLVMContext::MD_nontemporal)) Info.flags |= MachineMemOperand::MONonTemporal; @@ -2604,6 +2662,42 @@ void RISCVTargetLowering::getTgtMemIntrinsic( Info.flags |= MachineMemOperand::MOStore; Infos.push_back(Info); return; + case Intrinsic::riscv_xsai_zames_mla: + case Intrinsic::riscv_xsai_zames_mlat: + case Intrinsic::riscv_xsai_zames_mlb: + case Intrinsic::riscv_xsai_zames_mlbt: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_xsai_zames_mlc: + case Intrinsic::riscv_xsai_zames_mlct: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_xsai_zames_msa: + case Intrinsic::riscv_xsai_zames_msat: + case Intrinsic::riscv_xsai_zames_msb: + case Intrinsic::riscv_xsai_zames_msbt: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_xsai_zames_msc: + case Intrinsic::riscv_xsai_zames_msct: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ false); + return; + case Intrinsic::riscv_xsai_zames_mla_whole: + case Intrinsic::riscv_xsai_zames_mlb_whole: + case Intrinsic::riscv_xsai_zames_mlc_whole: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 0, /*IsStore*/ false, + /*UsePtrVal*/ true); + return; + case Intrinsic::riscv_xsai_zames_msa_whole: + case Intrinsic::riscv_xsai_zames_msb_whole: + case Intrinsic::riscv_xsai_zames_msc_whole: + SetXSAIZamesMemoryInfo(/*PtrOp*/ 1, /*IsStore*/ true, + /*UsePtrVal*/ true); + return; } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index ba9331d368a235..663c0162872ba0 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -2528,6 +2528,10 @@ include "RISCVInstrInfoSFB.td" // CHERI include "RISCVInstrInfoY.td" +// ZAMES/ZMASYNC matrix extensions +include "RISCVInstrInfoXSAIZames.td" +include "RISCVInstrInfoXSAIZamesPatterns.td" + //===----------------------------------------------------------------------===// // Vendor extensions //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td new file mode 100644 index 00000000000000..c669fb4c2ac26b --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZames.td @@ -0,0 +1,309 @@ +//===-- XSAI Zames instructions ----------------------------*- tablegen -*-===// +// RISCVInstrInfoXSAIZames.td +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC real instruction definitions (MC layer). Codegen-only +// pseudos and SelectionDAG patterns are intentionally not part of this file. +// +//===----------------------------------------------------------------------===// + +def xsai_zames_uimm10 : RISCVUImmLeafOp<10>; + +// mcfg0..mcfg7 select the hardware matrix configuration register configured by +// msetcfg/mgetcfg. The value is encoded as a 3-bit immediate. +def XSAIZamesMCFGSelAsmOperand : AsmOperandClass { + let Name = "XSAIZamesMCFGSel"; + let RenderMethod = "addImmOperands"; + let ParserMethod = "parseXSAIZamesMCFGSel"; +} + +def xsai_zames_mcfgsel : RISCVUImmLeafOp<3> { + let ParserMatchClass = XSAIZamesMCFGSelAsmOperand; + let PrintMethod = "printXSAIZamesMCFGSel"; +} + +class XSAI_ZAMES_InstConfigImm func4, string opcodestr> + : RVInst<(outs), (ins xsai_zames_uimm10:$imm), opcodestr, "$imm", [], + InstFormatI> { + bits<10> imm; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = 0b0; + let Inst{24-15} = imm; + let Inst{14-12} = 0b000; + let Inst{11-7} = 0b00000; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstConfigReg func4, string opcodestr> + : RVInst<(outs), (ins GPR:$rs1), opcodestr, "$rs1", [], + InstFormatI> { + bits<5> rs1; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = 0b1; + let Inst{24-20} = 0b00000; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; + let Inst{11-7} = 0b00000; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstSetCfg + : RVInst<(outs), (ins xsai_zames_mcfgsel:$cfg, GPR:$rs1), opcodestr, "$cfg, $rs1", + [], InstFormatI> { + bits<3> cfg; + bits<5> rs1; + + let Inst{31-28} = 0b0100; + let Inst{27-26} = 0b00; + let Inst{25} = 0b1; + let Inst{24-20} = 0b00000; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; + let Inst{11-10} = 0b00; + let Inst{9-7} = cfg; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstGetCfg + : RVInst<(outs GPR:$rd), (ins xsai_zames_mcfgsel:$cfg), opcodestr, "$rd, $cfg", + [], InstFormatI> { + bits<5> rd; + bits<3> cfg; + + let Inst{31-28} = 0b0101; + let Inst{27-26} = 0b00; + let Inst{25} = 0b0; + let Inst{24-20} = 0b00000; + let Inst{19-18} = 0b00; + let Inst{17-15} = cfg; + let Inst{14-12} = 0b000; + let Inst{11-7} = rd; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstInit + : RVInst<(outs), (ins), opcodestr, "", [], InstFormatI> { + let Inst{31-7} = 0; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstLoadStore func4, bit ls, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + bits<5> rs1; + bits<5> rs2; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = ls; + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b001; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstWholeLoadStore func4, bit ls, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + bits<5> rs1; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = ls; + let Inst{24-20} = 0b00000; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b001; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstMMAcc func4, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + bits<5> ms1; + bits<5> ms2; + + let Inst{31-28} = func4; + let Inst{27-25} = 0b000; + let Inst{24-20} = ms2; + let Inst{19-15} = ms1; + let Inst{14-12} = 0b010; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +class XSAI_ZAMES_InstMisc func4, dag outs, dag ins, + string opcodestr> + : RVInst { + bits<5> md; + + let Inst{31-28} = func4; + let Inst{27-15} = 0b0000000000000; + let Inst{14-12} = 0b011; + let Inst{11-7} = md; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def XSAI_ZAMES_MSETTILEMI : XSAI_ZAMES_InstConfigImm<0b0010, "msettilemi">; + def XSAI_ZAMES_MSETTILENI : XSAI_ZAMES_InstConfigImm<0b0011, "msettileni">; + def XSAI_ZAMES_MSETTILEKI : XSAI_ZAMES_InstConfigImm<0b0001, "msettileki">; + + def XSAI_ZAMES_MSETTILEM : XSAI_ZAMES_InstConfigReg<0b0010, "msettilem">; + def XSAI_ZAMES_MSETTILEN : XSAI_ZAMES_InstConfigReg<0b0011, "msettilen">; + def XSAI_ZAMES_MSETTILEK : XSAI_ZAMES_InstConfigReg<0b0001, "msettilek">; + + def XSAI_ZAMES_MSETCFG : XSAI_ZAMES_InstSetCfg<"msetcfg">; + def XSAI_ZAMES_MGETCFG : XSAI_ZAMES_InstGetCfg<"mgetcfg">; + + def XSAI_ZAMES_MINIT : XSAI_ZAMES_InstInit<"minit">; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 1, + mayStore = 0 in { + def XSAI_ZAMES_MLA : XSAI_ZAMES_InstLoadStore<0b0000, 0b0, (outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mla">; + def XSAI_ZAMES_MLB : XSAI_ZAMES_InstLoadStore<0b0001, 0b0, (outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlb">; + def XSAI_ZAMES_MLC : XSAI_ZAMES_InstLoadStore<0b0010, 0b0, (outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlc">; + def XSAI_ZAMES_MLBT : XSAI_ZAMES_InstLoadStore<0b0101, 0b0, (outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlbt">; + def XSAI_ZAMES_MLAT : XSAI_ZAMES_InstLoadStore<0b0100, 0b0, (outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlat">; + def XSAI_ZAMES_MLCT : XSAI_ZAMES_InstLoadStore<0b0110, 0b0, (outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), + "mlct">; + + def XSAI_ZAMES_MLA_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1000, 0b0, (outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1), "mla.whole">; + def XSAI_ZAMES_MLB_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1001, 0b0, (outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1), "mlb.whole">; + def XSAI_ZAMES_MLC_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1010, 0b0, (outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1), "mlc.whole">; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, + mayStore = 1 in { + def XSAI_ZAMES_MSC : XSAI_ZAMES_InstLoadStore<0b0010, 0b1, (outs), + (ins XSAI_ZAMES_AccReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msc">; + def XSAI_ZAMES_MSA : XSAI_ZAMES_InstLoadStore<0b0000, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msa">; + def XSAI_ZAMES_MSB : XSAI_ZAMES_InstLoadStore<0b0001, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msb">; + def XSAI_ZAMES_MSAT : XSAI_ZAMES_InstLoadStore<0b0100, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msat">; + def XSAI_ZAMES_MSBT : XSAI_ZAMES_InstLoadStore<0b0101, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msbt">; + def XSAI_ZAMES_MSCT : XSAI_ZAMES_InstLoadStore<0b0110, 0b1, (outs), + (ins XSAI_ZAMES_AccReg:$md, + GPRMemZeroOffset:$rs1, GPR:$rs2), + "msct">; + + def XSAI_ZAMES_MSA_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1000, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1), + "msa.whole">; + def XSAI_ZAMES_MSB_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1001, 0b1, (outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1), + "msb.whole">; + def XSAI_ZAMES_MSC_WHOLE + : XSAI_ZAMES_InstWholeLoadStore<0b1010, 0b1, (outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1), + "msc.whole">; +} + +let Predicates = [HasStdExtZames], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def XSAI_ZAMES_MZERO : XSAI_ZAMES_InstMisc<0b0000, (outs XSAI_ZAMES_TileOrAccReg:$md), (ins), + "mzero">; + + def XSAI_ZAMES_MMACC : XSAI_ZAMES_InstMMAcc<0b0000, (outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, + XSAI_ZAMES_TileReg:$ms1), + "mmacc"> { + let Constraints = "$md_wb = $md"; + } +} + +//===----------------------------------------------------------------------===// +// ZMASYNC Matrix Async Synchronization Instructions +//===----------------------------------------------------------------------===// + +class XSAI_ZMASYNC_Inst func4, dag outs, dag ins, + string opcodestr, string argstr> + : RVInst { + bits<5> rs1; + bits<5> sync; + + let Inst{31-28} = func4; + let Inst{27-26} = 0b00; + let Inst{25} = 0b0; + let Inst{24-20} = sync; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b100; + let Inst{11-7} = 0b00000; + let Inst{6-0} = OPC_CUSTOM_1.Value; +} + +let Predicates = [HasStdExtZmasync], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def XSAI_ZMASYNC_MSYNCREGRESET + : XSAI_ZMASYNC_Inst<0b1000, (outs XSAI_ZMASYNC_SyncR:$sync), (ins), + "msyncregreset", "$sync"> { + let rs1 = 0b00000; + } + + let Constraints = "$sync = $src" in { + def XSAI_ZMASYNC_MRELEASE + : XSAI_ZMASYNC_Inst<0b1001, (outs XSAI_ZMASYNC_SyncR:$sync), (ins XSAI_ZMASYNC_SyncR:$src), + "mrelease", "$sync"> { + let rs1 = 0b00000; + } + } + + def XSAI_ZMASYNC_MACQUIRE + : XSAI_ZMASYNC_Inst<0b1010, (outs), (ins XSAI_ZMASYNC_SyncR:$sync, GPR:$rs1), + "macquire", "$sync, $rs1">; + + def XSAI_ZMASYNC_MFENCE + : XSAI_ZMASYNC_Inst<0b1011, (outs), (ins), "mfence", ""> { + let rs1 = 0b00000; + let sync = 0b00000; + } +} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td new file mode 100644 index 00000000000000..2ac553c2d8428c --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSAIZamesPatterns.td @@ -0,0 +1,595 @@ +//===-- XSAI Zames patterns --------------------------------*- tablegen -*-===// +// RISCVInstrInfoXSAIZamesPatterns.td +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC codegen-only pseudos and SelectionDAG patterns. +// +// Ordinary load/store, mzero and mmacc go through a two-level pseudo: +// intrinsic -> PSEUDO_XSAI_ZAMES_*_PRE -> (pre-RA: materialize cfg payload GPR) +// -> PSEUDO_XSAI_ZAMES_* with cfg operands -> RA -> (post-RA: emit msetcfg by +// final TR/ACC encoding) -> real ZAMES instruction. +// +// msettile*, msetcfg/mgetcfg, minit, *.whole and the four ZMASYNC +// instructions are selected directly to real instructions. +// +// mmacc operand reorder: LLVM IR operands are A, B, C (the intrinsic +// overload suffix lists the C, A, B *type* sequence, not the operand order); +// the typed pseudo and real instruction use the machine order C, B, A. +// +//===----------------------------------------------------------------------===// + +// Defensive second-layer immediate constraints. The ImmArg/Range intrinsic +// attributes reject out-of-range constants in the IR verifier; these leaves +// additionally prevent ISel from silently truncating an out-of-range target +// constant into the 10-bit tile field or the 3-bit config selector when the +// verifier is disabled or a transform built an illegal DAG. +def xsai_zames_tuimm10 : TImmLeaf(Imm); }]>; +def xsai_zames_tuimm3 : TImmLeaf(Imm); }]>; + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 1, + mayStore = 0, isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + + def PSEUDO_XSAI_ZAMES_MLA_A_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLB_B_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLBT_B_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLAT_A_I8 + : Pseudo<(outs XSAI_ZAMES_TileReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLC_C_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MLCT_C_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; + } +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, + mayStore = 0, isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE : Pseudo<(outs XSAI_ZAMES_AccReg:$md), (ins), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1), []> { + let Constraints = "$md_wb = $md"; + let Size = 28; + } + + def PSEUDO_XSAI_ZAMES_MZERO_C_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md), (ins GPR:$cfg), []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MMACC_I8_I32 + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1, GPR:$cfg_c, + GPR:$cfg_b, GPR:$cfg_a), []> { + let Constraints = "$md_wb = $md"; + let Size = 16; + } +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 1, mayLoad = 0, + mayStore = 1, isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + def PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; + } + + def PSEUDO_XSAI_ZAMES_MSC_C_I32 + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSA_A_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSAT_A_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSB_B_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSBT_B_I8 + : Pseudo<(outs), + (ins XSAI_ZAMES_TileReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } + def PSEUDO_XSAI_ZAMES_MSCT_C_I32 + : Pseudo<(outs), + (ins XSAI_ZAMES_AccReg:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), + []> { + let Size = 8; + } +} + +class PatXSAI_ZAMES_SetTileReg + : Pat<(OpNode (XLenVT GPR:$rs1)), (Inst GPR:$rs1)>; + +// Shared shells for the typed ZAMES pseudo instructions. Concrete defs carry +// the PSEUDO_XSAI_ZAMES___ names so every instance stays +// reachable from the pre-RA and late C++ expansion switches. +class XSAI_ZAMES_LdPseudoPRE + : Pseudo<(outs RC:$md), (ins GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; +} +class XSAI_ZAMES_LdPseudoCFG + : Pseudo<(outs RC:$md), + (ins GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; +} +class XSAI_ZAMES_StPseudoPRE + : Pseudo<(outs), (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2), []> { + let Size = 12; +} +class XSAI_ZAMES_StPseudoCFG + : Pseudo<(outs), + (ins RC:$md, GPRMemZeroOffset:$rs1, GPR:$rs2, GPR:$cfg), []> { + let Size = 8; +} +class XSAI_ZAMES_MZeroPseudoPRE + : Pseudo<(outs RC:$md), (ins), []> { + let Size = 12; +} +class XSAI_ZAMES_MZeroPseudoCFG + : Pseudo<(outs RC:$md), (ins GPR:$cfg), []> { + let Size = 8; +} +class XSAI_ZAMES_MMAccPseudoPRE + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1), []> { + let Constraints = "$md_wb = $md"; + let Size = 28; +} +class XSAI_ZAMES_MMAccPseudoCFG + : Pseudo<(outs XSAI_ZAMES_AccReg:$md_wb), + (ins XSAI_ZAMES_AccReg:$md, XSAI_ZAMES_TileReg:$ms2, XSAI_ZAMES_TileReg:$ms1, GPR:$cfg_c, + GPR:$cfg_b, GPR:$cfg_a), []> { + let Constraints = "$md_wb = $md"; + let Size = 16; +} + +// Typed load pseudo instances: A/B tiles use every supported input element; +// C accumulators use fp32. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 1, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLA_A_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLAT_A_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLB_B_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_F16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLBT_B_BF16 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLC_C_F32 : XSAI_ZAMES_LdPseudoCFG; + def PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE : XSAI_ZAMES_LdPseudoPRE; + def PSEUDO_XSAI_ZAMES_MLCT_C_F32 : XSAI_ZAMES_LdPseudoCFG; +} + +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 1, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSA_A_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSAT_A_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSB_B_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_F16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSBT_B_BF16 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSC_C_F32 : XSAI_ZAMES_StPseudoCFG; + def PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE : XSAI_ZAMES_StPseudoPRE; + def PSEUDO_XSAI_ZAMES_MSCT_C_F32 : XSAI_ZAMES_StPseudoCFG; +} + +// Typed mzero pseudo instances preserve the input/output register partition. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_I8 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_F16 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_T_BF16 : XSAI_ZAMES_MZeroPseudoCFG; + def PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE : XSAI_ZAMES_MZeroPseudoPRE; + def PSEUDO_XSAI_ZAMES_MZERO_C_F32 : XSAI_ZAMES_MZeroPseudoCFG; +} + +// Typed mmacc pseudo instances for the six supported combinations. +let Predicates = [HasStdExtZames, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_I8_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_F16_F32 : XSAI_ZAMES_MMAccPseudoCFG; + def PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE : XSAI_ZAMES_MMAccPseudoPRE; + def PSEUDO_XSAI_ZAMES_MMACC_BF16_F32 : XSAI_ZAMES_MMAccPseudoCFG; +} + +class PatXSAI_ZAMES_Load + : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1, GPR:$rs2)), + (Inst GPRMemZeroOffset:$rs1, GPR:$rs2)>; + +class PatXSAI_ZAMES_Store + : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2), + (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1, GPR:$rs2)>; + +class PatXSAI_ZAMES_WholeLoad + : Pat<(RetTy (OpNode GPRMemZeroOffset:$rs1)), + (Inst GPRMemZeroOffset:$rs1)>; + +class PatXSAI_ZAMES_WholeStore + : Pat<(OpNode (RegTy Reg:$md), GPRMemZeroOffset:$rs1), + (Inst (RegTy Reg:$md), GPRMemZeroOffset:$rs1)>; + +let Predicates = [HasStdExtZames, IsRV64] in { + def : Pat<(int_riscv_xsai_zames_msettilemi xsai_zames_tuimm10:$imm), + (XSAI_ZAMES_MSETTILEMI timm:$imm)>; + def : Pat<(int_riscv_xsai_zames_msettileni xsai_zames_tuimm10:$imm), + (XSAI_ZAMES_MSETTILENI timm:$imm)>; + def : Pat<(int_riscv_xsai_zames_msettileki xsai_zames_tuimm10:$imm), + (XSAI_ZAMES_MSETTILEKI timm:$imm)>; + + def : PatXSAI_ZAMES_SetTileReg; + def : PatXSAI_ZAMES_SetTileReg; + def : PatXSAI_ZAMES_SetTileReg; + + def : Pat<(int_riscv_xsai_zames_msetcfg xsai_zames_tuimm3:$cfg, (XLenVT GPR:$rs1)), + (XSAI_ZAMES_MSETCFG timm:$cfg, GPR:$rs1)>; + + def : Pat<(int_riscv_xsai_zames_minit), (XSAI_ZAMES_MINIT)>; + def : Pat<(XLenVT (int_riscv_xsai_zames_mgetcfg xsai_zames_tuimm3:$cfg)), + (XSAI_ZAMES_MGETCFG timm:$cfg)>; + + def : Pat<(xsai_mi32_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE)>; + + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + + def : Pat<(xsai_mi32_t (int_riscv_xsai_zames_mmacc (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mi32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE (xsai_mi32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1))>; + + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + def : PatXSAI_ZAMES_Load; + + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeLoad; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + def : PatXSAI_ZAMES_WholeStore; + + def : Pat<(xsai_mi8_t (int_riscv_xsai_zames_mzero)), (PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE)>; + def : Pat<(xsai_mf8e4_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE)>; + def : Pat<(xsai_mf8e5_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE)>; + def : Pat<(xsai_mf16_t (int_riscv_xsai_zames_mzero)), (PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE)>; + def : Pat<(xsai_mbf16_t (int_riscv_xsai_zames_mzero)), + (PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE)>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mzero)), (PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE)>; + + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mi8_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf8e4_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf8e5_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mf16_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf16_t XSAI_ZAMES_TileReg:$rs1))>; + def : Pat<(xsai_mf32_t (int_riscv_xsai_zames_mmacc (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs1), + (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3))), + (PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE (xsai_mf32_t XSAI_ZAMES_AccReg:$rs3), + (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs2), + (xsai_mbf16_t XSAI_ZAMES_TileReg:$rs1))>; + + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; + def : PatXSAI_ZAMES_Store; +} + +let Predicates = [HasStdExtZmasync, IsRV64] in { + def : Pat<(xsai_msync_t (int_riscv_xsai_zmasync_msyncregreset)), + (XSAI_ZMASYNC_MSYNCREGRESET)>; + def : Pat<(xsai_msync_t (int_riscv_xsai_zmasync_mrelease (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync))), + (XSAI_ZMASYNC_MRELEASE (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync))>; + def : Pat<(int_riscv_xsai_zmasync_macquire (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync), + (XLenVT GPR:$rs1)), + (XSAI_ZMASYNC_MACQUIRE (xsai_msync_t XSAI_ZMASYNC_SyncR:$sync), (XLenVT GPR:$rs1))>; + def : Pat<(int_riscv_xsai_zmasync_mfence), (XSAI_ZMASYNC_MFENCE)>; +} diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td new file mode 100644 index 00000000000000..e56c09a33e9505 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfoXSAIZames.td @@ -0,0 +1,97 @@ +//===-- XSAI Zames registers -------------------------------*- tablegen -*-===// +// RISCVRegisterInfoXSAIZames.td +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ZAMES/ZMASYNC register definitions. +// +// - 4 Tile Registers (tr0-tr3): input matrices A and B +// - 4 Accumulation Registers (acc0-acc3): output/accumulation matrix C +// - 16 Sync Registers (sync0-sync15): ZMASYNC synchronization +// +// RegisterClass Size values are placeholders; the real MLEN/AMUL ABI is not +// part of the MC-level migration. +// +//===----------------------------------------------------------------------===// + +let Namespace = "RISCV" in { + +// Base class for Tile Registers +class XSAI_ZAMES_TileRegBase Enc, string n> : Register { + let HWEncoding{2-0} = Enc; + let HWEncoding{4-3} = 0b00; +} + +// Keep ZAMES records distinct from BOSC AME's identically spelled registers. +def XSAI_ZAMES_TR0 : XSAI_ZAMES_TileRegBase<0, "tr0">; +def XSAI_ZAMES_TR1 : XSAI_ZAMES_TileRegBase<1, "tr1">; +def XSAI_ZAMES_TR2 : XSAI_ZAMES_TileRegBase<2, "tr2">; +def XSAI_ZAMES_TR3 : XSAI_ZAMES_TileRegBase<3, "tr3">; + +// Base class for Accumulation Registers +class XSAI_ZAMES_AccRegBase Enc, string n> : Register { + let HWEncoding{2-0} = Enc; + let HWEncoding{4-3} = 0b00; +} + +// Define 4 Accumulation Registers: acc0-acc3 occupy the hardware register +// slots 4-7. +def XSAI_ZAMES_ACC0 : XSAI_ZAMES_AccRegBase<4, "acc0">; +def XSAI_ZAMES_ACC1 : XSAI_ZAMES_AccRegBase<5, "acc1">; +def XSAI_ZAMES_ACC2 : XSAI_ZAMES_AccRegBase<6, "acc2">; +def XSAI_ZAMES_ACC3 : XSAI_ZAMES_AccRegBase<7, "acc3">; + +// ZMASYNC Sync Registers (sync0-sync15) +class XSAI_ZMASYNC_SyncReg Enc, string n> : Register { + let HWEncoding{4-0} = Enc; +} + +foreach Index = 0-15 in { + def XSAI_ZMASYNC_SYNC#Index : XSAI_ZMASYNC_SyncReg; +} + +} // End Namespace = "RISCV" + +defvar xsai_mi8_t = riscv_xsai_zames_tile_i8; +defvar xsai_mi32_t = riscv_xsai_zames_tile_i32; +defvar xsai_msync_t = riscv_xsai_zmasync_sync; +defvar xsai_mf8e4_t = riscv_xsai_zames_tile_fp8e4; +defvar xsai_mf8e5_t = riscv_xsai_zames_tile_fp8e5; +defvar xsai_mf16_t = riscv_xsai_zames_tile_f16; +defvar xsai_mbf16_t = riscv_xsai_zames_tile_bf16; +defvar xsai_mf32_t = riscv_xsai_zames_tile_f32; + +// Tile Register class (tr0-tr3) +// Note: Size is set to 256 as a placeholder; actual size depends on MLEN +def XSAI_ZAMES_TileReg + : RegisterClass<"RISCV", [xsai_mi8_t, xsai_mf8e4_t, xsai_mf8e5_t, xsai_mf16_t, xsai_mbf16_t], 256, + (add XSAI_ZAMES_TR0, XSAI_ZAMES_TR1, XSAI_ZAMES_TR2, XSAI_ZAMES_TR3)> { + let Size = 256; // Placeholder: actual MLEN is hardware-defined +} + +// Accumulation Register class (acc0-acc3) +def XSAI_ZAMES_AccReg + : RegisterClass<"RISCV", [xsai_mi32_t, xsai_mf32_t], 1024, + (add XSAI_ZAMES_ACC0, XSAI_ZAMES_ACC1, XSAI_ZAMES_ACC2, XSAI_ZAMES_ACC3)> { + let Size = 1024; // Placeholder: actual MLEN×AMUL is hardware-defined +} + +// Union of XSAI_ZAMES_TileReg and XSAI_ZAMES_AccReg used only as a machine-instruction +// operand class for real instructions whose encoding accepts both register +// files (e.g. mzero). Virtual registers are still allocated through +// XSAI_ZAMES_TileReg or XSAI_ZAMES_AccReg according to their MVT; this class never +// participates in register allocation. +def XSAI_ZAMES_TileOrAccReg : RegisterClass<"RISCV", + [xsai_mi8_t, xsai_mf8e4_t, xsai_mf8e5_t, xsai_mi32_t, xsai_mf16_t, + xsai_mbf16_t, xsai_mf32_t], 1024, + (add XSAI_ZAMES_TileReg, XSAI_ZAMES_AccReg)> { + let Size = 1024; // Placeholder, matches XSAI_ZAMES_AccReg +} + +// Sync register class used by ZMASYNC instructions. +def XSAI_ZMASYNC_SyncR : RegisterClass<"RISCV", [xsai_msync_t], 64, + (add (sequence "XSAI_ZMASYNC_SYNC%u", 0, 15))>; diff --git a/llvm/test/Assembler/riscv-xsai-type-properties.ll b/llvm/test/Assembler/riscv-xsai-type-properties.ll new file mode 100644 index 00000000000000..5d2760b045c51b --- /dev/null +++ b/llvm/test/Assembler/riscv-xsai-type-properties.ll @@ -0,0 +1,132 @@ +; RUN: split-file %s %t +; RUN: not llvm-as %t/tile-alloca.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-ALLOCA +; RUN: not llvm-as %t/sync-alloca.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-ALLOCA +; RUN: not llvm-as %t/tile-byval.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-BYVAL +; RUN: not llvm-as %t/sync-byval.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-BYVAL +; RUN: not llvm-as %t/tile-zero.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-ZERO +; RUN: not llvm-as %t/sync-zero.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-ZERO +; RUN: not llvm-as %t/tile-load.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-LOAD +; RUN: not llvm-as %t/sync-load.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-LOAD +; RUN: not llvm-as %t/tile-store.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-STORE +; RUN: not llvm-as %t/sync-store.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-STORE +; RUN: not llvm-as %t/tile-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-GEP +; RUN: not llvm-as %t/sync-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-GEP +; RUN: not llvm-as %t/tile-constant-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=TILE-CONSTANT-GEP +; RUN: not llvm-as %t/sync-constant-gep.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SYNC-CONSTANT-GEP +; RUN: opt -passes=verify -disable-output %t/ssa.ll +; +; Native matrix and synchronization values are unsized register resources. +; Ordinary memory objects and zero constants are invalid; SSA carriers remain valid. + +;--- tile-alloca.ll +; TILE-ALLOCA: Cannot allocate unsized type +define void @tile_alloca() { + %value = alloca target("riscv.xsai.zames.tile.int8") + ret void +} + +;--- sync-alloca.ll +; SYNC-ALLOCA: Cannot allocate unsized type +define void @sync_alloca() { + %value = alloca target("riscv.xsai.zmasync.sync") + ret void +} + +;--- tile-byval.ll +; TILE-BYVAL: Attribute 'byval' does not support unsized types! +declare void @tile_byval(ptr byval(target("riscv.xsai.zames.tile.int32"))) + +;--- sync-byval.ll +; SYNC-BYVAL: Attribute 'byval' does not support unsized types! +declare void @sync_byval(ptr byval(target("riscv.xsai.zmasync.sync"))) + +;--- tile-zero.ll +; TILE-ZERO: error: invalid type for null constant +define target("riscv.xsai.zames.tile.fp32") @tile_zero() { + ret target("riscv.xsai.zames.tile.fp32") zeroinitializer +} + +;--- sync-zero.ll +; SYNC-ZERO: error: invalid type for null constant +define target("riscv.xsai.zmasync.sync") @sync_zero() { + ret target("riscv.xsai.zmasync.sync") zeroinitializer +} + +;--- tile-load.ll +; TILE-LOAD: loading unsized types is not allowed +define target("riscv.xsai.zames.tile.int8") @tile_load(ptr %p) { + %value = load target("riscv.xsai.zames.tile.int8"), ptr %p, align 1 + ret target("riscv.xsai.zames.tile.int8") %value +} + +;--- sync-load.ll +; SYNC-LOAD: loading unsized types is not allowed +define target("riscv.xsai.zmasync.sync") @sync_load(ptr %p) { + %value = load target("riscv.xsai.zmasync.sync"), ptr %p, align 1 + ret target("riscv.xsai.zmasync.sync") %value +} + +;--- tile-store.ll +; TILE-STORE: storing unsized types is not allowed +define void @tile_store(ptr %p, target("riscv.xsai.zames.tile.fp16") %value) { + store target("riscv.xsai.zames.tile.fp16") %value, ptr %p, align 1 + ret void +} + +;--- sync-store.ll +; SYNC-STORE: storing unsized types is not allowed +define void @sync_store(ptr %p, target("riscv.xsai.zmasync.sync") %value) { + store target("riscv.xsai.zmasync.sync") %value, ptr %p, align 1 + ret void +} + +;--- tile-gep.ll +; TILE-GEP: base element of getelementptr must be sized +define ptr @tile_gep(ptr %p) { + %next = getelementptr target("riscv.xsai.zames.tile.int32"), ptr %p, i64 1 + ret ptr %next +} + +;--- sync-gep.ll +; SYNC-GEP: base element of getelementptr must be sized +define ptr @sync_gep(ptr %p) { + %next = getelementptr target("riscv.xsai.zmasync.sync"), ptr %p, i64 1 + ret ptr %next +} + +;--- tile-constant-gep.ll +; TILE-CONSTANT-GEP: base element of getelementptr must be sized +define ptr @tile_constant_gep() { + ret ptr getelementptr (target("riscv.xsai.zames.tile.bf16"), ptr null, i64 1) +} + +;--- sync-constant-gep.ll +; SYNC-CONSTANT-GEP: base element of getelementptr must be sized +define ptr @sync_constant_gep() { + ret ptr getelementptr (target("riscv.xsai.zmasync.sync"), ptr null, i64 1) +} + +;--- ssa.ll +define target("riscv.xsai.zames.tile.int8") @tile_ssa(i1 %condition, target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b) { + br i1 %condition, label %left, label %right +left: + br label %merge +right: + br label %merge +merge: + %value = phi target("riscv.xsai.zames.tile.int8") [ %a, %left ], [ %b, %right ] + %result = select i1 %condition, target("riscv.xsai.zames.tile.int8") %value, target("riscv.xsai.zames.tile.int8") %b + ret target("riscv.xsai.zames.tile.int8") %result +} + +define target("riscv.xsai.zmasync.sync") @sync_ssa(i1 %condition, target("riscv.xsai.zmasync.sync") %a, target("riscv.xsai.zmasync.sync") %b) { + br i1 %condition, label %left, label %right +left: + br label %merge +right: + br label %merge +merge: + %value = phi target("riscv.xsai.zmasync.sync") [ %a, %left ], [ %b, %right ] + %result = select i1 %condition, target("riscv.xsai.zmasync.sync") %value, target("riscv.xsai.zmasync.sync") %b + ret target("riscv.xsai.zmasync.sync") %result +} diff --git a/llvm/test/Assembler/riscv-zames-intrinsics.ll b/llvm/test/Assembler/riscv-zames-intrinsics.ll new file mode 100644 index 00000000000000..9fc0eeffc31245 --- /dev/null +++ b/llvm/test/Assembler/riscv-zames-intrinsics.ll @@ -0,0 +1,130 @@ +; RUN: llvm-as %s -o %t.bc +; RUN: llvm-dis %t.bc -o - | FileCheck %s +; RUN: opt -passes=verify -disable-output %s + +; Round-trip every ZAMES/ZMASYNC intrinsic and every target extension type +; through the standard LLVM assembly/disassembly path. + +; --- Non-overloaded configuration intrinsics ---------------------------- + +; CHECK-LABEL: define void @config( +; CHECK: call void @llvm.riscv.xsai.zames.msettilemi(i64 4) +; CHECK: call void @llvm.riscv.xsai.zames.msettileni(i64 8) +; CHECK: call void @llvm.riscv.xsai.zames.msettileki(i64 16) +; CHECK: call void @llvm.riscv.xsai.zames.msettilem(i64 %m) +; CHECK: call void @llvm.riscv.xsai.zames.msettilen(i64 %n) +; CHECK: call void @llvm.riscv.xsai.zames.msettilek(i64 %k) +; CHECK: call void @llvm.riscv.xsai.zames.msetcfg(i64 4, i64 %payload) +; CHECK: %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 4) +; CHECK: call void @llvm.riscv.xsai.zames.minit() +define void @config(i64 %m, i64 %n, i64 %k, i64 %payload) { + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 8) + call void @llvm.riscv.xsai.zames.msettileki(i64 16) + call void @llvm.riscv.xsai.zames.msettilem(i64 %m) + call void @llvm.riscv.xsai.zames.msettilen(i64 %n) + call void @llvm.riscv.xsai.zames.msettilek(i64 %k) + call void @llvm.riscv.xsai.zames.msetcfg(i64 4, i64 %payload) + %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 4) + call void @llvm.riscv.xsai.zames.minit() + ret void +} + +; --- Overloaded load/store on int8 tiles -------------------------------- + +; CHECK-LABEL: define void @load_store_i8( +; CHECK: %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) +; CHECK: %at = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) +; CHECK: %aw = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %pa) +; CHECK: call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %at, ptr %pa, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %aw, ptr %pa) +define void @load_store_i8(ptr %pa, i64 %s) { + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) + %at = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %pa, i64 %s) + %aw = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %pa) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %pa, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %at, ptr %pa, i64 %s) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %aw, ptr %pa) + ret void +} + +; --- B/C tile class + transposed/whole on int32 accumulator ------------- + +; CHECK-LABEL: define void @load_store_bc( +; CHECK: %b = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bt = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) +; CHECK: %bw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int32t(ptr %pb) +; CHECK: %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) +; CHECK: %ct = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) +; CHECK: %cw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %pc) +; CHECK: call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %b, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bt, ptr %pb, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bw, ptr %pb) +; CHECK: call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %ct, ptr %pc, i64 %s) +; CHECK: call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %cw, ptr %pc) +define void @load_store_bc(ptr %pb, ptr %pc, i64 %s) { + %b = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) + %bt = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int32t(ptr %pb, i64 %s) + %bw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int32t(ptr %pb) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) + %ct = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %pc, i64 %s) + %cw = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %pc) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %b, ptr %pb, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bt, ptr %pb, i64 %s) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %bw, ptr %pb) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %pc, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %ct, ptr %pc, i64 %s) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %cw, ptr %pc) + ret void +} + +; --- mzero / mmacc ------------------------------------------------------ + +; CHECK-LABEL: define target("riscv.xsai.zames.tile.int32") @zero_acc() { +; CHECK: %z = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +; CHECK: ret target("riscv.xsai.zames.tile.int32") %z +define target("riscv.xsai.zames.tile.int32") @zero_acc() { + %z = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + ret target("riscv.xsai.zames.tile.int32") %z +} + +; CHECK-LABEL: define target("riscv.xsai.zames.tile.int32") @matmul( +; CHECK: %r = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b, target("riscv.xsai.zames.tile.int32") %c) +define target("riscv.xsai.zames.tile.int32") @matmul(target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b, target("riscv.xsai.zames.tile.int32") %c) { + %r = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, target("riscv.xsai.zames.tile.int8") %b, target("riscv.xsai.zames.tile.int32") %c) + ret target("riscv.xsai.zames.tile.int32") %r +} + +; --- Every floating tile type parses and prints ------------------------- + +; CHECK-LABEL: define void @fp_types( +; CHECK: call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr %p, i64 %s) +; CHECK: call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp32t(ptr %p, i64 %s) +define void @fp_types(ptr %p, i64 %s) { + %e4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr %p, i64 %s) + %e5 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr %p, i64 %s) + %f16 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr %p, i64 %s) + %bf16 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr %p, i64 %s) + %f32 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp32t(ptr %p, i64 %s) + ret void +} + +; --- ZMASYNC ------------------------------------------------------------ + +; CHECK-LABEL: define void @sync_ops( +; CHECK: %s0 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.zmasync.synct() +; CHECK: %s1 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s0) +; CHECK: call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s1, i64 %sem) +; CHECK: call void @llvm.riscv.xsai.zmasync.mfence() +define void @sync_ops(i64 %sem) { + %s0 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset.triscv.xsai.zmasync.synct() + %s1 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s0) + call void @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s1, i64 %sem) + call void @llvm.riscv.xsai.zmasync.mfence() + ret void +} diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll index c7042ce2431968..917cae82bc871d 100644 --- a/llvm/test/CodeGen/RISCV/features-info.ll +++ b/llvm/test/CodeGen/RISCV/features-info.ll @@ -349,6 +349,7 @@ ; CHECK-NEXT: zalasr - 'Zalasr' (Load-Acquire and Store-Release Instructions). ; CHECK-NEXT: zalrsc - 'Zalrsc' (Load-Reserved/Store-Conditional). ; CHECK-NEXT: zama16b - 'Zama16b' (Atomic 16-byte misaligned loads, stores and AMOs). +; CHECK: zames ; CHECK-NEXT: zawrs - 'Zawrs' (Wait on Reservation Set). ; CHECK-NEXT: zba - 'Zba' (Address Generation Instructions). ; CHECK-NEXT: zbb - 'Zbb' (Basic Bit-Manipulation). @@ -405,6 +406,7 @@ ; CHECK-NEXT: zksed - 'Zksed' (ShangMi Suite: SM4 Block Cipher Instructions). ; CHECK-NEXT: zksh - 'Zksh' (ShangMi Suite: SM3 Hash Function Instructions). ; CHECK-NEXT: zkt - 'Zkt' (Data Independent Execution Latency). +; CHECK: zmasync ; CHECK-NEXT: zmmul - 'Zmmul' (Integer Multiplication). ; CHECK-NEXT: ztso - 'Ztso' (Memory Model - Total Store Order). ; CHECK-NEXT: zvbb - 'Zvbb' (Vector basic bit-manipulation instructions). diff --git a/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll new file mode 100644 index 00000000000000..1984917312b34e --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/rv32-unsupported.ll @@ -0,0 +1,26 @@ +; RV32 is not a supported ZAMES/ZMASYNC code generation target: the +; intrinsic scalar interfaces are i64 and the SelectionDAG patterns and +; pseudos are constrained to RV64. llc must reject RV32 with a stable +; diagnostic before instruction selection, without "Cannot select", a +; stack trace, or a crash. +; +; RUN: not llc -mtriple=riscv32 -mattr=+zames,+zmasync %s -o /dev/null 2>&1 | FileCheck %s + +; CHECK-NOT: Cannot select +; CHECK: error: {{.*}}ZAMES/ZMASYNC code generation requires RV64 +; CHECK-NOT: Cannot select + +define target("riscv.xsai.zames.tile.int8") @rv32_zames(ptr %p, i64 %stride) { + %t = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %p, i64 %stride) + ret target("riscv.xsai.zames.tile.int8") %t +} + +define void @rv32_zmasync() { + %s = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() + call void @llvm.riscv.xsai.zmasync.mfence() + ret void +} + +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() +declare void @llvm.riscv.xsai.zmasync.mfence() diff --git a/llvm/test/CodeGen/RISCV/zames/zames-attributes.ll b/llvm/test/CodeGen/RISCV/zames/zames-attributes.ll new file mode 100644 index 00000000000000..0c78edf4b78db8 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-attributes.ll @@ -0,0 +1,16 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zmasync < %s \ +; RUN: | FileCheck %s --check-prefix=ASM +; RUN: llc -mtriple=riscv64 -mattr=+zmasync -filetype=obj -o %t.o < %s +; RUN: llvm-readelf -A %t.o | FileCheck %s --check-prefix=OBJ + +; Pins the exact ELF arch-attribute string for ZMASYNC: +zmasync implies +; +zames and the canonical ordering lists zmasync before zames. + +; ASM: .attribute 5, "rv64i2p1_zmasync1p0_zames1p0" + +; OBJ: TagName: arch +; OBJ-NEXT: Value: rv64i2p1_zmasync1p0_zames1p0 + +define void @empty() { + ret void +} diff --git a/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll new file mode 100644 index 00000000000000..e2bcdfe7c91621 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-codegen.ll @@ -0,0 +1,266 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames -verify-machineinstrs < %s | FileCheck %s --implicit-check-not=llvm.riscv.buddy +; RUN: llc -mtriple=riscv64 -mattr=+zames -filetype=obj %s -o %t.o +; RUN: llvm-objdump --mattr=+zames -d %t.o | FileCheck %s --check-prefix=CHECK-OBJ + +; This file intentionally keeps every matrix value in one basic block with +; short live ranges. Matrix-register copy, spill, reload, frame layout, and ABI +; handling are outside the current backend contract. + +define void @zames_settile_immediates() { +; CHECK-LABEL: zames_settile_immediates: +; CHECK: msettilemi 4 +; CHECK-NEXT: msettileni 8 +; CHECK-NEXT: msettileki 16 +; CHECK-NEXT: ret + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 8) + call void @llvm.riscv.xsai.zames.msettileki(i64 16) + ret void +} + +; Encoding boundary of the uimm10 tile field and the 3-bit config selector: +; the inclusive hardware endpoints must still select. tile=0 is an LLVM +; encoding-boundary check only; the ZAMES dialect never emits zero sizes. +define void @zames_settile_immediate_boundaries() { +; CHECK-LABEL: zames_settile_immediate_boundaries: +; CHECK: msettilemi 0 +; CHECK-NEXT: msettileni 1023 +; CHECK-NEXT: msettileki 0 +; CHECK-NEXT: ret + call void @llvm.riscv.xsai.zames.msettilemi(i64 0) + call void @llvm.riscv.xsai.zames.msettileni(i64 1023) + call void @llvm.riscv.xsai.zames.msettileki(i64 0) + ret void +} + +define void @zames_settile_registers(i64 %m, i64 %n, i64 %k) { +; CHECK-LABEL: zames_settile_registers: +; CHECK: msettilem a0 +; CHECK-NEXT: msettilen a1 +; CHECK-NEXT: msettilek a2 +; CHECK-NEXT: ret +; CHECK-OBJ-LABEL: : +; CHECK-OBJ: msettilem a0 +; CHECK-OBJ-NEXT: msettilen a1 +; CHECK-OBJ-NEXT: msettilek a2 + call void @llvm.riscv.xsai.zames.msettilem(i64 %m) + call void @llvm.riscv.xsai.zames.msettilen(i64 %n) + call void @llvm.riscv.xsai.zames.msettilek(i64 %k) + ret void +} + +define void @zames_minit() { +; CHECK-LABEL: zames_minit: +; CHECK: # %bb.0: +; CHECK-NEXT: minit +; CHECK-NEXT: ret +; CHECK-OBJ-LABEL: : +; CHECK-OBJ: minit{{$}} + call void @llvm.riscv.xsai.zames.minit() + ret void +} + +define i64 @zames_cfg_intrinsics(i64 %value) { +; CHECK-LABEL: zames_cfg_intrinsics: +; CHECK: msetcfg mcfg3, a0 +; CHECK-NEXT: mgetcfg a0, mcfg3 +; CHECK-NEXT: ret + call void @llvm.riscv.xsai.zames.msetcfg(i64 3, i64 %value) + %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 3) + ret i64 %cfg +} + +define i64 @zames_cfg_immediate_boundaries(i64 %value) { +; CHECK-LABEL: zames_cfg_immediate_boundaries: +; CHECK: msetcfg mcfg0, a0 +; CHECK-NEXT: mgetcfg a0, mcfg7 +; CHECK-NEXT: ret + call void @llvm.riscv.xsai.zames.msetcfg(i64 0, i64 %value) + %cfg = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 7) + ret i64 %cfg +} + +; Check matrix register/configuration relationships, not GPR materialization +; order or a particular physical register allocation. +define void @zames_straight_line(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %stride_b, i64 %stride_c) { +; CHECK-LABEL: zames_straight_line: +; CHECK: msettilemi 4 +; CHECK: msettileni 4 +; CHECK: msettileki 8 +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mla tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlbt tr[[#B]], (a1), a4 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mzero acc[[#C-4]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: mmacc acc[[#C-4]], tr[[#B]], tr[[#A]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msc acc[[#C-4]], (a2), a5 +; CHECK: ret + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 4) + call void @llvm.riscv.xsai.zames.msettileki(i64 8) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_mlb_straight_line(ptr %a, ptr %b, ptr %c, + i64 %stride_a, i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: zames_mlb_straight_line: +; CHECK: msettilemi 4 +; CHECK: msettileni 4 +; CHECK: msettileki 8 +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mla tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlb tr[[#B]], (a1), a4 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mzero acc[[#C-4]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: mmacc acc[[#C-4]], tr[[#B]], tr[[#A]] +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msc acc[[#C-4]], (a2), a5 +; CHECK: ret + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 4) + call void @llvm.riscv.xsai.zames.msettileki(i64 8) + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_mlc_straight_line(ptr %c, i64 %stride_c) { +; CHECK-LABEL: zames_mlc_straight_line: +; CHECK: msettilemi 4 +; CHECK: msettileni 4 +; CHECK: msettileki 8 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlc acc[[#C-4]], (a0), a1 +; CHECK: ret + call void @llvm.riscv.xsai.zames.msettilemi(i64 4) + call void @llvm.riscv.xsai.zames.msettileni(i64 4) + call void @llvm.riscv.xsai.zames.msettileki(i64 8) + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) + ret void +} + +define void @zames_transpose_load_store(ptr %a, ptr %c, i64 %stride_a, i64 %stride_c) { +; CHECK-LABEL: zames_transpose_load_store: +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlat tr[[#A]], (a0), a2 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlct acc[[#C-4]], (a1), a3 +; CHECK: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: msat tr[[#A]], (a0), a2 +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msct acc[[#C-4]], (a1), a3 +; CHECK: ret + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %stride_a) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_store_variants(ptr %a, ptr %b, ptr %c, i64 %stride) { +; CHECK-LABEL: zames_store_variants: +; CHECK: msetcfg mcfg[[#A:]], {{[a-z0-9]+}} +; CHECK-NEXT: mla tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlb tr[[#B]], (a1), a3 +; CHECK: msetcfg mcfg[[#C:]], {{[a-z0-9]+}} +; CHECK-NEXT: mlc acc[[#C-4]], (a2), a3 +; CHECK: msetcfg mcfg[[#A]], {{[a-z0-9]+}} +; CHECK-NEXT: msa tr[[#A]], (a0), a3 +; CHECK: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msb tr[[#B]], (a1), a3 +; CHECK: msetcfg mcfg[[#B]], {{[a-z0-9]+}} +; CHECK-NEXT: msbt tr[[#B]], (a1), a3 +; CHECK: msetcfg mcfg[[#C]], {{[a-z0-9]+}} +; CHECK-NEXT: msct acc[[#C-4]], (a2), a3 +; CHECK: ret + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %stride) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %stride) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %stride) + ret void +} + +define void @zames_whole(ptr %src, ptr %dst) { +; CHECK-LABEL: zames_whole: +; CHECK-NOT: msetcfg +; CHECK: mla.whole tr0, (a0) +; CHECK-NEXT: mlb.whole tr1, (a0) +; CHECK-NEXT: mlc.whole acc0, (a0) +; CHECK-NEXT: msa.whole tr0, (a1) +; CHECK-NEXT: msb.whole tr1, (a1) +; CHECK-NEXT: msc.whole acc0, (a1) +; CHECK-NEXT: ret +; CHECK-OBJ-LABEL: : +; CHECK-OBJ-NOT: msetcfg +; CHECK-OBJ: mla.whole tr0, (a0) +; CHECK-OBJ-NEXT: mlb.whole tr1, (a0) +; CHECK-OBJ-NEXT: mlc.whole acc0, (a0) +; CHECK-OBJ-NEXT: msa.whole tr0, (a1) +; CHECK-OBJ-NEXT: msb.whole tr1, (a1) +; CHECK-OBJ-NEXT: msc.whole acc0, (a1) + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) + ret void +} + +declare void @llvm.riscv.xsai.zames.msettilemi(i64) +declare void @llvm.riscv.xsai.zames.msettileni(i64) +declare void @llvm.riscv.xsai.zames.msettileki(i64) +declare void @llvm.riscv.xsai.zames.msettilem(i64) +declare void @llvm.riscv.xsai.zames.msettilen(i64) +declare void @llvm.riscv.xsai.zames.msettilek(i64) +declare void @llvm.riscv.xsai.zames.minit() +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) +declare i64 @llvm.riscv.xsai.zames.mgetcfg(i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir b/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir new file mode 100644 index 00000000000000..8a8efbc92810b4 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-expand.mir @@ -0,0 +1,54 @@ +# RUN: llc -mtriple=riscv64 -mattr=+zames \ +# RUN: -run-pass=riscv-expand-pseudo-pre-emit -o - %s | FileCheck %s + +# Late expansion must derive the msetcfg selector from the physical matrix +# register on the pseudo operand: tr0-tr3 -> 0-3, acc0-acc3 -> 4-7. It must +# also preserve the machine operand order C, B, A on the final mmacc. +# These relationships cannot be checked in the .ll tests because register +# allocation picks the registers there. +# Matrix operands are live on entry to this post-RA pass fixture. + +--- | + define void @zames_mir_cfg() { ret void } +... +--- +name: zames_mir_cfg +body: | + bb.0: + liveins: $xsai_zames_acc2, $xsai_zames_tr3, $xsai_zames_tr1 + + ; f16 x f16 -> f32 with C = acc2, B = tr3, A = tr1 + $x10 = ADDI $x0, 12 + $x11 = ADDI $x0, 9 + ; CHECK: XSAI_ZAMES_MSETCFG 6, {{[^:]*}}$x10{{$}} + ; CHECK-NEXT: XSAI_ZAMES_MSETCFG 3, {{[^:]*}}$x11{{$}} + ; CHECK-NEXT: XSAI_ZAMES_MSETCFG 1, {{[^:]*}}$x11{{$}} + ; CHECK-NEXT: $xsai_zames_acc2 = XSAI_ZAMES_MMACC {{[^:]*}}$xsai_zames_acc2, {{[^:]*}}$xsai_zames_tr3, {{[^:]*}}$xsai_zames_tr1{{$}} + $xsai_zames_acc2 = PSEUDO_XSAI_ZAMES_MMACC_F16_F32 $xsai_zames_acc2, $xsai_zames_tr3, $xsai_zames_tr1, $x10, $x11, $x11 + + ; fp16 tile load into tr2 + $x12 = ADDI $x0, 9 + ; CHECK: XSAI_ZAMES_MSETCFG 2, {{[^:]*}}$x12{{$}} + ; CHECK-NEXT: $xsai_zames_tr2 = XSAI_ZAMES_MLA + $xsai_zames_tr2 = PSEUDO_XSAI_ZAMES_MLA_A_F16 $x10, $x11, $x12 + + ; bf16 tile store from tr3 + $x13 = ADDI $x0, 10 + ; CHECK: XSAI_ZAMES_MSETCFG 3, {{[^:]*}}$x13{{$}} + ; CHECK-NEXT: XSAI_ZAMES_MSA + PSEUDO_XSAI_ZAMES_MSA_A_BF16 $xsai_zames_tr3, $x10, $x11, $x13 + + ; mzero into an fp16 tile register + $x14 = ADDI $x0, 9 + ; CHECK: XSAI_ZAMES_MSETCFG 1, {{[^:]*}}$x14{{$}} + ; CHECK-NEXT: $xsai_zames_tr1 = XSAI_ZAMES_MZERO{{$}} + $xsai_zames_tr1 = PSEUDO_XSAI_ZAMES_MZERO_T_F16 $x14 + + ; mzero into accumulator acc2 + $x15 = ADDI $x0, 12 + ; CHECK: XSAI_ZAMES_MSETCFG 6, {{[^:]*}}$x15{{$}} + ; CHECK-NEXT: $xsai_zames_acc2 = XSAI_ZAMES_MZERO{{$}} + $xsai_zames_acc2 = PSEUDO_XSAI_ZAMES_MZERO_C_F32 $x15 + + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll new file mode 100644 index 00000000000000..2042d37e40a40f --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative-acc.ll @@ -0,0 +1,21 @@ +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames %s -o /dev/null 2>&1 \ +; RUN: | FileCheck %s + +; FP8 inputs only accumulate into fp32. An i32 accumulator has no mmacc +; pattern and must fail selection explicitly. + +; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.xsai.zames.mmacc + +define void @bad_mmacc_acc(ptr %src, ptr %dst) { + %a4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + %b4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %dst) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + %r = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %a4, target("riscv.xsai.zames.tile.fp8e4") %b4, target("riscv.xsai.zames.tile.int32") %c) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %r, ptr %dst) + ret void +} + +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll new file mode 100644 index 00000000000000..a6d2bab08daf0d --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types-negative.ll @@ -0,0 +1,22 @@ +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames %s -o /dev/null 2>&1 \ +; RUN: | FileCheck %s + +; Mixed input element types have no mmacc pattern: fp8e4 x fp8e5 is not a +; supported combination and must fail selection explicitly. + +; CHECK: LLVM ERROR: Cannot select: intrinsic %llvm.riscv.xsai.zames.mmacc + +define void @bad_mmacc(ptr %src, ptr %dst) { + %a4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + %a5 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src) + %c = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e4") %a4, target("riscv.xsai.zames.tile.fp8e5") %a5, target("riscv.xsai.zames.tile.fp32") %c) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r, ptr %dst) + ret void +} + +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp8e5"), target("riscv.xsai.zames.tile.fp32")) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll new file mode 100644 index 00000000000000..22f31c35832917 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-fp-types.ll @@ -0,0 +1,621 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=finalize-isel -o - %s \ +; RUN: | FileCheck %s --check-prefix=ISEL +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-ra -o - %s \ +; RUN: | FileCheck %s --check-prefix=PRERA +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-emit -o - %s \ +; RUN: | FileCheck %s --check-prefix=POSTRA +; RUN: llc -mtriple=riscv64 -mattr=+zames -filetype=obj %s -o %t.o +; RUN: llvm-objdump --mattr=+zames -d %t.o \ +; RUN: | FileCheck %s --check-prefix=OBJ + +; FP8E4/FP8E5/FP16/BF16 are input tile types; FP32 is an accumulator type. +; Typed loads, stores, mzero and mmacc expand through the two-level pseudo +; chain with the type payload from the spec (i8=0x02, i32=0x04, +; fp8e5m2=0x07, fp8e4m3=0x08, fp16=0x09, bf16=0x0a, fp32=0x0c). Pre-RA, +; payload materialization immediately precedes its typed pseudo; post-RA, +; msetcfg immediately precedes the real instruction. + +define void @zames_fp_ab_ls(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_fp_ab_ls +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5_PRE +; PRERA-LABEL: name: zames_fp_ab_ls +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E4 {{.*}}, %[[C8]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_FP8E5 {{.*}}, %[[C7]] :: +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_FP8E5 {{.*}}, %[[C7]] :: +; POSTRA-LABEL: name: zames_fp_ab_ls +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 7 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size +; OBJ-LABEL: : +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} + %t1 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e4t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e5t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %t8, ptr %dst, i64 %s) + ret void +} + +define void @zames_fp_c_ls(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_fp_c_ls +; ISEL: PSEUDO_XSAI_ZAMES_MLC_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSC_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLCT_C_F32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSCT_C_F32_PRE +; PRERA-LABEL: name: zames_fp_c_ls +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLC_C_F32 {{.*}}, %[[C12]] :: +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_F32 {{.*}}, %[[C12]] :: +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLCT_C_F32 {{.*}}, %[[C12]] :: +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSCT_C_F32 {{.*}}, %[[C12]] :: +; POSTRA-LABEL: name: zames_fp_c_ls +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLC {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSC {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLCT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 12 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSCT {{.*}}:: (store unknown-size +; OBJ-LABEL: : +; OBJ: mlc{{[ \t]}} +; OBJ: msc{{[ \t]}} +; OBJ: mlct{{[ \t]}} +; OBJ: msct{{[ \t]}} + %c5 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %c5, ptr %dst, i64 %s) + %c6 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.fp32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %c6, ptr %dst, i64 %s) + ret void +} + +define void @zames_fp16_bf16_ab_ls(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_fp16_bf16_ab_ls +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_BF16_PRE +; PRERA-LABEL: name: zames_fp16_bf16_ab_ls +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_F16 {{.*}}, %[[C9]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_BF16 {{.*}}, %[[C10]] :: +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_BF16 {{.*}}, %[[C10]] :: +; POSTRA-LABEL: name: zames_fp16_bf16_ab_ls +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 9 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: {{.*}} = XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: [[P:\$x[0-9]+]] = ADDI $x0, 10 +; POSTRA: XSAI_ZAMES_MSETCFG {{[0-9]+}}, {{.*}}[[P]]{{$}} +; POSTRA-NEXT: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size +; OBJ-LABEL: : +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} +; OBJ: mla{{[ \t]}} +; OBJ: msa{{[ \t]}} +; OBJ: mlat{{[ \t]}} +; OBJ: msat{{[ \t]}} +; OBJ: mlb{{[ \t]}} +; OBJ: msb{{[ \t]}} +; OBJ: mlbt{{[ \t]}} +; OBJ: msbt{{[ \t]}} + %t1 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t1, ptr %dst, i64 %s) + %t2 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t2, ptr %dst, i64 %s) + %t3 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t3, ptr %dst, i64 %s) + %t4 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %t4, ptr %dst, i64 %s) + %t5 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t5, ptr %dst, i64 %s) + %t6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t6, ptr %dst, i64 %s) + %t7 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t7, ptr %dst, i64 %s) + %t8 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.bf16t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %t8, ptr %dst, i64 %s) + ret void +} + +; Whole loads and stores select the real instruction directly; the transfer +; does not read the type config, so no msetcfg is emitted for them. +define void @zames_fp_whole(ptr %src, ptr %dst) { +; ISEL-LABEL: name: zames_fp_whole +; ISEL: XSAI_ZAMES_MLA_WHOLE +; ISEL: XSAI_ZAMES_MSA_WHOLE +; ISEL: XSAI_ZAMES_MLB_WHOLE +; ISEL: XSAI_ZAMES_MSB_WHOLE +; ISEL: XSAI_ZAMES_MLA_WHOLE +; ISEL: XSAI_ZAMES_MSA_WHOLE +; ISEL: XSAI_ZAMES_MLB_WHOLE +; ISEL: XSAI_ZAMES_MSB_WHOLE +; ISEL: XSAI_ZAMES_MLC_WHOLE +; ISEL: XSAI_ZAMES_MSC_WHOLE +; OBJ-LABEL: : +; OBJ-NOT: msetcfg +; OBJ: mla.whole{{[ \t]}} +; OBJ: msa.whole{{[ \t]}} +; OBJ: mlb.whole{{[ \t]}} +; OBJ: msb.whole{{[ \t]}} +; OBJ: mla.whole{{[ \t]}} +; OBJ: msa.whole{{[ \t]}} +; OBJ: mlb.whole{{[ \t]}} +; OBJ: msb.whole{{[ \t]}} +; OBJ: mlc.whole{{[ \t]}} +; OBJ: msc.whole{{[ \t]}} + %a = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %a, ptr %dst) + %b = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %b, ptr %dst) + %a16 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp16t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %a16, ptr %dst) + %bb16 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.bf16t(ptr %src) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %bb16, ptr %dst) + %c = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %c, ptr %dst) + ret void +} + +; mzero covers all five input tile types and both accumulator types, each +; bound to its payload config and register class. +define void @zames_fp_mzero(ptr %dst) { +; ISEL-LABEL: name: zames_fp_mzero +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_I8_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_F16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_T_BF16_PRE +; ISEL: PSEUDO_XSAI_ZAMES_MZERO_C_F32_PRE +; PRERA-LABEL: name: zames_fp_mzero +; PRERA: %[[C2:[0-9]+]]:gpr = ADDI $x0, 2 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_I8 %[[C2]]{{$}} +; PRERA: %[[C8:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_FP8E4 %[[C8]]{{$}} +; PRERA: %[[C7:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_FP8E5 %[[C7]]{{$}} +; PRERA: %[[C4:[0-9]+]]:gpr = ADDI $x0, 4 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_I32 %[[C4]]{{$}} +; PRERA: %[[C9:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_F16 %[[C9]]{{$}} +; PRERA: %[[C10:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MZERO_T_BF16 %[[C10]]{{$}} +; PRERA: %[[C12:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_F32 %[[C12]]{{$}} +; POSTRA-LABEL: name: zames_fp_mzero +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#T:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_tr[[#T]] = XSAI_ZAMES_MZERO{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MZERO{{$}} +; OBJ-LABEL: : +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}acc{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}tr{{[0-3]}} +; OBJ: mzero{{[ \t]}}acc{{[0-3]}} + %z1 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int8t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %z1, ptr %dst) + %z2 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e4t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %z2, ptr %dst) + %z3 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e5t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %z3, ptr %dst) + %z4 = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %z4, ptr %dst) + %z5 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp16t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %z5, ptr %dst) + %z6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.bf16t() + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %z6, ptr %dst) + %z7 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp32t() + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %z7, ptr %dst) + ret void +} + +; mmacc covers the six supported element combinations. Distinct A/B pointers +; and load opcodes keep the required machine operand order C, B, A observable. +; Pre-RA materializes the accumulator and input payloads; late expansion emits +; one msetcfg per physical matrix operand. +define void @zames_fp_mmacc(ptr %src, ptr %src2, ptr %dst) { +; ISEL-LABEL: name: zames_fp_mmacc +; ISEL: %[[A1:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B1:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C1:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE %[[C1]], {{.*}}%[[B1]], {{.*}}%[[A1]]{{$}} +; ISEL: %[[A2:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B2:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C2:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_I8_F32_PRE %[[C2]], {{.*}}%[[B2]], {{.*}}%[[A2]]{{$}} +; ISEL: %[[A3:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B3:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C3:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32_PRE %[[C3]], {{.*}}%[[B3]], {{.*}}%[[A3]]{{$}} +; ISEL: %[[A4:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B4:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C4:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32_PRE %[[C4]], {{.*}}%[[B4]], {{.*}}%[[A4]]{{$}} +; ISEL: %[[A5:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B5:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C5:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_F16_F32_PRE %[[C5]], {{.*}}%[[B5]], {{.*}}%[[A5]]{{$}} +; ISEL: %[[A6:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLA_WHOLE +; ISEL: %[[B6:[0-9]+]]:xsai_zames_tilereg = XSAI_ZAMES_MLB_WHOLE +; ISEL: %[[C6:[0-9]+]]:xsai_zames_accreg = XSAI_ZAMES_MLC_WHOLE +; ISEL: PSEUDO_XSAI_ZAMES_MMACC_BF16_F32_PRE %[[C6]], {{.*}}%[[B6]], {{.*}}%[[A6]]{{$}} +; PRERA-LABEL: name: zames_fp_mmacc +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 4 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 2 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 2 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 8 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_FP8E4_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 7 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_FP8E5_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 9 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_F16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; PRERA: %[[CC:[0-9]+]]:gpr = ADDI $x0, 12 +; PRERA-NEXT: %[[MC:[0-9]+]]:gpr = ADDI $x0, 10 +; PRERA-NEXT: %{{[0-9]+}}:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_BF16_F32 %{{[0-9]+}}, {{.*}}%{{[0-9]+}}, {{.*}}%{{[0-9]+}}, %[[CC]], %[[MC]], %[[MC]]{{$}} +; POSTRA-LABEL: name: zames_fp_mmacc +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; POSTRA: XSAI_ZAMES_MSETCFG [[#C:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#B:]], {{.*}} +; POSTRA-NEXT: XSAI_ZAMES_MSETCFG [[#A:]], {{.*}} +; POSTRA-NEXT: {{.*}}$xsai_zames_acc[[#C-4]] = XSAI_ZAMES_MMACC {{.*}}$xsai_zames_acc[[#C-4]], {{.*}}$xsai_zames_tr[[#B]], {{.*}}$xsai_zames_tr[[#A]]{{$}} +; OBJ-LABEL: : +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} +; OBJ: mmacc{{[ \t]}}acc{{[0-3]}}, tr{{[0-3]}}, tr{{[0-3]}} + %a1 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b1 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src2) + %c1 = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + %r1 = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a1, target("riscv.xsai.zames.tile.int8") %b1, target("riscv.xsai.zames.tile.int32") %c1) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %r1, ptr %dst) + %a2 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b2 = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src2) + %c2 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r2 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a2, target("riscv.xsai.zames.tile.int8") %b2, target("riscv.xsai.zames.tile.fp32") %c2) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r2, ptr %dst) + %a3 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src) + %b3 = call target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e4t(ptr %src2) + %c3 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r3 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4") %a3, target("riscv.xsai.zames.tile.fp8e4") %b3, target("riscv.xsai.zames.tile.fp32") %c3) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r3, ptr %dst) + %a4 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src) + %b4 = call target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e5t(ptr %src2) + %c4 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r4 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e5t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5") %a4, target("riscv.xsai.zames.tile.fp8e5") %b4, target("riscv.xsai.zames.tile.fp32") %c4) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r4, ptr %dst) + %a5 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp16t(ptr %src) + %b5 = call target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp16t(ptr %src2) + %c5 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r5 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp16t.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16") %a5, target("riscv.xsai.zames.tile.fp16") %b5, target("riscv.xsai.zames.tile.fp32") %c5) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r5, ptr %dst) + %a6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.bf16t(ptr %src) + %b6 = call target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.bf16t(ptr %src2) + %c6 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr %src) + %r6 = call target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.bf16t.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16") %a6, target("riscv.xsai.zames.tile.bf16") %b6, target("riscv.xsai.zames.tile.fp32") %c6) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32") %r6, ptr %dst) + ret void +} + +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e4t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp8e5t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.fp16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.bf16t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.fp32t(ptr, i64) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.fp32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr, i64) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp16t(ptr) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.bf16t(ptr) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e5t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp8e4t(ptr) +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.fp16t(ptr) +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.bf16t(ptr) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.fp32t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.fp8e5t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.fp32t(target("riscv.xsai.zames.tile.fp32"), ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int8t() +declare target("riscv.xsai.zames.tile.fp8e4") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e4t() +declare target("riscv.xsai.zames.tile.fp8e5") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp8e5t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.fp16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp16t() +declare target("riscv.xsai.zames.tile.bf16") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.bf16t() +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.fp32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e4t.triscv.xsai.zames.tile.fp8e4t(target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp8e4"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp8e5t.triscv.xsai.zames.tile.fp8e5t(target("riscv.xsai.zames.tile.fp8e5"), target("riscv.xsai.zames.tile.fp8e5"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.fp16t.triscv.xsai.zames.tile.fp16t(target("riscv.xsai.zames.tile.fp16"), target("riscv.xsai.zames.tile.fp16"), target("riscv.xsai.zames.tile.fp32")) +declare target("riscv.xsai.zames.tile.fp32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.fp32t.triscv.xsai.zames.tile.bf16t.triscv.xsai.zames.tile.bf16t(target("riscv.xsai.zames.tile.bf16"), target("riscv.xsai.zames.tile.bf16"), target("riscv.xsai.zames.tile.fp32")) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll b/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll new file mode 100644 index 00000000000000..df9b8c17844808 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-immediate-range-invalid.ll @@ -0,0 +1,55 @@ +; Defensive ISel negatives: with the IR verifier disabled, an out-of-range +; immediate must still fail selection instead of being truncated into the +; 10-bit tile field or the 3-bit config selector. + +; RUN: split-file %s %t +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/settilemi-neg.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=MI-NEG +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/settileni-1024.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=NI-1024 +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/settileki-1024.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=KI-1024 +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/msetcfg-8.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=SETCFG-8 +; RUN: not --crash llc -mtriple=riscv64 -mattr=+zames -disable-verify %t/mgetcfg-neg.ll -o /dev/null 2>&1 | FileCheck %s --check-prefix=GETCFG-NEG + +;--- settilemi-neg.ll +declare void @llvm.riscv.xsai.zames.msettilemi(i64) + +define void @f() { +; MI-NEG: Cannot select + call void @llvm.riscv.xsai.zames.msettilemi(i64 -1) + ret void +} + +;--- settileni-1024.ll +declare void @llvm.riscv.xsai.zames.msettileni(i64) + +define void @f() { +; NI-1024: Cannot select + call void @llvm.riscv.xsai.zames.msettileni(i64 1024) + ret void +} + +;--- settileki-1024.ll +declare void @llvm.riscv.xsai.zames.msettileki(i64) + +define void @f() { +; KI-1024: Cannot select + call void @llvm.riscv.xsai.zames.msettileki(i64 1024) + ret void +} + +;--- msetcfg-8.ll +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) + +define void @f(i64 %payload) { +; SETCFG-8: Cannot select + call void @llvm.riscv.xsai.zames.msetcfg(i64 8, i64 %payload) + ret void +} + +;--- mgetcfg-neg.ll +declare i64 @llvm.riscv.xsai.zames.mgetcfg(i64) + +define i64 @f() { +; GETCFG-NEG: Cannot select + %r = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 -1) + ret i64 %r +} diff --git a/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll new file mode 100644 index 00000000000000..5065ab2fa6d3de --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-isel-pseudos.ll @@ -0,0 +1,115 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames -stop-after=finalize-isel -o - %s | FileCheck %s + +; The matrix data intrinsics must select to typed pre-RA pseudos first. cfg GPR +; payloads are materialized by riscv-expand-pseudo-pre-ra before RA. + +define void @zames_selects_typed_pseudos(ptr %a, ptr %b, ptr %c, i64 %stride_a, i64 %stride_b, i64 %stride_c) { +; CHECK-LABEL: name: zames_selects_typed_pseudos +; CHECK: PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_selects_mlb_pseudo(ptr %a, ptr %b, ptr %c, + i64 %stride_a, i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: name: zames_selects_mlb_pseudo +; CHECK: PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MZERO_C_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MMACC_I8_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_selects_mlc_pseudo(ptr %c, i64 %stride_c) { +; CHECK-LABEL: name: zames_selects_mlc_pseudo +; CHECK: PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) + ret void +} + +define void @zames_selects_transpose_and_store_pseudos(ptr %a, ptr %c, + i64 %sa, i64 %sc) { +; CHECK-LABEL: name: zames_selects_transpose_and_store_pseudos +; CHECK: PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %sc) + ret void +} + +define void @zames_selects_ab_store_pseudos(ptr %a, ptr %b, i64 %s) { +; CHECK-LABEL: name: zames_selects_ab_store_pseudos +; CHECK: PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE +; CHECK: PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) + ret void +} + +define void @zames_selects_whole_direct(ptr %src, ptr %dst) { +; CHECK-LABEL: name: zames_selects_whole_direct +; CHECK-NOT: PSEUDO +; CHECK: XSAI_ZAMES_MLA_WHOLE +; CHECK: XSAI_ZAMES_MLB_WHOLE +; CHECK: XSAI_ZAMES_MLC_WHOLE +; CHECK: XSAI_ZAMES_MSA_WHOLE +; CHECK: XSAI_ZAMES_MSB_WHOLE +; CHECK: XSAI_ZAMES_MSC_WHOLE + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) + ret void +} + +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll new file mode 100644 index 00000000000000..0470863467cc18 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-memory-operands.ll @@ -0,0 +1,127 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=finalize-isel -o - %s \ +; RUN: | FileCheck %s --check-prefix=ISEL +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-ra -o - %s \ +; RUN: | FileCheck %s --check-prefix=PRERA +; RUN: llc -mtriple=riscv64 -mattr=+zames \ +; RUN: -stop-after=riscv-expand-pseudo-pre-emit -o - %s \ +; RUN: | FileCheck %s --check-prefix=POSTRA + +; Every ZAMES load/store MachineInstr must carry a MachineMemOperand through +; all three expansion stages. Normal/transpose accesses use the conservative +; address-space fallback (no %ir value), whole accesses keep the exact base +; pointer. Load results feed matching stores so nothing is deleted as unused. + +define void @zames_mem_ab_i8(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_mem_ab_i8 +; ISEL: PSEUDO_XSAI_ZAMES_MLA_A_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLAT_A_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLB_B_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLBT_B_I8_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSA_A_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSAT_A_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSB_B_I8_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSBT_B_I8_PRE {{.*}}:: (store unknown-size +; PRERA-LABEL: name: zames_mem_ab_i8 +; PRERA: PSEUDO_XSAI_ZAMES_MLA_A_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLAT_A_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLB_B_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLBT_B_I8 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSA_A_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSAT_A_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSB_B_I8 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSBT_B_I8 {{.*}}:: (store unknown-size +; POSTRA-LABEL: name: zames_mem_ab_i8 +; POSTRA: XSAI_ZAMES_MSETCFG {{[^:]*$}} +; POSTRA: XSAI_ZAMES_MLA {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLAT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLB {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLBT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSA {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSAT {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSB {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSBT {{.*}}:: (store unknown-size + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + %tat = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + %tbt = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tat, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tbt, ptr %dst, i64 %s) + ret void +} + +define void @zames_mem_c_i32(ptr %src, ptr %dst, i64 %s) { +; ISEL-LABEL: name: zames_mem_c_i32 +; ISEL: PSEUDO_XSAI_ZAMES_MLC_C_I32_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MLCT_C_I32_PRE {{.*}}:: (load unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSC_C_I32_PRE {{.*}}:: (store unknown-size +; ISEL: PSEUDO_XSAI_ZAMES_MSCT_C_I32_PRE {{.*}}:: (store unknown-size +; PRERA-LABEL: name: zames_mem_c_i32 +; PRERA: PSEUDO_XSAI_ZAMES_MLC_C_I32 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MLCT_C_I32 {{.*}}:: (load unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSC_C_I32 {{.*}}:: (store unknown-size +; PRERA: PSEUDO_XSAI_ZAMES_MSCT_C_I32 {{.*}}:: (store unknown-size +; POSTRA-LABEL: name: zames_mem_c_i32 +; POSTRA: XSAI_ZAMES_MLC {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MLCT {{.*}}:: (load unknown-size +; POSTRA: XSAI_ZAMES_MSC {{.*}}:: (store unknown-size +; POSTRA: XSAI_ZAMES_MSCT {{.*}}:: (store unknown-size + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %src, i64 %s) + %tct = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %src, i64 %s) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %dst, i64 %s) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tct, ptr %dst, i64 %s) + ret void +} + +define void @zames_mem_whole(ptr %src, ptr %dst) { +; ISEL-LABEL: name: zames_mem_whole +; ISEL: XSAI_ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: XSAI_ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: XSAI_ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; ISEL: XSAI_ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; ISEL: XSAI_ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; ISEL: XSAI_ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA-LABEL: name: zames_mem_whole +; PRERA: XSAI_ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: XSAI_ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: XSAI_ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; PRERA: XSAI_ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA: XSAI_ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; PRERA: XSAI_ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA-LABEL: name: zames_mem_whole +; POSTRA: XSAI_ZAMES_MLA_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: XSAI_ZAMES_MLB_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: XSAI_ZAMES_MLC_WHOLE {{.*}}:: (load unknown-size from %ir.src +; POSTRA: XSAI_ZAMES_MSA_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA: XSAI_ZAMES_MSB_WHOLE {{.*}}:: (store unknown-size into %ir.dst +; POSTRA: XSAI_ZAMES_MSC_WHOLE {{.*}}:: (store unknown-size into %ir.dst + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) + ret void +} + +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll new file mode 100644 index 00000000000000..d4c9bca5e23a68 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-no-feature.ll @@ -0,0 +1,12 @@ +; RUN: not --crash llc -mtriple=riscv64 -verify-machineinstrs < %s 2>&1 | FileCheck %s + +; Without +zames, the target must not legally select ZAMES matrix intrinsics. + +define void @zames_requires_feature(ptr %c, i64 %stride) { +; CHECK: Cannot select: intrinsic %llvm.riscv.xsai.zames.msc + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t( + target("riscv.xsai.zames.tile.int32") undef, ptr %c, i64 %stride) + ret void +} + +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) diff --git a/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll new file mode 100644 index 00000000000000..65ec31f658e6a1 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zames-prera-cfg-pseudos.ll @@ -0,0 +1,140 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zames -stop-after=riscv-expand-pseudo-pre-ra -o - %s | FileCheck %s + +define void @zames_prera_materializes_cfg(ptr %a, ptr %b, ptr %c, + i64 %stride_a, i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: name: zames_prera_materializes_cfg +; CHECK: %[[CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_A]] +; CHECK: %[[CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[TILE_B:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLBT_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_B]] +; CHECK: %[[CFG_ZERO:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[ZERO:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_I32 %[[CFG_ZERO]] +; CHECK: %[[CFG_ACC:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[CFG_MAT:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %[[ZERO]], %[[TILE_B]], %[[TILE_A]], %[[CFG_ACC]], %[[CFG_MAT]], %[[CFG_MAT]] +; CHECK: %[[CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_I32 %[[ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[CFG_STORE]] + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_prera_materializes_mlb_cfg(ptr %a, ptr %b, ptr %c, + i64 %stride_a, + i64 %stride_b, + i64 %stride_c) { +; CHECK-LABEL: name: zames_prera_materializes_mlb_cfg +; CHECK: %[[MLB_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[MLB_TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_A]] +; CHECK: %[[MLB_CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[MLB_TILE_B:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_B]] +; CHECK: %[[MLB_CFG_ZERO:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[MLB_ZERO:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MZERO_C_I32 %[[MLB_CFG_ZERO]] +; CHECK: %[[MLB_CFG_ACC:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[MLB_CFG_MAT:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[MLB_ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MMACC_I8_I32 %[[MLB_ZERO]], %[[MLB_TILE_B]], %[[MLB_TILE_A]], %[[MLB_CFG_ACC]], %[[MLB_CFG_MAT]], %[[MLB_CFG_MAT]] +; CHECK: %[[MLB_CFG_STORE:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSC_C_I32 %[[MLB_ACC]], %{{[0-9]+}}, %{{[0-9]+}}, %[[MLB_CFG_STORE]] + %tile_a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %stride_a) + %tile_b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %stride_b) + %zero = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t( + target("riscv.xsai.zames.tile.int8") %tile_a, + target("riscv.xsai.zames.tile.int8") %tile_b, + target("riscv.xsai.zames.tile.int32") %zero) + call void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %acc, ptr %c, i64 %stride_c) + ret void +} + +define void @zames_prera_materializes_mlc_cfg(ptr %c, i64 %stride_c) { +; CHECK-LABEL: name: zames_prera_materializes_mlc_cfg +; CHECK: %[[MLC_CFG:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[MLC_ACC:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLC_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[MLC_CFG]] + %acc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr %c, i64 %stride_c) + ret void +} + +define void @zames_prera_transpose_store_cfg(ptr %a, ptr %c, + i64 %sa, i64 %sc) { +; CHECK-LABEL: name: zames_prera_transpose_store_cfg +; CHECK: %[[T_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[T_TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLAT_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_A]] +; CHECK: %[[T_CFG_C:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: %[[T_ACC_C:[0-9]+]]:xsai_zames_accreg = PSEUDO_XSAI_ZAMES_MLCT_C_I32 %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_C]] +; CHECK: %[[T_CFG_SA:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSAT_A_I8 %[[T_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SA]] +; CHECK: %[[T_CFG_SC:[0-9]+]]:gpr = ADDI $x0, 4 +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSCT_C_I32 %[[T_ACC_C]], %{{[0-9]+}}, %{{[0-9]+}}, %[[T_CFG_SC]] + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr %a, i64 %sa) + %tc = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr %c, i64 %sc) + call void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %sa) + call void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %tc, ptr %c, i64 %sc) + ret void +} + +define void @zames_prera_ab_store_cfg(ptr %a, ptr %b, i64 %s) { +; CHECK-LABEL: name: zames_prera_ab_store_cfg +; CHECK: %[[AB_CFG_A:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[AB_TILE_A:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLA_A_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_A]] +; CHECK: %[[AB_CFG_B:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: %[[AB_TILE_B:[0-9]+]]:xsai_zames_tilereg = PSEUDO_XSAI_ZAMES_MLB_B_I8 %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_B]] +; CHECK: %[[AB_CFG_SA:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSA_A_I8 %[[AB_TILE_A]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SA]] +; CHECK: %[[AB_CFG_SB:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSB_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_SB]] +; CHECK: %[[AB_CFG_ST:[0-9]+]]:gpr = ADDI $x0, 2 +; CHECK-NEXT: PSEUDO_XSAI_ZAMES_MSBT_B_I8 %[[AB_TILE_B]], %{{[0-9]+}}, %{{[0-9]+}}, %[[AB_CFG_ST]] + %ta = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr %a, i64 %s) + %tb = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %ta, ptr %a, i64 %s) + call void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) + call void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %tb, ptr %b, i64 %s) + ret void +} + +define void @zames_prera_whole_no_cfg(ptr %src, ptr %dst) { +; CHECK-LABEL: name: zames_prera_whole_no_cfg +; CHECK-NOT: ADDI +; CHECK-NOT: PSEUDO +; CHECK: XSAI_ZAMES_MLA_WHOLE +; CHECK: XSAI_ZAMES_MLB_WHOLE +; CHECK: XSAI_ZAMES_MLC_WHOLE +; CHECK: XSAI_ZAMES_MSA_WHOLE +; CHECK: XSAI_ZAMES_MSB_WHOLE +; CHECK: XSAI_ZAMES_MSC_WHOLE + %a = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %b = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr %src) + %c = call target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr %src) + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %a, ptr %dst) + call void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %b, ptr %dst) + call void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32") %c, ptr %dst) + ret void +} + +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlbt.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.triscv.xsai.zames.tile.int32t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mzero.triscv.xsai.zames.tile.int32t() +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mmacc.triscv.xsai.zames.tile.int32t.triscv.xsai.zames.tile.int8t.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int8"), target("riscv.xsai.zames.tile.int32")) +declare void @llvm.riscv.xsai.zames.msc.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlat.triscv.xsai.zames.tile.int8t(ptr, i64) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlct.triscv.xsai.zames.tile.int32t(ptr, i64) +declare void @llvm.riscv.xsai.zames.msa.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msb.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msat.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msbt.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) +declare void @llvm.riscv.xsai.zames.msct.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr, i64) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mlb.whole.triscv.xsai.zames.tile.int8t(ptr) +declare target("riscv.xsai.zames.tile.int32") @llvm.riscv.xsai.zames.mlc.whole.triscv.xsai.zames.tile.int32t(ptr) +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msb.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr) +declare void @llvm.riscv.xsai.zames.msc.whole.triscv.xsai.zames.tile.int32t(target("riscv.xsai.zames.tile.int32"), ptr) diff --git a/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll new file mode 100644 index 00000000000000..e013d630295a7d --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zames/zmasync-codegen.ll @@ -0,0 +1,20 @@ +; RUN: llc -mtriple=riscv64 -mattr=+zmasync -verify-machineinstrs < %s | FileCheck %s + +define void @zmasync_intrinsics(i64 %target) { +; CHECK-LABEL: zmasync_intrinsics: +; CHECK: msyncregreset sync0 +; CHECK-NEXT: mrelease sync0 +; CHECK-NEXT: macquire sync0, a0 +; CHECK-NEXT: mfence +; CHECK-NEXT: ret + %sync0 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() + %sync1 = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.zmasync.sync") %sync0) + call void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.zmasync.sync") %sync1, i64 %target) + call void @llvm.riscv.xsai.zmasync.mfence() + ret void +} + +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.msyncregreset() +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.mrelease(target("riscv.xsai.zmasync.sync")) +declare void @llvm.riscv.xsai.zmasync.macquire(target("riscv.xsai.zmasync.sync"), i64) +declare void @llvm.riscv.xsai.zmasync.mfence() diff --git a/llvm/test/MC/RISCV/zames-invalid.s b/llvm/test/MC/RISCV/zames-invalid.s new file mode 100644 index 00000000000000..f6057e4257918b --- /dev/null +++ b/llvm/test/MC/RISCV/zames-invalid.s @@ -0,0 +1,23 @@ +# RUN: not llvm-mc %s -triple=riscv64 -mattr=+zames 2>&1 | FileCheck %s + +# ACC register where only TileReg is accepted. +mla acc0, (a0), a1 +# CHECK: error: + +# Out-of-range tile/accumulator registers. +mla tr4, (a0), a1 +# CHECK: error: +mzero acc4 +# CHECK: error: + +# Wrong operand count. +msetcfg a0 +# CHECK: error: + +# Illegal mcfg selector. +msetcfg mcfg8, a0 +# CHECK: error: + +# Wrong operand type. +msetcfg mcfg0, mcfg1 +# CHECK: error: diff --git a/llvm/test/MC/RISCV/zames-no-feature.s b/llvm/test/MC/RISCV/zames-no-feature.s new file mode 100644 index 00000000000000..045a7bb338c19e --- /dev/null +++ b/llvm/test/MC/RISCV/zames-no-feature.s @@ -0,0 +1,62 @@ +# RUN: not llvm-mc %s -triple=riscv64 2>&1 | FileCheck %s + +# Every ZAMES mnemonic must be rejected when the extension is not enabled. + +# CHECK: error: instruction requires the following: 'ZAMES' +msettilemi 4 +# CHECK: error: instruction requires the following: 'ZAMES' +msettileni 4 +# CHECK: error: instruction requires the following: 'ZAMES' +msettileki 8 +# CHECK: error: instruction requires the following: 'ZAMES' +msettilem a0 +# CHECK: error: instruction requires the following: 'ZAMES' +msettilen a1 +# CHECK: error: instruction requires the following: 'ZAMES' +msettilek a2 +# CHECK: error: instruction requires the following: 'ZAMES' +msetcfg mcfg0, a0 +# CHECK: error: instruction requires the following: 'ZAMES' +mgetcfg a0, mcfg5 +# CHECK: error: instruction requires the following: 'ZAMES' +minit +# CHECK: error: instruction requires the following: 'ZAMES' +mzero acc0 +# CHECK: error: instruction requires the following: 'ZAMES' +mmacc acc0, tr1, tr0 +# CHECK: error: instruction requires the following: 'ZAMES' +mla tr0, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlb tr2, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlbt tr1, (a1), a4 +# CHECK: error: instruction requires the following: 'ZAMES' +mlc acc2, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlat tr1, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +mlct acc1, (a0), a3 +# CHECK: error: instruction requires the following: 'ZAMES' +msa tr0, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msb tr1, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msat tr2, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msbt tr3, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msc acc0, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +msct acc0, (a2), a5 +# CHECK: error: instruction requires the following: 'ZAMES' +mla.whole tr0, (a0) +# CHECK: error: instruction requires the following: 'ZAMES' +mlb.whole tr1, (a0) +# CHECK: error: instruction requires the following: 'ZAMES' +mlc.whole acc0, (a0) +# CHECK: error: instruction requires the following: 'ZAMES' +msa.whole tr0, (a2) +# CHECK: error: instruction requires the following: 'ZAMES' +msb.whole tr1, (a2) +# CHECK: error: instruction requires the following: 'ZAMES' +msc.whole acc0, (a2) diff --git a/llvm/test/MC/RISCV/zames.s b/llvm/test/MC/RISCV/zames.s new file mode 100644 index 00000000000000..238f5cb3560ac8 --- /dev/null +++ b/llvm/test/MC/RISCV/zames.s @@ -0,0 +1,147 @@ +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zames -show-encoding | FileCheck %s --check-prefix=CHECK-ASM +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zames -filetype=obj | llvm-objdump --mattr=+zames -d - | FileCheck %s --check-prefix=CHECK-OBJ + +# CHECK-ASM: msettilemi 4 # encoding: [0x2b,0x00,0x02,0x20] +# CHECK-OBJ: msettilemi 0x4 +msettilemi 4 + +# CHECK-ASM: msettileni 4 # encoding: [0x2b,0x00,0x02,0x30] +# CHECK-OBJ: msettileni 0x4 +msettileni 4 + +# CHECK-ASM: msettileki 8 # encoding: [0x2b,0x00,0x04,0x10] +# CHECK-OBJ: msettileki 0x8 +msettileki 8 + +# CHECK-ASM: msettilem a0 # encoding: [0x2b,0x00,0x05,0x22] +# CHECK-OBJ: msettilem a0 +msettilem a0 + +# CHECK-ASM: msettilen a1 # encoding: [0x2b,0x80,0x05,0x32] +# CHECK-OBJ: msettilen a1 +msettilen a1 + +# CHECK-ASM: msettilek a2 # encoding: [0x2b,0x00,0x06,0x12] +# CHECK-OBJ: msettilek a2 +msettilek a2 + +# CHECK-ASM: msetcfg mcfg0, a0 # encoding: [0x2b,0x00,0x05,0x42] +# CHECK-OBJ: msetcfg mcfg0, a0 +msetcfg mcfg0, a0 + +# CHECK-ASM: msetcfg mcfg4, a1 # encoding: [0x2b,0x82,0x05,0x42] +# CHECK-OBJ: msetcfg mcfg4, a1 +msetcfg mcfg4, a1 + +# CHECK-ASM: msetcfg mcfg7, a1 # encoding: [0xab,0x83,0x05,0x42] +# CHECK-OBJ: msetcfg mcfg7, a1 +msetcfg mcfg7, a1 + +# CHECK-ASM: mgetcfg a0, mcfg5 # encoding: [0x2b,0x85,0x02,0x50] +# CHECK-OBJ: mgetcfg a0, mcfg5 +mgetcfg a0, mcfg5 + +# CHECK-ASM: minit # encoding: [0x2b,0x00,0x00,0x00] +# CHECK-OBJ: minit +minit + +# CHECK-ASM: mla tr0, (a0), a3 # encoding: [0x2b,0x10,0xd5,0x00] +# CHECK-OBJ: mla tr0, (a0), a3 +mla tr0, (a0), a3 + +# CHECK-ASM: mlb tr2, (a0), a3 # encoding: [0x2b,0x11,0xd5,0x10] +# CHECK-OBJ: mlb tr2, (a0), a3 +mlb tr2, (a0), a3 + +# CHECK-ASM: mlbt tr1, (a1), a4 # encoding: [0xab,0x90,0xe5,0x50] +# CHECK-OBJ: mlbt tr1, (a1), a4 +mlbt tr1, (a1), a4 + +# CHECK-ASM: mlc acc2, (a0), a3 # encoding: [0x2b,0x13,0xd5,0x20] +# CHECK-OBJ: mlc acc2, (a0), a3 +mlc acc2, (a0), a3 + +# CHECK-ASM: mzero acc0 # encoding: [0x2b,0x32,0x00,0x00] +# CHECK-OBJ: mzero acc0 +mzero acc0 + +# CHECK-ASM: mzero tr0 # encoding: [0x2b,0x30,0x00,0x00] +# CHECK-OBJ: mzero tr0 +mzero tr0 + +# CHECK-ASM: mzero tr3 # encoding: [0xab,0x31,0x00,0x00] +# CHECK-OBJ: mzero tr3 +mzero tr3 + +# CHECK-ASM: mzero acc3 # encoding: [0xab,0x33,0x00,0x00] +# CHECK-OBJ: mzero acc3 +mzero acc3 + +# CHECK-ASM: mmacc acc0, tr1, tr0 # encoding: [0x2b,0x22,0x10,0x00] +# CHECK-OBJ: mmacc acc0, tr1, tr0 +mmacc acc0, tr1, tr0 + +# CHECK-ASM: mmacc acc3, tr2, tr1 # encoding: [0xab,0xa3,0x20,0x00] +# CHECK-OBJ: mmacc acc3, tr2, tr1 +mmacc acc3, tr2, tr1 + +# CHECK-ASM: msc acc0, (a2), a5 # encoding: [0x2b,0x12,0xf6,0x22] +# CHECK-OBJ: msc acc0, (a2), a5 +msc acc0, (a2), a5 + +# CHECK-ASM: msc acc3, (a2), a5 # encoding: [0xab,0x13,0xf6,0x22] +# CHECK-OBJ: msc acc3, (a2), a5 +msc acc3, (a2), a5 + +# CHECK-ASM: mlat tr1, (a0), a3 # encoding: [0xab,0x10,0xd5,0x40] +# CHECK-OBJ: mlat tr1, (a0), a3 +mlat tr1, (a0), a3 + +# CHECK-ASM: mlct acc1, (a0), a3 # encoding: [0xab,0x12,0xd5,0x60] +# CHECK-OBJ: mlct acc1, (a0), a3 +mlct acc1, (a0), a3 + +# CHECK-ASM: msa tr0, (a2), a5 # encoding: [0x2b,0x10,0xf6,0x02] +# CHECK-OBJ: msa tr0, (a2), a5 +msa tr0, (a2), a5 + +# CHECK-ASM: msb tr1, (a2), a5 # encoding: [0xab,0x10,0xf6,0x12] +# CHECK-OBJ: msb tr1, (a2), a5 +msb tr1, (a2), a5 + +# CHECK-ASM: msat tr2, (a2), a5 # encoding: [0x2b,0x11,0xf6,0x42] +# CHECK-OBJ: msat tr2, (a2), a5 +msat tr2, (a2), a5 + +# CHECK-ASM: msbt tr3, (a2), a5 # encoding: [0xab,0x11,0xf6,0x52] +# CHECK-OBJ: msbt tr3, (a2), a5 +msbt tr3, (a2), a5 + +# CHECK-ASM: msct acc0, (a2), a5 # encoding: [0x2b,0x12,0xf6,0x62] +# CHECK-OBJ: msct acc0, (a2), a5 +msct acc0, (a2), a5 + +# .whole loads/stores encode rs2 as zero. +# CHECK-ASM: mla.whole tr0, (a0) # encoding: [0x2b,0x10,0x05,0x80] +# CHECK-OBJ: mla.whole tr0, (a0) +mla.whole tr0, (a0) + +# CHECK-ASM: mlb.whole tr1, (a0) # encoding: [0xab,0x10,0x05,0x90] +# CHECK-OBJ: mlb.whole tr1, (a0) +mlb.whole tr1, (a0) + +# CHECK-ASM: mlc.whole acc0, (a0) # encoding: [0x2b,0x12,0x05,0xa0] +# CHECK-OBJ: mlc.whole acc0, (a0) +mlc.whole acc0, (a0) + +# CHECK-ASM: msa.whole tr0, (a2) # encoding: [0x2b,0x10,0x06,0x82] +# CHECK-OBJ: msa.whole tr0, (a2) +msa.whole tr0, (a2) + +# CHECK-ASM: msb.whole tr1, (a2) # encoding: [0xab,0x10,0x06,0x92] +# CHECK-OBJ: msb.whole tr1, (a2) +msb.whole tr1, (a2) + +# CHECK-ASM: msc.whole acc0, (a2) # encoding: [0x2b,0x12,0x06,0xa2] +# CHECK-OBJ: msc.whole acc0, (a2) +msc.whole acc0, (a2) diff --git a/llvm/test/MC/RISCV/zmasync-implies-zames.s b/llvm/test/MC/RISCV/zmasync-implies-zames.s new file mode 100644 index 00000000000000..66d631e4687f11 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync-implies-zames.s @@ -0,0 +1,11 @@ +# RUN: llvm-mc %s -triple=riscv64 -show-encoding | FileCheck %s + +# `zmasync` is a superset of `zames`: enabling it through `.option arch` +# must make ordinary ZAMES instructions available as well. +.option arch, rv64i_zmasync + +# CHECK: msettilemi 4 +msettilemi 4 + +# CHECK: msyncregreset sync0 +msyncregreset sync0 diff --git a/llvm/test/MC/RISCV/zmasync-invalid.s b/llvm/test/MC/RISCV/zmasync-invalid.s new file mode 100644 index 00000000000000..8aed77d984e777 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync-invalid.s @@ -0,0 +1,19 @@ +# RUN: not llvm-mc %s -triple=riscv64 -mattr=+zmasync 2>&1 | FileCheck %s + +# Wrong operand type. +msyncregreset a0 +# CHECK: error: +mrelease a0 +# CHECK: error: +mrelease sr0 +# CHECK: error: + +# Out-of-range sync registers. +msyncregreset sync16 +# CHECK: error: +macquire sync31, a0 +# CHECK: error: + +# Wrong operand type. +macquire sync0, sync1 +# CHECK: error: diff --git a/llvm/test/MC/RISCV/zmasync-no-feature.s b/llvm/test/MC/RISCV/zmasync-no-feature.s new file mode 100644 index 00000000000000..48193f3c83b214 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync-no-feature.s @@ -0,0 +1,18 @@ +# RUN: not llvm-mc %s -triple=riscv64 2>&1 | FileCheck %s --check-prefix=CHECK-NONE +# RUN: not llvm-mc %s -triple=riscv64 -mattr=+zames 2>&1 | FileCheck %s --check-prefix=CHECK-ZAMES + +# Every ZMASYNC mnemonic must be rejected when the extension is not enabled, +# including when only the base ZAMES extension is enabled. + +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +msyncregreset sync0 +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +mrelease sync1 +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +macquire sync15, a0 +# CHECK-NONE: error: instruction requires the following: 'ZMASYNC' +# CHECK-ZAMES: error: instruction requires the following: 'ZMASYNC' +mfence diff --git a/llvm/test/MC/RISCV/zmasync.s b/llvm/test/MC/RISCV/zmasync.s new file mode 100644 index 00000000000000..b923f70dae3940 --- /dev/null +++ b/llvm/test/MC/RISCV/zmasync.s @@ -0,0 +1,22 @@ +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zmasync -show-encoding | FileCheck %s --check-prefix=CHECK-ASM +# RUN: llvm-mc %s -triple=riscv64 -mattr=+zmasync -filetype=obj | llvm-objdump --mattr=+zmasync -d - | FileCheck %s --check-prefix=CHECK-OBJ + +# CHECK-ASM: msyncregreset sync0 # encoding: [0x2b,0x40,0x00,0x80] +# CHECK-OBJ: msyncregreset sync0 +msyncregreset sync0 + +# CHECK-ASM: msyncregreset sync15 # encoding: [0x2b,0x40,0xf0,0x80] +# CHECK-OBJ: msyncregreset sync15 +msyncregreset sync15 + +# CHECK-ASM: mrelease sync1 # encoding: [0x2b,0x40,0x10,0x90] +# CHECK-OBJ: mrelease sync1 +mrelease sync1 + +# CHECK-ASM: macquire sync15, a0 # encoding: [0x2b,0x40,0xf5,0xa0] +# CHECK-OBJ: macquire sync15, a0 +macquire sync15, a0 + +# CHECK-ASM: mfence # encoding: [0x2b,0x40,0x00,0xb0] +# CHECK-OBJ: mfence +mfence diff --git a/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll new file mode 100644 index 00000000000000..1724b25a13f020 --- /dev/null +++ b/llvm/test/Verifier/riscv-zames-intrinsics-invalid.ll @@ -0,0 +1,140 @@ +; Wrong signatures must be rejected by the verifier, proving the names are +; resolved as real intrinsics (an unregistered name would be accepted as an +; ordinary external function). + +; RUN: split-file %s %t +; RUN: not llvm-as %t/wrong-ret-settilem.ll -o /dev/null +; RUN: not llvm-as %t/whole-load-stride.ll -o /dev/null +; RUN: not llvm-as %t/acquire-wrong-ret.ll -o /dev/null +; RUN: not llvm-as %t/whole-store-stride.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-cfg.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-settilemi.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-settileni.ll -o /dev/null +; RUN: not llvm-as %t/nonimm-settileki.ll -o /dev/null +; RUN: not llvm-as %t/range-settilemi-neg.ll -o /dev/null +; RUN: not llvm-as %t/range-settileni-1024.ll -o /dev/null +; RUN: not llvm-as %t/range-settileki-1024.ll -o /dev/null +; RUN: not llvm-as %t/range-msetcfg-8.ll -o /dev/null +; RUN: not llvm-as %t/range-mgetcfg-neg.ll -o /dev/null + +; Note: an overload suffix that disagrees with the argument types (e.g. +; msa.triscv.xsai.zames.tile.int8t taking an int32 tile) is NOT a verifier error — +; the parser re-mangles the name from the argument types. Signature +; violations must therefore probe arity/type/immarg constraints. + +;--- wrong-ret-settilem.ll +; msettilem returns void. +declare i64 @llvm.riscv.xsai.zames.msettilem(i64) + +define i64 @f(i64 %m) { + %r = call i64 @llvm.riscv.xsai.zames.msettilem(i64 %m) + ret i64 %r +} + +;--- whole-load-stride.ll +; .whole loads are pointer-only; a stride operand is not allowed. +declare target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr, i64) + +define target("riscv.xsai.zames.tile.int8") @f(ptr %p, i64 %s) { + %r = call target("riscv.xsai.zames.tile.int8") @llvm.riscv.xsai.zames.mla.whole.triscv.xsai.zames.tile.int8t(ptr %p, i64 %s) + ret target("riscv.xsai.zames.tile.int8") %r +} + +;--- acquire-wrong-ret.ll +; macquire returns void. +declare target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync"), i64) + +define target("riscv.xsai.zmasync.sync") @f(target("riscv.xsai.zmasync.sync") %s, i64 %sem) { + %r = call target("riscv.xsai.zmasync.sync") @llvm.riscv.xsai.zmasync.macquire.triscv.xsai.zmasync.synct(target("riscv.xsai.zmasync.sync") %s, i64 %sem) + ret target("riscv.xsai.zmasync.sync") %r +} + +;--- whole-store-stride.ll +; .whole stores take tile + pointer only. +declare void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8"), ptr, i64) + +define void @f(target("riscv.xsai.zames.tile.int8") %t, ptr %p, i64 %s) { + call void @llvm.riscv.xsai.zames.msa.whole.triscv.xsai.zames.tile.int8t(target("riscv.xsai.zames.tile.int8") %t, ptr %p, i64 %s) + ret void +} + +;--- nonimm-cfg.ll +; The msetcfg selector operand is an ImmArg. +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) + +define void @f(i64 %sel, i64 %payload) { + call void @llvm.riscv.xsai.zames.msetcfg(i64 %sel, i64 %payload) + ret void +} + +;--- nonimm-settilemi.ll +; The immediate tile setters take a constant uimm10; a dynamic value is an +; ImmArg violation. Range selection (register form for >=1024) is done by +; the Buddy lowering, not the verifier. +declare void @llvm.riscv.xsai.zames.msettilemi(i64) + +define void @f(i64 %m) { + call void @llvm.riscv.xsai.zames.msettilemi(i64 %m) + ret void +} + +;--- nonimm-settileni.ll +declare void @llvm.riscv.xsai.zames.msettileni(i64) + +define void @f(i64 %n) { + call void @llvm.riscv.xsai.zames.msettileni(i64 %n) + ret void +} + +;--- nonimm-settileki.ll +declare void @llvm.riscv.xsai.zames.msettileki(i64) + +define void @f(i64 %k) { + call void @llvm.riscv.xsai.zames.msettileki(i64 %k) + ret void +} + +;--- range-settilemi-neg.ll +; The immediate tile setters encode a uimm10 field: the LLVM contract is the +; half-open range [0, 1024). Out-of-range constants must be rejected even +; though they are immediates. +declare void @llvm.riscv.xsai.zames.msettilemi(i64) + +define void @f() { + call void @llvm.riscv.xsai.zames.msettilemi(i64 -1) + ret void +} + +;--- range-settileni-1024.ll +declare void @llvm.riscv.xsai.zames.msettileni(i64) + +define void @f() { + call void @llvm.riscv.xsai.zames.msettileni(i64 1024) + ret void +} + +;--- range-settileki-1024.ll +declare void @llvm.riscv.xsai.zames.msettileki(i64) + +define void @f() { + call void @llvm.riscv.xsai.zames.msettileki(i64 1024) + ret void +} + +;--- range-msetcfg-8.ll +; The config selector is a 3-bit field: the LLVM contract is [0, 8). The +; payload argument is a full GPR value and is intentionally unconstrained. +declare void @llvm.riscv.xsai.zames.msetcfg(i64, i64) + +define void @f(i64 %payload) { + call void @llvm.riscv.xsai.zames.msetcfg(i64 8, i64 %payload) + ret void +} + +;--- range-mgetcfg-neg.ll +declare i64 @llvm.riscv.xsai.zames.mgetcfg(i64) + +define i64 @f() { + %r = call i64 @llvm.riscv.xsai.zames.mgetcfg(i64 -1) + ret i64 %r +}