[RISCV] Add XSAI ZAMES and ZMASYNC vendor extensions

clang/test/Driver/print-supported-extensions-riscv.c

@@ -35,6 +35,7 @@
 // CHECK-NEXT:     zihpm                2.0       'Zihpm' (Hardware Performance Counters)
 // CHECK-NEXT:     zilsd                1.0       'Zilsd' (Load/Store Pair Instructions)
 // CHECK-NEXT:     zimop                1.0       'Zimop' (May-Be-Operations)
+// CHECK: zmasync{{ +}}1.0{{( +|$)}}
 // CHECK-NEXT:     zmmul                1.0       'Zmmul' (Integer Multiplication)
 // CHECK-NEXT:     za128rs              1.0       'Za128rs' (Reservation Set Size of at Most 128 Bytes)
 // CHECK-NEXT:     za64rs               1.0       'Za64rs' (Reservation Set Size of at Most 64 Bytes)
@@ -44,6 +45,7 @@
 // CHECK-NEXT:     zalasr               1.0       'Zalasr' (Load-Acquire and Store-Release Instructions)
 // CHECK-NEXT:     zalrsc               1.0       'Zalrsc' (Load-Reserved/Store-Conditional)
 // CHECK-NEXT:     zama16b              1.0       'Zama16b' (Atomic 16-byte misaligned loads, stores and AMOs)
+// CHECK: zames{{ +}}1.0{{( +|$)}}
 // CHECK-NEXT:     zawrs                1.0       'Zawrs' (Wait on Reservation Set)
 // CHECK-NEXT:     zfa                  1.0       'Zfa' (Additional Floating-Point)
 // CHECK-NEXT:     zfbfmin              1.0       'Zfbfmin' (Scalar BF16 Converts)

llvm/include/llvm/CodeGen/ValueTypes.td

@@ -409,6 +409,15 @@ def amdgpuBufferStridedPointer : ValueType<192>;
 
 def aarch64mfp8 : ValueType<8>; // 8-bit value in FPR (AArch64)
 
+def riscv_xsai_zames_tile_i8  : ValueType<0>; // XSAI ZAMES tile with i8 elements
+def riscv_xsai_zames_tile_i32 : ValueType<0>; // XSAI ZAMES tile with i32 elements
+def riscv_xsai_zmasync_sync     : ValueType<0>; // XSAI ZMASYNC sync register
+def riscv_xsai_zames_tile_fp8e4 : ValueType<0>; // XSAI ZAMES tile with fp8e4m3 elements
+def riscv_xsai_zames_tile_fp8e5 : ValueType<0>; // XSAI ZAMES tile with fp8e5m2 elements
+def riscv_xsai_zames_tile_f16   : ValueType<0>; // XSAI ZAMES tile with fp16 elements
+def riscv_xsai_zames_tile_bf16  : ValueType<0>; // XSAI ZAMES tile with bf16 elements
+def riscv_xsai_zames_tile_f32   : ValueType<0>; // XSAI ZAMES tile with fp32 elements
+
 // CHERI capabilities. Pointer-like values that carry additional metadata
 // for enforcing safety guarantees on CHERI-enabled targets.
 def c64 : VTCheriCapability<64>;   // 64-bit CHERI capability value

llvm/include/llvm/IR/IntrinsicsRISCV.td

@@ -2498,3 +2498,7 @@ include "llvm/IR/IntrinsicsRISCVExt.td"
 include "llvm/IR/IntrinsicsRISCVBOSCExt.td"
 include "llvm/IR/IntrinsicsRISCVXTExt.td"
 include "llvm/IR/IntrinsicsRISCVBuckyballExt.td"
+
+// XSAI ZAMES/ZMASYNC extensions
+//===----------------------------------------------------------------------===//
+include "llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td"

llvm/include/llvm/IR/IntrinsicsRISCVXSAIZamesZmasync.td

@@ -0,0 +1,107 @@
+//===- IntrinsicsRISCVXSAIZamesZmasync.td - XSAI matrix -*- tablegen -*-===//
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//===----------------------------------------------------------------------===//
+
+let TargetPrefix = "riscv" in {
+  class XSAI_ZAMES_SetCfgIntrinsic
+      : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],
+                  [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>,
+                   Range<ArgIndex<0>, 0, 8>]>;
+  class XSAI_ZAMES_GetCfgIntrinsic
+      : Intrinsic<[llvm_i64_ty], [llvm_i64_ty],
+                  [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>,
+                   Range<ArgIndex<0>, 0, 8>]>;
+  class XSAI_ZAMES_SetTileIntrinsic
+      : Intrinsic<[], [llvm_i64_ty],
+                  [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>,
+                   Range<ArgIndex<0>, 0, 1024>]>;
+  class XSAI_ZAMES_SetTileRegIntrinsic
+      : Intrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>;
+  class XSAI_ZAMES_InitIntrinsic
+      : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
+
+  class XSAI_ZAMES_MZeroIntrinsic
+      : Intrinsic<[llvm_any_ty], [], [IntrNoMem]>;
+  class XSAI_ZAMES_LoadIntrinsic
+      : Intrinsic<[llvm_any_ty], [llvm_ptr_ty, llvm_i64_ty],
+                  [NoCapture<ArgIndex<0>>, IntrReadMem], "",
+                  [SDNPMemOperand]>;
+  class XSAI_ZAMES_StoreIntrinsic
+      : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty, llvm_i64_ty],
+                  [NoCapture<ArgIndex<1>>, IntrWriteMem], "",
+                  [SDNPMemOperand]>;
+  class XSAI_ZAMES_WholeLoadIntrinsic
+      : Intrinsic<[llvm_any_ty], [llvm_ptr_ty],
+                  [NoCapture<ArgIndex<0>>, IntrReadMem], "",
+                  [SDNPMemOperand]>;
+  class XSAI_ZAMES_WholeStoreIntrinsic
+      : Intrinsic<[], [llvm_any_ty, llvm_ptr_ty],
+                  [NoCapture<ArgIndex<1>>, IntrWriteMem], "",
+                  [SDNPMemOperand]>;
+  class XSAI_ZAMES_MMAccIntrinsic
+      : Intrinsic<[llvm_any_ty],
+                  [llvm_any_ty, llvm_any_ty, LLVMMatchType<0>],
+                  [IntrNoMem]>;
+
+  def int_riscv_xsai_zames_msetcfg : XSAI_ZAMES_SetCfgIntrinsic;
+  def int_riscv_xsai_zames_mgetcfg : XSAI_ZAMES_GetCfgIntrinsic;
+  def int_riscv_xsai_zames_msettilemi : XSAI_ZAMES_SetTileIntrinsic;
+  def int_riscv_xsai_zames_msettileni : XSAI_ZAMES_SetTileIntrinsic;
+  def int_riscv_xsai_zames_msettileki : XSAI_ZAMES_SetTileIntrinsic;
+  def int_riscv_xsai_zames_msettilem : XSAI_ZAMES_SetTileRegIntrinsic;
+  def int_riscv_xsai_zames_msettilen : XSAI_ZAMES_SetTileRegIntrinsic;
+  def int_riscv_xsai_zames_msettilek : XSAI_ZAMES_SetTileRegIntrinsic;
+
+  def int_riscv_xsai_zames_minit : XSAI_ZAMES_InitIntrinsic;
+
+  def int_riscv_xsai_zames_mzero : XSAI_ZAMES_MZeroIntrinsic;
+  def int_riscv_xsai_zames_mmacc : XSAI_ZAMES_MMAccIntrinsic;
+
+  def int_riscv_xsai_zames_mla : XSAI_ZAMES_LoadIntrinsic;
+  def int_riscv_xsai_zames_mlb : XSAI_ZAMES_LoadIntrinsic;
+  def int_riscv_xsai_zames_mlc : XSAI_ZAMES_LoadIntrinsic;
+  def int_riscv_xsai_zames_mlat : XSAI_ZAMES_LoadIntrinsic;
+  def int_riscv_xsai_zames_mlbt : XSAI_ZAMES_LoadIntrinsic;
+  def int_riscv_xsai_zames_mlct : XSAI_ZAMES_LoadIntrinsic;
+  def int_riscv_xsai_zames_mla_whole : XSAI_ZAMES_WholeLoadIntrinsic;
+  def int_riscv_xsai_zames_mlb_whole : XSAI_ZAMES_WholeLoadIntrinsic;
+  def int_riscv_xsai_zames_mlc_whole : XSAI_ZAMES_WholeLoadIntrinsic;
+
+  def int_riscv_xsai_zames_msa : XSAI_ZAMES_StoreIntrinsic;
+  def int_riscv_xsai_zames_msb : XSAI_ZAMES_StoreIntrinsic;
+  def int_riscv_xsai_zames_msc : XSAI_ZAMES_StoreIntrinsic;
+  def int_riscv_xsai_zames_msat : XSAI_ZAMES_StoreIntrinsic;
+  def int_riscv_xsai_zames_msbt : XSAI_ZAMES_StoreIntrinsic;
+  def int_riscv_xsai_zames_msct : XSAI_ZAMES_StoreIntrinsic;
+  def int_riscv_xsai_zames_msa_whole : XSAI_ZAMES_WholeStoreIntrinsic;
+  def int_riscv_xsai_zames_msb_whole : XSAI_ZAMES_WholeStoreIntrinsic;
+  def int_riscv_xsai_zames_msc_whole : XSAI_ZAMES_WholeStoreIntrinsic;
+
+  class XSAI_ZMASYNC_SyncResetIntrinsic
+      : Intrinsic<[llvm_any_ty], [], [IntrNoMem, IntrHasSideEffects]>;
+  class XSAI_ZMASYNC_ReleaseIntrinsic
+      : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>],
+                  [IntrNoMem, IntrHasSideEffects]>;
+  class XSAI_ZMASYNC_AcquireIntrinsic
+      : Intrinsic<[], [llvm_any_ty, llvm_i64_ty],
+                  [IntrNoMem, IntrHasSideEffects]>;
+  class XSAI_ZMASYNC_FenceIntrinsic
+      : Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>;
+
+  def int_riscv_xsai_zmasync_msyncregreset : XSAI_ZMASYNC_SyncResetIntrinsic;
+  def int_riscv_xsai_zmasync_mrelease : XSAI_ZMASYNC_ReleaseIntrinsic;
+  def int_riscv_xsai_zmasync_macquire : XSAI_ZMASYNC_AcquireIntrinsic;
+  def int_riscv_xsai_zmasync_mfence : XSAI_ZMASYNC_FenceIntrinsic;
+}

llvm/lib/CodeGen/ValueTypes.cpp

@@ -158,7 +158,7 @@ TypeSize EVT::getExtendedSizeInBits() const {
   llvm_unreachable("Unrecognized extended type!");
 }
 
-/// getEVTString - This function returns value type as a string, e.g. "i32".
+/// Print the value type, e.g. "i32".
 void EVT::print(raw_ostream &OS) const {
   switch (V.SimpleTy) {
   default:
@@ -242,6 +242,30 @@ void EVT::print(raw_ostream &OS) const {
   case MVT::aarch64mfp8:
     OS << "aarch64mfp8";
     return;
+  case MVT::riscv_xsai_zames_tile_i8:
+    OS << "riscv_xsai_zames_tile_i8";
+    return;
+  case MVT::riscv_xsai_zames_tile_i32:
+    OS << "riscv_xsai_zames_tile_i32";
+    return;
+  case MVT::riscv_xsai_zmasync_sync:
+    OS << "riscv_xsai_zmasync_sync";
+    return;
+  case MVT::riscv_xsai_zames_tile_fp8e4:
+    OS << "riscv_xsai_zames_tile_fp8e4";
+    return;
+  case MVT::riscv_xsai_zames_tile_fp8e5:
+    OS << "riscv_xsai_zames_tile_fp8e5";
+    return;
+  case MVT::riscv_xsai_zames_tile_f16:
+    OS << "riscv_xsai_zames_tile_f16";
+    return;
+  case MVT::riscv_xsai_zames_tile_bf16:
+    OS << "riscv_xsai_zames_tile_bf16";
+    return;
+  case MVT::riscv_xsai_zames_tile_f32:
+    OS << "riscv_xsai_zames_tile_f32";
+    return;
   }
 }
 
@@ -267,6 +291,22 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
     return TargetExtType::get(Context, "aarch64.svcount");
   case MVT::aarch64mfp8:
     return FixedVectorType::get(IntegerType::get(Context, 8), 1);
+  case MVT::riscv_xsai_zames_tile_i8:
+    return TargetExtType::get(Context, "riscv.xsai.zames.tile.int8");
+  case MVT::riscv_xsai_zames_tile_i32:
+    return TargetExtType::get(Context, "riscv.xsai.zames.tile.int32");
+  case MVT::riscv_xsai_zmasync_sync:
+    return TargetExtType::get(Context, "riscv.xsai.zmasync.sync");
+  case MVT::riscv_xsai_zames_tile_fp8e4:
+    return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp8e4");
+  case MVT::riscv_xsai_zames_tile_fp8e5:
+    return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp8e5");
+  case MVT::riscv_xsai_zames_tile_f16:
+    return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp16");
+  case MVT::riscv_xsai_zames_tile_bf16:
+    return TargetExtType::get(Context, "riscv.xsai.zames.tile.bf16");
+  case MVT::riscv_xsai_zames_tile_f32:
+    return TargetExtType::get(Context, "riscv.xsai.zames.tile.fp32");
   case MVT::x86amx:  return Type::getX86_AMXTy(Context);
   case MVT::i64x8:   return IntegerType::get(Context, 512);
   case MVT::amdgpuBufferFatPointer:  return IntegerType::get(Context, 160);
@@ -322,6 +362,22 @@ MVT MVT::getVT(Type *Ty, bool HandleUnknown){
 
       return MVT::getRISCVVectorTupleVT(Sz * NF, NF);
     }
+    if (TargetExtTy->getName() == "riscv.xsai.zames.tile.int8")
+      return MVT(MVT::riscv_xsai_zames_tile_i8);
+    if (TargetExtTy->getName() == "riscv.xsai.zames.tile.int32")
+      return MVT(MVT::riscv_xsai_zames_tile_i32);
+    if (TargetExtTy->getName() == "riscv.xsai.zmasync.sync")
+      return MVT(MVT::riscv_xsai_zmasync_sync);
+    if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp8e4")
+      return MVT(MVT::riscv_xsai_zames_tile_fp8e4);
+    if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp8e5")
+      return MVT(MVT::riscv_xsai_zames_tile_fp8e5);
+    if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp16")
+      return MVT(MVT::riscv_xsai_zames_tile_f16);
+    if (TargetExtTy->getName() == "riscv.xsai.zames.tile.bf16")
+      return MVT(MVT::riscv_xsai_zames_tile_bf16);
+    if (TargetExtTy->getName() == "riscv.xsai.zames.tile.fp32")
+      return MVT(MVT::riscv_xsai_zames_tile_f32);
     if (HandleUnknown)
       return MVT(MVT::Other);
     llvm_unreachable("Unknown target ext type!");

llvm/lib/IR/Type.cpp

@@ -981,6 +981,18 @@ Expected<TargetExtType *> TargetExtType::checkParams(TargetExtType *TTy) {
         "target extension type riscv.vector.tuple should have one "
         "type parameter and one integer parameter");
 
+  if ((TTy->Name == "riscv.xsai.zames.tile.int8" ||
+       TTy->Name == "riscv.xsai.zames.tile.int32" ||
+       TTy->Name == "riscv.xsai.zames.tile.fp8e4" ||
+       TTy->Name == "riscv.xsai.zames.tile.fp8e5" ||
+       TTy->Name == "riscv.xsai.zames.tile.fp16" ||
+       TTy->Name == "riscv.xsai.zames.tile.bf16" ||
+       TTy->Name == "riscv.xsai.zames.tile.fp32" ||
+       TTy->Name == "riscv.xsai.zmasync.sync") &&
+      (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 0))
+    return createStringError("target extension type " + TTy->Name +
+                             " should have no parameters");
+
   // Opaque types in the AMDGPU name space.
   if (TTy->Name == "amdgcn.named.barrier" &&
       (TTy->getNumTypeParameters() != 0 || TTy->getNumIntParameters() != 1)) {
@@ -1077,6 +1089,19 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
         TargetExtType::CanBeLocal, TargetExtType::HasZeroInit);
   }
 
+  // Native XSAI values are register-only SSA resources with no memory layout.
+  if (Name == "riscv.xsai.zames.tile.int8" ||
+      Name == "riscv.xsai.zames.tile.int32" ||
+      Name == "riscv.xsai.zames.tile.fp8e4" ||
+      Name == "riscv.xsai.zames.tile.fp8e5" ||
+      Name == "riscv.xsai.zames.tile.fp16" ||
+      Name == "riscv.xsai.zames.tile.bf16" ||
+      Name == "riscv.xsai.zames.tile.fp32")
+    return TargetTypeInfo(Type::getVoidTy(C));
+
+  if (Name == "riscv.xsai.zmasync.sync")
+    return TargetTypeInfo(Type::getVoidTy(C));
+
   // DirectX resources
   if (Name == "dx.Padding")
     return TargetTypeInfo(

llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp

@@ -240,6 +240,7 @@ class RISCVAsmParser : public MCTargetAsmParser {
   }
 
   ParseStatus parseRegReg(OperandVector &Operands);
+  ParseStatus parseXSAIZamesMCFGSel(OperandVector &Operands);
   ParseStatus parseXSfmmVType(OperandVector &Operands);
   ParseStatus parseZcmpStackAdj(OperandVector &Operands,
                                 bool ExpectNegative = false);
@@ -761,6 +762,8 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     return IsConstantImm && isUInt<N>(Imm);
   }
 
+  bool isXSAIZamesMCFGSel() const { return isUImm<3>(); }
+
   template <unsigned N, unsigned S> bool isUImmShifted() const {
     int64_t Imm;
     if (!isExpr())
@@ -1428,6 +1431,19 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
       getRISCVMCRegisterClass(RISCV::FPR64CRegClassID).contains(Reg);
   bool IsRegVR = getRISCVMCRegisterClass(RISCV::VRRegClassID).contains(Reg);
 
+  // ZAMES shares assembly names with BOSC AME, but has distinct register
+  // records and accumulator encodings. Coerce only for a ZAMES operand class.
+  if ((Kind == MCK_XSAI_ZAMES_TileReg || Kind == MCK_XSAI_ZAMES_TileOrAccReg) &&
+      Reg >= RISCV::TR0 && Reg <= RISCV::TR3) {
+    Op.Reg.Reg = RISCV::XSAI_ZAMES_TR0 + (Reg - RISCV::TR0);
+    return Match_Success;
+  }
+  if ((Kind == MCK_XSAI_ZAMES_AccReg || Kind == MCK_XSAI_ZAMES_TileOrAccReg) &&
+      Reg >= RISCV::ACC0 && Reg <= RISCV::ACC3) {
+    Op.Reg.Reg = RISCV::XSAI_ZAMES_ACC0 + (Reg - RISCV::ACC0);
+    return Match_Success;
+  }
+
   if (Op.isGPR() && Kind == MCK_YGPR) {
     // GPR and capability GPR use the same register names, convert if required.
     Op.Reg.Reg = convertGPRToYGPR(Reg);
@@ -1887,6 +1903,12 @@ MCRegister RISCVAsmParser::matchRegisterNameHelper(StringRef Name) const {
   static_assert(RISCV::F0_D < RISCV::F0_H, "FPR matching must be updated");
   static_assert(RISCV::F0_D < RISCV::F0_F, "FPR matching must be updated");
   static_assert(RISCV::F0_D < RISCV::F0_Q, "FPR matching must be updated");
+  // Shared matrix names initially resolve to BOSC AME; operand validation
+  // converts them to ZAMES registers only when the instruction requires it.
+  static_assert(RISCV::TR0 < RISCV::XSAI_ZAMES_TR0,
+                "Matrix register matching must be updated");
+  static_assert(RISCV::ACC0 < RISCV::XSAI_ZAMES_ACC0,
+                "Matrix register matching must be updated");
   if (!Reg)
     Reg = MatchRegisterAltName(Name);
   if (isRVE() && Reg >= RISCV::X16 && Reg <= RISCV::X31)
@@ -2573,6 +2595,27 @@ bool RISCVAsmParser::generateVTypeError(SMLoc ErrorLoc) {
                "e[8|8alt|16|16alt|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]");
 }
 
+ParseStatus RISCVAsmParser::parseXSAIZamesMCFGSel(OperandVector &Operands) {
+  const AsmToken &Tok = getLexer().getTok();
+  if (!Tok.is(AsmToken::Identifier))
+    return ParseStatus::NoMatch;
+
+  StringRef Name = Tok.getIdentifier();
+  if (!Name.consume_front("mcfg"))
+    return ParseStatus::NoMatch;
+
+  unsigned Cfg = 0;
+  if (Name.consumeInteger(10, Cfg) || !Name.empty() || Cfg > 7)
+    return ParseStatus::NoMatch;
+
+  SMLoc S = getLoc();
+  SMLoc E = Tok.getEndLoc();
+  Operands.push_back(RISCVOperand::createExpr(
+      MCConstantExpr::create(Cfg, getContext()), S, E, isRV64()));
+  Lex();
+  return ParseStatus::Success;
+}
+
 ParseStatus RISCVAsmParser::parseXSfmmVType(OperandVector &Operands) {
   SMLoc S = getLoc();