[RISCV][CodeGen] Implement IR Intrinsic support for K extension

[VincentWu]

Summary

This revision implements IR Intrinsic support for RISCV Scalar Crypto extension according to the specification of version 1.0
Co-author：@ksyx & @VincentWu & @lihongliang & @achieveartificialintelligence

[ksyx]

fix: patch does not apply (retrigger ci after adding parent revision)

[Jim]

llvm/include/llvm/IR/IntrinsicsRISCV.td

1449

one space before ":"
And the same case in the following.

[ksyx]

refactor: remove redundant spaces around colons

[ksyx]

refactor: fix spaces

[ksyx]

fix: types of immargs to better fit the proposal

[achieveartificialintelligence]

Updated to k-1.0-rc4

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

264

This should be done in the else block where hasStdExtZbb() is handled. Zbp is for GREVI.

2925

This should not be needed with the other change.

llvm/lib/Target/RISCV/RISCVInstrInfoZk.td

24

This needs to be rebased on the current Zk patch.

141

There are no _K instructions in the most recent Zk patches.

All Zb* related changes should be in RISCVInstrInfoZb.td

[VincentWu]

rebase and update

[VincentWu]

rename testcase

[VincentWu]

fix intrinsic

[VincentWu]

fix intrinsic and testcase

[craig.topper]

llvm/include/llvm/IR/IntrinsicsRISCV.td

129

Capital Z

130

rev8 is the same as llvm.bswap.i32 or llvm.bswap.i64. We don't need an intrinsic for it.

1495

Capitalize Z in the comments.

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

282

You want Zbkb in addition to Zbb here.

llvm/lib/Target/RISCV/RISCVInstrInfoZb.td

892

This should stay Zbp only.

901

This should stay Zbp only.

1196

We don't need int_riscv_rev8. We can use bswap

llvm/test/CodeGen/RISCV/rv32zbc-zbkc-intrinsic.ll

5

Can we use a single check-prefix?

llvm/test/CodeGen/RISCV/rv32zbp-zbkb.ll

7 ↗

(On Diff #402875)

Can we use one check-prefix for Zbp and Zbkb?

llvm/test/CodeGen/RISCV/rv64zbb-zbp-zbkb.ll

9 ↗

(On Diff #402875)

Can we use one check-prefix for Zbb, Zbp, and Zbkb?

llvm/test/CodeGen/RISCV/rv64zbc-zbkc-intrinsic.ll

5

Can we use a single check-prefix?

llvm/test/CodeGen/RISCV/rv64zbp-zbkb.ll

7 ↗

(On Diff #402875)

Can we use one check-prefix for Zbp and Zbkb?

llvm/test/CodeGen/RISCV/rv64zksed-intrinsic.ll

14

Something went wrong here

30

And here

[VincentWu]

address comments

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

282

Does this line exceed 80 columns?

llvm/lib/Target/RISCV/RISCVInstrInfoZb.td

890

Revert this

892

Revert this

899

Revert this

[VincentWu]

remove duplicate code & clang-format

[craig.topper]

llvm/lib/Target/RISCV/RISCV.td

175

The string for Zbb was recently updated. Please update to match.

llvm/test/CodeGen/RISCV/rv32zbb-zbp-zbkb.ll

9 ↗

(On Diff #403139)

Can we share a prefix for Zbb/Zbp/Zbkb?

llvm/test/CodeGen/RISCV/rv64zbb-zbp-zbkb.ll

507 ↗

(On Diff #403139)

This a bug that Zbp doesn't match Zbb. I'll fix it. You'll need to add Zbkb to RISCVTargetLowering::hasAndNotCompare in this patch.

[VincentWu]

update & address comment

[craig.topper]

LGTM

[VincentWu]

rebase & read to submit

[VincentWu]

In D102310#3274695, @craig.topper wrote:

LGTM

Tks.

[craig.topper]

llvm/include/llvm/IR/IntrinsicsRISCV.td

1494

Why does this intrinsic have the Returned attribute?

[ksyx]

llvm/include/llvm/IR/IntrinsicsRISCV.td

1494

Maybe this is following the old 0.9.0 version that mixes the use of rs1/rd registers.

[craig.topper]

llvm/include/llvm/IR/IntrinsicsRISCV.td

1494

I see, but that was still a mistake. The Returned property means that the *value* passed into the argument is the *value* that is returned. Even if rd/rs1 are the same register, the operation still changes the value in that register.

[ksyx]

llvm/include/llvm/IR/IntrinsicsRISCV.td

1494

Thanks for pointing out this as I may misunderstood this while implementing.

[achieveartificialintelligence]

llvm/include/llvm/IR/IntrinsicsRISCV.td

1494

Thanks! I've committed in 352e19c.

llvm/include/llvm/IR/IntrinsicsRISCV.td

let TargetPrefix = "riscv" in {
   class BitManipGPRGPRGRIntrinsics
       : Intrinsic<[llvm_any_ty],
                   [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
                   [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
 
   // Zbb
   def int_riscv_orc_b : BitManipGPRIntrinsics;
 
-  // Zbc
+  // Zbc or Zbkc
   def int_riscv_clmul  : BitManipGPRGPRIntrinsics;
   def int_riscv_clmulh : BitManipGPRGPRIntrinsics;
+
+  // Zbc
   def int_riscv_clmulr : BitManipGPRGPRIntrinsics;
 
   // Zbe
   def int_riscv_bcompress   : BitManipGPRGPRIntrinsics;
   def int_riscv_bdecompress : BitManipGPRGPRIntrinsics;
 
   // Zbf
   def int_riscv_bfp  : BitManipGPRGPRIntrinsics;
let TargetPrefix = "riscv" in {
   def int_riscv_crc32c_b : BitManipGPRIntrinsics;
   def int_riscv_crc32c_h : BitManipGPRIntrinsics;
   def int_riscv_crc32c_w : BitManipGPRIntrinsics;
   def int_riscv_crc32c_d : BitManipGPRIntrinsics;
 
   // Zbt
   def int_riscv_fsl : BitManipGPRGPRGRIntrinsics;
   def int_riscv_fsr : BitManipGPRGPRGRIntrinsics;
+
+  // zbkb
+  def int_riscv_rev8 : BitManipGPRIntrinsics;
+  def int_riscv_brev8 : BitManipGPRIntrinsics;
+  def int_riscv_zip   : BitManipGPRIntrinsics;
+  def int_riscv_unzip : BitManipGPRIntrinsics;
+
+  // Zbkx
+  def int_riscv_xperm4  : BitManipGPRGPRIntrinsics;
+  def int_riscv_xperm8  : BitManipGPRGPRIntrinsics;
 } // TargetPrefix = "riscv"
 
 //===----------------------------------------------------------------------===//
 // Vectors
 
 // The intrinsic does not have any operand that must be extended.
 defvar NoSplatOperand = 0xF;
 

   }
 
   // Strided loads/stores for fixed vectors.
   def int_riscv_masked_strided_load
         : Intrinsic<[llvm_anyvector_ty],
                     [LLVMMatchType<0>, llvm_anyptr_ty,
                      llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                     [NoCapture<ArgIndex<1>>, IntrReadMem]>;
   def int_riscv_masked_strided_store
         : Intrinsic<[],
                     [llvm_anyvector_ty, llvm_anyptr_ty,
                      llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                     [NoCapture<ArgIndex<1>>, IntrWriteMem]>;
 } // TargetPrefix = "riscv"
+
+//===----------------------------------------------------------------------===//
+// Scalar Cryptography
+//
+// These intrinsics will lower directly into the corresponding instructions
+// added by the scalar cyptography extension, if the extension is present.
+
+let TargetPrefix = "riscv" in {
+
+class ScalarCryptoGprIntrinsicAny
+    : Intrinsic<[llvm_anyint_ty],
+                [LLVMMatchType<0>],
+                [IntrNoMem, IntrSpeculatable]>;
+
+class ScalarCryptoGPRGPRIntrinsics
+    : Intrinsic<[llvm_any_ty],
+                [LLVMMatchType<0>, LLVMMatchType<0>],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable]>;
+
+class ScalarCryptoByteSelect32
+    : Intrinsic<[llvm_i32_ty],
+                [llvm_i32_ty, llvm_i32_ty, llvm_i8_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable,
+                 ImmArg<ArgIndex<2>>]>;
+
+class ScalarCryptoGprGprIntrinsic32
+    : Intrinsic<[llvm_i32_ty],
+                [llvm_i32_ty, llvm_i32_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable]>;
+
+class ScalarCryptoGprGprIntrinsic64
+    : Intrinsic<[llvm_i64_ty],
+                [llvm_i64_ty, llvm_i64_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable]>;
+
+class ScalarCryptoGprIntrinsic64
+    : Intrinsic<[llvm_i64_ty],
+                [llvm_i64_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable]>;
+
+// zknd
+def int_riscv_aes32dsi  : ScalarCryptoByteSelect32;
+def int_riscv_aes32dsmi : ScalarCryptoByteSelect32;
+
+def int_riscv_aes64ds   : ScalarCryptoGprGprIntrinsic64;
+def int_riscv_aes64dsm  : ScalarCryptoGprGprIntrinsic64;
+
+def int_riscv_aes64im   : ScalarCryptoGprIntrinsic64;
+
+// zkne
+def int_riscv_aes32esi  : ScalarCryptoByteSelect32;
+def int_riscv_aes32esmi : ScalarCryptoByteSelect32;
+
+def int_riscv_aes64es   : ScalarCryptoGprGprIntrinsic64;
+def int_riscv_aes64esm  : ScalarCryptoGprGprIntrinsic64;
+
+// zknd & zkne
+def int_riscv_aes64ks2  : ScalarCryptoGprGprIntrinsic64;
+def int_riscv_aes64ks1i : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty],
+                                    [IntrNoMem, IntrSpeculatable,
+                                     IntrWillReturn, ImmArg<ArgIndex<1>>]>;
+
+// zknh
+def int_riscv_sha256sig0 : ScalarCryptoGprIntrinsicAny;
+def int_riscv_sha256sig1 : ScalarCryptoGprIntrinsicAny;
+def int_riscv_sha256sum0 : ScalarCryptoGprIntrinsicAny;
+def int_riscv_sha256sum1 : ScalarCryptoGprIntrinsicAny;
+
+def int_riscv_sha512sig0l : ScalarCryptoGprGprIntrinsic32;
+def int_riscv_sha512sig0h : ScalarCryptoGprGprIntrinsic32;
+def int_riscv_sha512sig1l : ScalarCryptoGprGprIntrinsic32;
+def int_riscv_sha512sig1h : ScalarCryptoGprGprIntrinsic32;
+def int_riscv_sha512sum0r : ScalarCryptoGprGprIntrinsic32;
+def int_riscv_sha512sum1r : ScalarCryptoGprGprIntrinsic32;
+
+def int_riscv_sha512sig0 : ScalarCryptoGprIntrinsic64;
+def int_riscv_sha512sig1 : ScalarCryptoGprIntrinsic64;
+def int_riscv_sha512sum0 : ScalarCryptoGprIntrinsic64;
+def int_riscv_sha512sum1 : ScalarCryptoGprIntrinsic64;
+
+// zksed
+def int_riscv_sm4ks      : ScalarCryptoByteSelect32;
+def int_riscv_sm4ed      : ScalarCryptoByteSelect32;
+
+// zksh
+def int_riscv_sm3p0      : ScalarCryptoGprIntrinsicAny;
+def int_riscv_sm3p1      : ScalarCryptoGprIntrinsicAny;
+} // TargetPrefix = "riscv"

llvm/lib/Target/RISCV/RISCV.td

: Predicate<"Subtarget->hasStdExtZbp() || Subtarget->hasStdExtZbkx()">,
                                    "'Zbkx' (Crossbar permutation instructions)">;
 
 def HasStdExtZbpOrZbkb
     : Predicate<"Subtarget->hasStdExtZbp() || Subtarget->hasStdExtZbkb()">,
                 AssemblerPredicate<(any_of FeatureStdExtZbp, FeatureStdExtZbkb),
                                    "'Zbp' (Permutation 'B' Instructions) or "
                                    "'Zbkb' (Bitmanip instructions for Cryptography)">;
 
+def HasStdExtZbbOrZbkb
+    : Predicate<"Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbkb()">,
+                AssemblerPredicate<(any_of FeatureStdExtZbb, FeatureStdExtZbkb),
+                                   "'Zbb' (Base 'B' Instructions) or "
+                                   "'Zbkb' (Bitmanip instructions for Cryptography)">;
+
 def HasStdExtZbbOrZbpOrZbkb
     : Predicate<"Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp() || Subtarget->hasStdExtZbkb()">,
                 AssemblerPredicate<(any_of FeatureStdExtZbb, FeatureStdExtZbp, FeatureStdExtZbkb),
                                    "'Zbb' (Base 'B' Instructions) or "
                                    "'Zbp' (Permutation 'B' Instructions) or "
                                    "'Zbkb' (Bitmanip instructions for Cryptography)">;
 
 // The Carry-less multiply subextension for cryptography is a subset of basic carry-less multiply subextension. The former should be enabled if the latter is enabled.

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
 
   setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
   setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
   setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
 
-  if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
+  if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
+      Subtarget.hasStdExtZbkb()) {
     if (Subtarget.is64Bit()) {
       setOperationAction(ISD::ROTL, MVT::i32, Custom);
       setOperationAction(ISD::ROTR, MVT::i32, Custom);
     }
   } else {
     setOperationAction(ISD::ROTL, XLenVT, Expand);
     setOperationAction(ISD::ROTR, XLenVT, Expand);
   }
 
   if (Subtarget.hasStdExtZbp()) {
     // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
     // more combining.
     setOperationAction(ISD::BITREVERSE, XLenVT,   Custom);
     setOperationAction(ISD::BSWAP,      XLenVT,   Custom);
     setOperationAction(ISD::BITREVERSE, MVT::i8,  Custom);
     // BSWAP i8 doesn't exist.
     setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
     setOperationAction(ISD::BSWAP,      MVT::i16, Custom);
 
     if (Subtarget.is64Bit()) {
       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
       setOperationAction(ISD::BSWAP,      MVT::i32, Custom);
     }
   } else {
     // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
     // pattern match it directly in isel.
     setOperationAction(ISD::BSWAP, XLenVT,
                        Subtarget.hasStdExtZbb() ? Legal : Expand);
   }
 
   if (Subtarget.hasStdExtZbb()) {
     setOperationAction(ISD::SMIN, XLenVT, Legal);
     setOperationAction(ISD::SMAX, XLenVT, Legal);
     setOperationAction(ISD::UMIN, XLenVT, Legal);
     setOperationAction(ISD::UMAX, XLenVT, Legal);
 
case ISD::INTRINSIC_WO_CHAIN:
     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::INTRINSIC_W_CHAIN:
     return LowerINTRINSIC_W_CHAIN(Op, DAG);
   case ISD::INTRINSIC_VOID:
     return LowerINTRINSIC_VOID(Op, DAG);
   case ISD::BSWAP:
   case ISD::BITREVERSE: {
     // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
     assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
     MVT VT = Op.getSimpleValueType();
     SDLoc DL(Op);
     // Start with the maximum immediate value which is the bitwidth - 1.
     unsigned Imm = VT.getSizeInBits() - 1;
     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
     if (Op.getOpcode() == ISD::BSWAP)
       Imm &= ~0x7U;
     return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),

llvm/lib/Target/RISCV/RISCVInstrInfoZb.td

 def : InstAlias<"bext $rd, $rs1, $shamt",
                 (BEXTI GPR:$rd, GPR:$rs1, uimmlog2xlen:$shamt), 0>;
 } // Predicates = [HasStdExtZbs]
 
 //===----------------------------------------------------------------------===//
 // Codegen patterns
 //===----------------------------------------------------------------------===//
 
-let Predicates = [HasStdExtZbbOrZbp] in {
+let Predicates = [HasStdExtZbbOrZbpOrZbkb] in {
 def : Pat<(and GPR:$rs1, (not GPR:$rs2)), (ANDN GPR:$rs1, GPR:$rs2)>;
 def : Pat<(or  GPR:$rs1, (not GPR:$rs2)), (ORN  GPR:$rs1, GPR:$rs2)>;
 def : Pat<(xor GPR:$rs1, (not GPR:$rs2)), (XNOR GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtZbbOrZbp]
 
-let Predicates = [HasStdExtZbbOrZbp] in {
+let Predicates = [HasStdExtZbbOrZbpOrZbkb] in {
 def : PatGprGpr<rotl, ROL>;
 def : PatGprGpr<rotr, ROR>;
-} // Predicates = [HasStdExtZbbOrZbp]
+} // Predicates = [HasStdExtZbbOrZbpOrZbkb]
 
 let Predicates = [HasStdExtZbs] in {
 def : Pat<(and (not (shiftop<shl> 1, GPR:$rs2)), GPR:$rs1),
           (BCLR GPR:$rs1, GPR:$rs2)>;
 def : Pat<(and (rotl -2, GPR:$rs2), GPR:$rs1), (BCLR GPR:$rs1, GPR:$rs2)>;
 def : Pat<(or (shiftop<shl> 1, GPR:$rs2), GPR:$rs1),
           (BSET GPR:$rs1, GPR:$rs2)>;
 def : Pat<(xor (shiftop<shl> 1, GPR:$rs2), GPR:$rs1),
def : Pat<(and GPR:$r, BCLRITwoBitsMask:$i),
                  (BCLRITwoBitsMaskHigh BCLRITwoBitsMask:$i))>;
 def : Pat<(and GPR:$r, BCLRIANDIMask:$i),
           (BCLRI (ANDI GPR:$r, (BCLRIANDIMaskLow BCLRIANDIMask:$i)),
                  (BCLRITwoBitsMaskHigh BCLRIANDIMask:$i))>;
 }
 
 // There's no encoding for roli in the the 'B' extension as it can be
 // implemented with rori by negating the immediate.
-let Predicates = [HasStdExtZbbOrZbp] in {
+let Predicates = [HasStdExtZbbOrZbpOrZbkb] in {
 def : PatGprImm<rotr, RORI, uimmlog2xlen>;
 def : Pat<(rotl GPR:$rs1, uimmlog2xlen:$shamt),
           (RORI GPR:$rs1, (ImmSubFromXLen uimmlog2xlen:$shamt))>;
 
 // We treat orc.b as a separate instruction, so match it directly. We also
 // lower the Zbb orc.b intrinsic to this.
 def : Pat<(riscv_gorc GPR:$rs1, 7), (ORCB GPR:$rs1)>;
 }

 } // Predicates = [HasStdExtZbp]
 
 let Predicates = [HasStdExtZbp, IsRV64] in
 def : PatGprGpr<int_riscv_xperm_w, XPERMW>;
 
 let Predicates = [HasStdExtZbp, IsRV32] in {
 def : Pat<(i32 (rotr (riscv_grev GPR:$rs1, 24), (i32 16))), (GREVI GPR:$rs1, 8)>;
 def : Pat<(i32 (rotl (riscv_grev GPR:$rs1, 24), (i32 16))), (GREVI GPR:$rs1, 8)>;
+} // Predicates = [HasStdExtZbp, IsRV32]
 
+let Predicates = [HasStdExtZbpOrZbkb, IsRV32] in {
 // We treat rev8 as a separate instruction, so match it directly.
 def : Pat<(i32 (riscv_grev GPR:$rs1, 24)), (REV8_RV32 GPR:$rs1)>;
 
 // We treat zip and unzip as separate instructions, so match it directly.
 def : Pat<(i32 (riscv_shfl GPR:$rs1, 15)), (ZIP_RV32 GPR:$rs1)>;
 def : Pat<(i32 (riscv_unshfl GPR:$rs1, 15)), (UNZIP_RV32 GPR:$rs1)>;
-} // Predicates = [HasStdExtZbp, IsRV32]
+} // Predicates = [HasStdExtZbpOrZbkb, IsRV32]
 
-let Predicates = [HasStdExtZbp, IsRV64] in {
+let Predicates = [HasStdExtZbpOrZbkb, IsRV64] in {
 // We treat rev8 as a separate instruction, so match it directly.
 def : Pat<(i64 (riscv_grev GPR:$rs1, 56)), (REV8_RV64 GPR:$rs1)>;
-} // Predicates = [HasStdExtZbp, IsRV64]
+} // Predicates = [HasStdExtZbpOrZbkb, IsRV64]
 
 let Predicates = [HasStdExtZbt] in {
 def : Pat<(or (and (not GPR:$rs2), GPR:$rs3), (and GPR:$rs2, GPR:$rs1)),
           (CMIX GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 
 def : Pat<(select (XLenVT (setne GPR:$rs2, 0)), GPR:$rs1, GPR:$rs3),
           (CMOV GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 def : Pat<(select (XLenVT (seteq GPR:$rs2, 0)), GPR:$rs3, GPR:$rs1),

 
 let Predicates = [HasStdExtZbb] in {
 def : PatGprGpr<smin, MIN>;
 def : PatGprGpr<smax, MAX>;
 def : PatGprGpr<umin, MINU>;
 def : PatGprGpr<umax, MAXU>;
 } // Predicates = [HasStdExtZbb]
 
-let Predicates = [HasStdExtZbb, IsRV32] in {
+let Predicates = [HasStdExtZbbOrZbkb, IsRV32] in {
 def : Pat<(i32 (bswap GPR:$rs1)), (REV8_RV32 GPR:$rs1)>;
-} // Predicates = [HasStdExtZbb, IsRV32]
+} // Predicates = [HasStdExtZbbOrZbkb, IsRV32]
 
-let Predicates = [HasStdExtZbb, IsRV64] in {
+let Predicates = [HasStdExtZbbOrZbkb, IsRV64] in {
 def : Pat<(i64 (bswap GPR:$rs1)), (REV8_RV64 GPR:$rs1)>;
-} // Predicates = [HasStdExtZbb, IsRV64]
+} // Predicates = [HasStdExtZbbOrZbkb, IsRV64]
 
-let Predicates = [HasStdExtZbp, IsRV32] in {
+let Predicates = [HasStdExtZbp, IsRV32] in
 def : Pat<(i32 (or (and GPR:$rs1, 0x0000FFFF), (shl GPR:$rs2, (i32 16)))),
           (PACK GPR:$rs1, GPR:$rs2)>;
+
+let Predicates = [HasStdExtZbp, IsRV32] in
 def : Pat<(i32 (or (and GPR:$rs2, 0xFFFF0000), (srl GPR:$rs1, (i32 16)))),
           (PACKU GPR:$rs1, GPR:$rs2)>;
 
-}
-let Predicates = [HasStdExtZbp, IsRV64] in {
+let Predicates = [HasStdExtZbp, IsRV64] in
 def : Pat<(i64 (or (and GPR:$rs1, 0x00000000FFFFFFFF), (shl GPR:$rs2, (i64 32)))),
           (PACK GPR:$rs1, GPR:$rs2)>;
+
+let Predicates = [HasStdExtZbp, IsRV64] in
 def : Pat<(i64 (or (and GPR:$rs2, 0xFFFFFFFF00000000), (srl GPR:$rs1, (i64 32)))),
           (PACKU GPR:$rs1, GPR:$rs2)>;
-}
-let Predicates = [HasStdExtZbp] in {
+
+let Predicates = [HasStdExtZbpOrZbkb] in {
 def : Pat<(or (and (shl GPR:$rs2, (XLenVT 8)), 0xFFFF),
               (and GPR:$rs1, 0x00FF)),
           (PACKH GPR:$rs1, GPR:$rs2)>;
 def : Pat<(or (shl (and GPR:$rs2, 0x00FF), (XLenVT 8)),
               (and GPR:$rs1, 0x00FF)),
           (PACKH GPR:$rs1, GPR:$rs2)>;
-}
+} // Predicates = [HasStdExtZbpOrZbkb]
 
 let Predicates = [HasStdExtZbbOrZbp, IsRV32] in
 def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (ZEXTH_RV32 GPR:$rs)>;
 let Predicates = [HasStdExtZbbOrZbp, IsRV64] in
 def : Pat<(i64 (and GPR:$rs, 0xFFFF)), (ZEXTH_RV64 GPR:$rs)>;
 
 // Pattern to exclude simm12 immediates from matching.
 def non_imm12 : PatLeaf<(XLenVT GPR:$a), [{

 def : Pat<(i64 (add (and (shl GPR:$rs1, (i64 1)), 0x1FFFFFFFF), non_imm12:$rs2)),
           (SH1ADDUW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i64 (add (and (shl GPR:$rs1, (i64 2)), 0x3FFFFFFFF), non_imm12:$rs2)),
           (SH2ADDUW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i64 (add (and (shl GPR:$rs1, (i64 3)), 0x7FFFFFFFF), non_imm12:$rs2)),
           (SH3ADDUW GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtZba, IsRV64]
 
-let Predicates = [HasStdExtZbbOrZbp, IsRV64] in {
+let Predicates = [HasStdExtZbbOrZbpOrZbkb, IsRV64] in {
 def : PatGprGpr<riscv_rolw, ROLW>;
 def : PatGprGpr<riscv_rorw, RORW>;
 def : PatGprImm<riscv_rorw, RORIW, uimm5>;
 def : Pat<(riscv_rolw GPR:$rs1, uimm5:$rs2),
           (RORIW GPR:$rs1, (ImmSubFrom32 uimm5:$rs2))>;
-} // Predicates = [HasStdExtZbbOrZbp, IsRV64]
+} // Predicates = [HasStdExtZbbOrZbpOrZbkb, IsRV64]
 
 let Predicates = [HasStdExtZbp, IsRV64] in {
 def : Pat<(riscv_rorw (riscv_grevw GPR:$rs1, 24), 16), (GREVIW GPR:$rs1, 8)>;
 def : Pat<(riscv_rolw (riscv_grevw GPR:$rs1, 24), 16), (GREVIW GPR:$rs1, 8)>;
 def : PatGprGpr<riscv_grevw, GREVW>;
 def : PatGprGpr<riscv_gorcw, GORCW>;
 def : PatGprGpr<riscv_shflw, SHFLW>;
 def : PatGprGpr<riscv_unshflw, UNSHFLW>;

 } // Predicates = [HasStdExtZbt, IsRV64]
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
 def : PatGpr<riscv_clzw, CLZW>;
 def : PatGpr<riscv_ctzw, CTZW>;
 def : Pat<(i64 (ctpop (i64 (zexti32 (i64 GPR:$rs1))))), (CPOPW GPR:$rs1)>;
 } // Predicates = [HasStdExtZbb, IsRV64]
 
-let Predicates = [HasStdExtZbp, IsRV64] in {
+let Predicates = [HasStdExtZbpOrZbkb, IsRV64] in {
 def : Pat<(i64 (sext_inreg (or (shl GPR:$rs2, (i64 16)),
                                (and GPR:$rs1, 0x000000000000FFFF)),
                            i32)),
           (PACKW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i64 (or (sext_inreg (shl GPR:$rs2, (i64 16)), i32),
                    (and GPR:$rs1, 0x000000000000FFFF))),
           (PACKW GPR:$rs1, GPR:$rs2)>;
+}
+
+let Predicates = [HasStdExtZbp, IsRV64] in
 def : Pat<(i64 (or (and (assertsexti32 GPR:$rs2), 0xFFFFFFFFFFFF0000),
                    (srl (and GPR:$rs1, 0xFFFFFFFF), (i64 16)))),
           (PACKUW GPR:$rs1, GPR:$rs2)>;
-} // Predicates = [HasStdExtZbp, IsRV64]
 
-let Predicates = [HasStdExtZbc] in {
+
+let Predicates = [HasStdExtZbcOrZbkc] in {
 def : PatGprGpr<int_riscv_clmul, CLMUL>;
 def : PatGprGpr<int_riscv_clmulh, CLMULH>;
+} // Predicates = [HasStdExtZbcOrZbkc]
+
+let Predicates = [HasStdExtZbc] in
 def : PatGprGpr<int_riscv_clmulr, CLMULR>;
-} // Predicates = [HasStdExtZbc]
 
 let Predicates = [HasStdExtZbe] in {
 def : PatGprGpr<riscv_bcompress, BCOMPRESS>;
 def : PatGprGpr<riscv_bdecompress, BDECOMPRESS>;
 } // Predicates = [HasStdExtZbe]
 
 let Predicates = [HasStdExtZbe, IsRV64] in {
 def : PatGprGpr<riscv_bcompressw, BCOMPRESSW>;

 def : PatGpr<int_riscv_crc32c_d, CRC32CD>;
 } // Predicates = [HasStdExtZbr, IsRV64]
 
 let Predicates = [HasStdExtZbf] in
 def : PatGprGpr<riscv_bfp, BFP>;
 
 let Predicates = [HasStdExtZbf, IsRV64] in
 def : PatGprGpr<riscv_bfpw, BFPW>;
+
+let Predicates = [HasStdExtZbkb] in {
+def : PatGpr<int_riscv_brev8, BREV8>;
+} // Predicates = [HasStdExtZbkb]
+
+let Predicates = [HasStdExtZbkb, IsRV32] in {
+def : PatGpr<int_riscv_rev8, REV8_RV32>;
+def : PatGpr<int_riscv_zip, ZIP_RV32>;
+def : PatGpr<int_riscv_unzip, UNZIP_RV32>;
+} // Predicates = [HasStdExtZbkb, IsRV32]
+
+let Predicates = [HasStdExtZbkb, IsRV64] in
+def : PatGpr<int_riscv_rev8, REV8_RV64>;
+
+let Predicates = [HasStdExtZbkx] in {
+def : PatGprGpr<int_riscv_xperm4, XPERMN>;
+def : PatGprGpr<int_riscv_xperm8, XPERMB>;
+}

llvm/lib/Target/RISCV/RISCVInstrInfoZk.td

 //===----------------------------------------------------------------------===//
 
 def RnumArg : AsmOperandClass {
   let Name = "RnumArg";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidRnumArg";
 }
 
-def rnum : Operand<XLenVT>, ImmLeaf<XLenVT, [{return (Imm >= 0 && Imm <= 10);}]> {
+def rnum : Operand<i32>, TImmLeaf<i32, [{return (Imm >= 0 && Imm <= 10);}]> {
   let ParserMatchClass = RnumArg;
   let EncoderMethod = "getImmOpValue";
   let DecoderMethod = "decodeUImmOperand<4>";
   let OperandType = "OPERAND_RVKRNUM";
   let OperandNamespace = "RISCVOp";
 }
 
+def byteselect : Operand<i8>, TImmLeaf<i8, [{return isUInt<2>(Imm);}]> {
+  let ParserMatchClass = UImmAsmOperand<2>;
+  let DecoderMethod = "decodeUImmOperand<2>";
+  let OperandType = "OPERAND_UIMM2";
+  let OperandNamespace = "RISCVOp";
+}
+
 //===----------------------------------------------------------------------===//
 // Instruction class templates
 //===----------------------------------------------------------------------===//
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class RVKUnary<bits<12> imm12_in, bits<3> funct3, string opcodestr>
     : RVInstI<funct3, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1),
               opcodestr, "$rd, $rs1">{
   let imm12 = imm12_in;
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class RVKByteSelect<bits<5> funct5, string opcodestr>
     : RVInstR<{0b00, funct5}, 0b000, OPC_OP, (outs GPR:$rd),
-              (ins GPR:$rs1, GPR:$rs2, uimm2:$bs),
+              (ins GPR:$rs1, GPR:$rs2, byteselect:$bs),
               opcodestr, "$rd, $rs1, $rs2, $bs">{
   bits<2> bs;
   let Inst{31-30} = bs;
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class RVKUnary_rnum<bits<7> funct7, bits<3> funct3, string opcodestr>
     : RVInstI<funct3, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1, rnum:$rnum),

 def SM4ED : RVKByteSelect<0b11000, "sm4ed">;
 def SM4KS : RVKByteSelect<0b11010, "sm4ks">;
 } // Predicates = [HasStdExtZksed]
 
 let Predicates = [HasStdExtZksh] in {
 def SM3P0 : RVKUnary<0b000100001000, 0b001, "sm3p0">;
 def SM3P1 : RVKUnary<0b000100001001, 0b001, "sm3p1">;
 } // Predicates = [HasStdExtZksh]
+
+//===----------------------------------------------------------------------===//
+// Codegen patterns
+//===----------------------------------------------------------------------===//
+
+class PatGprGprByteSelect<SDPatternOperator OpNode, RVInst Inst>
+    : Pat<(OpNode GPR:$rs1, GPR:$rs2, i8:$imm),
+          (Inst GPR:$rs1, GPR:$rs2, byteselect:$imm)>;
+
+// Zknd
+let Predicates = [HasStdExtZknd, IsRV32] in {
+def : PatGprGprByteSelect<int_riscv_aes32dsi, AES32DSI>;
+def : PatGprGprByteSelect<int_riscv_aes32dsmi, AES32DSMI>;
+} // Predicates = [HasStdExtZknd, IsRV32]
+
+let Predicates = [HasStdExtZknd, IsRV64] in {
+def : PatGprGpr<int_riscv_aes64ds, AES64DS>;
+def : PatGprGpr<int_riscv_aes64dsm, AES64DSM>;
+def : PatGpr<int_riscv_aes64im, AES64IM>;
+} // Predicates = [HasStdExtZknd, IsRV64]
+
+let Predicates = [HasStdExtZkndOrZkne, IsRV64] in {
+def : PatGprGpr<int_riscv_aes64ks2, AES64KS2>;
+def : Pat<(int_riscv_aes64ks1i GPR:$rs1, i32:$rnum),
+          (AES64KS1I GPR:$rs1, rnum:$rnum)>;
+} // Predicates = [HasStdExtZkndOrZkne, IsRV64]
+
+// Zkne
+let Predicates = [HasStdExtZkne, IsRV32] in {
+def : PatGprGprByteSelect<int_riscv_aes32esi, AES32ESI>;
+def : PatGprGprByteSelect<int_riscv_aes32esmi, AES32ESMI>;
+} // Predicates = [HasStdExtZkne, IsRV32]
+
+let Predicates = [HasStdExtZkne, IsRV64] in {
+def : PatGprGpr<int_riscv_aes64es, AES64ES>;
+def : PatGprGpr<int_riscv_aes64esm, AES64ESM>;
+} // Predicates = [HasStdExtZkne, IsRV64]
+
+// Zknh
+let Predicates = [HasStdExtZknh] in {
+def : PatGpr<int_riscv_sha256sig0, SHA256SIG0>;
+def : PatGpr<int_riscv_sha256sig1, SHA256SIG1>;
+def : PatGpr<int_riscv_sha256sum0, SHA256SUM0>;
+def : PatGpr<int_riscv_sha256sum1, SHA256SUM1>;
+} // Predicates = [HasStdExtZknh]
+
+let Predicates = [HasStdExtZknh, IsRV32] in {
+def : PatGprGpr<int_riscv_sha512sig0l, SHA512SIG0L>;
+def : PatGprGpr<int_riscv_sha512sig0h, SHA512SIG0H>;
+def : PatGprGpr<int_riscv_sha512sig1l, SHA512SIG1L>;
+def : PatGprGpr<int_riscv_sha512sig1h, SHA512SIG1H>;
+def : PatGprGpr<int_riscv_sha512sum0r, SHA512SUM0R>;
+def : PatGprGpr<int_riscv_sha512sum1r, SHA512SUM1R>;
+} // Predicates = [HasStdExtZknh, IsRV32]
+
+let Predicates = [HasStdExtZknh, IsRV64] in {
+def : PatGpr<int_riscv_sha512sig0, SHA512SIG0>;
+def : PatGpr<int_riscv_sha512sig1, SHA512SIG1>;
+def : PatGpr<int_riscv_sha512sum0, SHA512SUM0>;
+def : PatGpr<int_riscv_sha512sum1, SHA512SUM1>;
+} // Predicates = [HasStdExtZknh, IsRV64]
+
+// Zksed
+let Predicates = [HasStdExtZksed] in {
+def : PatGprGprByteSelect<int_riscv_sm4ks, SM4KS>;
+def : PatGprGprByteSelect<int_riscv_sm4ed, SM4ED>;
+} // Predicates = [HasStdExtZksed]
+
+// Zksh
+let Predicates = [HasStdExtZksh] in {
+def : PatGpr<int_riscv_sm3p0, SM3P0>;
+def : PatGpr<int_riscv_sm3p1, SM3P1>;
+} // Predicates = [HasStdExtZksh]

llvm/test/CodeGen/RISCV/rv32zbc-intrinsic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+zbc -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV32ZBC
 
-declare i32 @llvm.riscv.clmul.i32(i32 %a, i32 %b)
-
-define i32 @clmul32(i32 %a, i32 %b) nounwind {
-; RV32ZBC-LABEL: clmul32:
-; RV32ZBC:       # %bb.0:
-; RV32ZBC-NEXT:    clmul a0, a0, a1
-; RV32ZBC-NEXT:    ret
-  %tmp = call i32 @llvm.riscv.clmul.i32(i32 %a, i32 %b)
- ret i32 %tmp
-}
-
-declare i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
-
-define i32 @clmul32h(i32 %a, i32 %b) nounwind {
-; RV32ZBC-LABEL: clmul32h:
-; RV32ZBC:       # %bb.0:
-; RV32ZBC-NEXT:    clmulh a0, a0, a1
-; RV32ZBC-NEXT:    ret
-  %tmp = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
- ret i32 %tmp
-}
-
 declare i32 @llvm.riscv.clmulr.i32(i32 %a, i32 %b)
 
 define i32 @clmul32r(i32 %a, i32 %b) nounwind {
 ; RV32ZBC-LABEL: clmul32r:
 ; RV32ZBC:       # %bb.0:
 ; RV32ZBC-NEXT:    clmulr a0, a0, a1
 ; RV32ZBC-NEXT:    ret
   %tmp = call i32 @llvm.riscv.clmulr.i32(i32 %a, i32 %b)
  ret i32 %tmp
 }

llvm/test/CodeGen/RISCV/rv32zbc-zbkc-intrinsic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+zbc -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV32ZBC
+; RUN: llc -mtriple=riscv32 -mattr=+zbkc -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZBKC
 
 declare i32 @llvm.riscv.clmul.i32(i32 %a, i32 %b)
 
 define i32 @clmul32(i32 %a, i32 %b) nounwind {
 ; RV32ZBC-LABEL: clmul32:
 ; RV32ZBC:       # %bb.0:
 ; RV32ZBC-NEXT:    clmul a0, a0, a1
 ; RV32ZBC-NEXT:    ret
+;
+; RV32ZBKC-LABEL: clmul32:
+; RV32ZBKC:       # %bb.0:
+; RV32ZBKC-NEXT:    clmul a0, a0, a1
+; RV32ZBKC-NEXT:    ret
   %tmp = call i32 @llvm.riscv.clmul.i32(i32 %a, i32 %b)
  ret i32 %tmp
 }
 
 declare i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
 
 define i32 @clmul32h(i32 %a, i32 %b) nounwind {
 ; RV32ZBC-LABEL: clmul32h:
 ; RV32ZBC:       # %bb.0:
 ; RV32ZBC-NEXT:    clmulh a0, a0, a1
 ; RV32ZBC-NEXT:    ret
+;
+; RV32ZBKC-LABEL: clmul32h:
+; RV32ZBKC:       # %bb.0:
+; RV32ZBKC-NEXT:    clmulh a0, a0, a1
+; RV32ZBKC-NEXT:    ret
   %tmp = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
  ret i32 %tmp
 }
-
-declare i32 @llvm.riscv.clmulr.i32(i32 %a, i32 %b)
-
-define i32 @clmul32r(i32 %a, i32 %b) nounwind {
-; RV32ZBC-LABEL: clmul32r:
-; RV32ZBC:       # %bb.0:
-; RV32ZBC-NEXT:    clmulr a0, a0, a1
-; RV32ZBC-NEXT:    ret
-  %tmp = call i32 @llvm.riscv.clmulr.i32(i32 %a, i32 %b)
- ret i32 %tmp
-}

llvm/test/CodeGen/RISCV/rv32zbkb-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zbkb -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZBKB
+
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+
+define i32 @ror_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBKB-LABEL: ror_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    ror a0, a0, a1
+; RV32ZBKB-NEXT:    ret
+  %or = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 %b)
+  ret i32 %or
+}
+
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+
+define i32 @rol_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBKB-LABEL: rol_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    rol a0, a0, a1
+; RV32ZBKB-NEXT:    ret
+  %or = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 %b)
+  ret i32 %or
+}
+
+define i32 @rori_i32_fshl(i32 %a) nounwind {
+; RV32ZBKB-LABEL: rori_i32_fshl:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    rori a0, a0, 1
+; RV32ZBKB-NEXT:    ret
+  %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 31)
+  ret i32 %1
+}
+
+define i32 @rori_i32_fshr(i32 %a) nounwind {
+; RV32ZBKB-LABEL: rori_i32_fshr:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    rori a0, a0, 31
+; RV32ZBKB-NEXT:    ret
+  %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 31)
+  ret i32 %1
+}
+
+declare i32 @llvm.riscv.brev8(i32);
+
+define i32 @brev8(i32 %a) nounwind {
+; RV32ZBKB-LABEL: brev8:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    brev8 a0, a0
+; RV32ZBKB-NEXT:    ret
+  %val = call i32 @llvm.riscv.brev8(i32 %a)
+  ret i32 %val
+}
+
+declare i32 @llvm.riscv.rev8(i32)
+
+define i32 @rev8_i32(i32 %a) nounwind {
+; RV32ZBKB-LABEL: rev8_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    rev8 a0, a0
+; RV32ZBKB-NEXT:    ret
+  %1 = tail call i32 @llvm.riscv.rev8(i32 %a)
+  ret i32 %1
+}
+
+declare i32 @llvm.riscv.zip(i32);
+
+define i32 @zip(i32 %a) nounwind {
+; RV32ZBKB-LABEL: zip:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    zip a0, a0
+; RV32ZBKB-NEXT:    ret
+  %val = call i32 @llvm.riscv.zip(i32 %a)
+  ret i32 %val
+}
+
+declare i32 @llvm.riscv.unzip(i32);
+
+define i32 @unzip(i32 %a) nounwind {
+; RV32ZBKB-LABEL: unzip:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    unzip a0, a0
+; RV32ZBKB-NEXT:    ret
+  %val = call i32 @llvm.riscv.unzip(i32 %a)
+  ret i32 %val
+}

llvm/test/CodeGen/RISCV/rv32zbkb.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zbkb -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZBKB
+
+define i32 @andn_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBKB-LABEL: andn_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    andn a0, a0, a1
+; RV32ZBKB-NEXT:    ret
+  %neg = xor i32 %b, -1
+  %and = and i32 %neg, %a
+  ret i32 %and
+}
+
+define i64 @andn_i64(i64 %a, i64 %b) nounwind {
+; RV32ZBKB-LABEL: andn_i64:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    andn a0, a0, a2
+; RV32ZBKB-NEXT:    andn a1, a1, a3
+; RV32ZBKB-NEXT:    ret
+  %neg = xor i64 %b, -1
+  %and = and i64 %neg, %a
+  ret i64 %and
+}
+
+define i32 @orn_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBKB-LABEL: orn_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    orn a0, a0, a1
+; RV32ZBKB-NEXT:    ret
+  %neg = xor i32 %b, -1
+  %or = or i32 %neg, %a
+  ret i32 %or
+}
+
+define i64 @orn_i64(i64 %a, i64 %b) nounwind {
+; RV32ZBKB-LABEL: orn_i64:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    orn a0, a0, a2
+; RV32ZBKB-NEXT:    orn a1, a1, a3
+; RV32ZBKB-NEXT:    ret
+  %neg = xor i64 %b, -1
+  %or = or i64 %neg, %a
+  ret i64 %or
+}
+
+define i32 @xnor_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBKB-LABEL: xnor_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    xnor a0, a0, a1
+; RV32ZBKB-NEXT:    ret
+  %neg = xor i32 %a, -1
+  %xor = xor i32 %neg, %b
+  ret i32 %xor
+}
+
+define i64 @xnor_i64(i64 %a, i64 %b) nounwind {
+; RV32ZBKB-LABEL: xnor_i64:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    xnor a0, a0, a2
+; RV32ZBKB-NEXT:    xnor a1, a1, a3
+; RV32ZBKB-NEXT:    ret
+  %neg = xor i64 %a, -1
+  %xor = xor i64 %neg, %b
+  ret i64 %xor
+}
+
+define i32 @pack_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBKB-LABEL: pack_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    slli a0, a0, 16
+; RV32ZBKB-NEXT:    srli a0, a0, 16
+; RV32ZBKB-NEXT:    slli a1, a1, 16
+; RV32ZBKB-NEXT:    or a0, a1, a0
+; RV32ZBKB-NEXT:    ret
+  %shl = and i32 %a, 65535
+  %shl1 = shl i32 %b, 16
+  %or = or i32 %shl1, %shl
+  ret i32 %or
+}
+
+define i64 @pack_i64(i64 %a, i64 %b) nounwind {
+; RV32ZBKB-LABEL: pack_i64:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    mv a1, a2
+; RV32ZBKB-NEXT:    ret
+  %shl = and i64 %a, 4294967295
+  %shl1 = shl i64 %b, 32
+  %or = or i64 %shl1, %shl
+  ret i64 %or
+}
+
+define i32 @packh_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBKB-LABEL: packh_i32:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    packh a0, a0, a1
+; RV32ZBKB-NEXT:    ret
+  %and = and i32 %a, 255
+  %and1 = shl i32 %b, 8
+  %shl = and i32 %and1, 65280
+  %or = or i32 %shl, %and
+  ret i32 %or
+}
+
+define i64 @packh_i64(i64 %a, i64 %b) nounwind {
+; RV32ZBKB-LABEL: packh_i64:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    packh a0, a0, a2
+; RV32ZBKB-NEXT:    li a1, 0
+; RV32ZBKB-NEXT:    ret
+  %and = and i64 %a, 255
+  %and1 = shl i64 %b, 8
+  %shl = and i64 %and1, 65280
+  %or = or i64 %shl, %and
+  ret i64 %or
+}

llvm/test/CodeGen/RISCV/rv32zbkx-intrinsic.ll

+; RUN: llc -mtriple=riscv32 -mattr=+zbkx -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZBKX
+
+declare i32 @llvm.riscv.xperm8.i32(i32 %a, i32 %b)
+
+define i32 @xperm8(i32 %a, i32 %b) nounwind {
+; RV32ZBKX-LABEL: xperm8:
+; RV32ZBKX:       # %bb.0:
+; RV32ZBKX-NEXT:    xperm8 a0, a0, a1
+; RV32ZBKX-NEXT:    ret
+  %tmp = call i32 @llvm.riscv.xperm8.i32(i32 %a, i32 %b)
+ ret i32 %tmp
+}
+
+declare i32 @llvm.riscv.xperm4.i32(i32 %a, i32 %b)
+
+define i32 @xperm4(i32 %a, i32 %b) nounwind {
+; RV32ZBKX-LABEL: xperm4:
+; RV32ZBKX:       # %bb.0:
+; RV32ZBKX-NEXT:    xperm4 a0, a0, a1
+; RV32ZBKX-NEXT:    ret
+  %tmp = call i32 @llvm.riscv.xperm4.i32(i32 %a, i32 %b)
+ ret i32 %tmp
+}

llvm/test/CodeGen/RISCV/rv32zknd-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zknd -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZKND
+
+declare i32 @llvm.riscv.aes32dsi(i32, i32, i8);
+
+define i32 @aes32dsi(i32 %a, i32 %b) nounwind {
+; RV32ZKND-LABEL: aes32dsi
+; RV32ZKND: # %bb.0:
+; RV32ZKND-NEXT: aes32dsi a0, a0, a1, 0
+; RV32ZKND-NEXT: ret
+    %val = call i32 @llvm.riscv.aes32dsi(i32 %a, i32 %b, i8 0)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.aes32dsmi(i32, i32, i8);
+
+define i32 @aes32dsmi(i32 %a, i32 %b) nounwind {
+; RV32ZKND-LABEL: aes32dsmi
+; RV32ZKND: # %bb.0:
+; RV32ZKND-NEXT: aes32dsmi a0, a0, a1, 1
+; RV32ZKND-NEXT: ret
+    %val = call i32 @llvm.riscv.aes32dsmi(i32 %a, i32 %b, i8 1)
+    ret i32 %val
+}

llvm/test/CodeGen/RISCV/rv32zkne-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zkne -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZKNE
+
+declare i32 @llvm.riscv.aes32esi(i32, i32, i8);
+
+define i32 @aes32esi(i32 %a, i32 %b) nounwind {
+; RV32ZKNE-LABEL: aes32esi
+; RV32ZKNE: # %bb.0:
+; RV32ZKNE-NEXT: aes32esi a0, a0, a1, 2
+; RV32ZKNE-NEXT: ret
+    %val = call i32 @llvm.riscv.aes32esi(i32 %a, i32 %b, i8 2)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.aes32esmi(i32, i32, i8);
+
+define i32 @aes32esmi(i32 %a, i32 %b) nounwind {
+; RV32ZKNE-LABEL: aes32esmi
+; RV32ZKNE: # %bb.0:
+; RV32ZKNE-NEXT: aes32esmi a0, a0, a1, 3
+; RV32ZKNE-NEXT: ret
+    %val = call i32 @llvm.riscv.aes32esmi(i32 %a, i32 %b, i8 3)
+    ret i32 %val
+}

llvm/test/CodeGen/RISCV/rv32zknh-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zknh -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZKNH
+
+
+declare i32 @llvm.riscv.sha256sig0.i32(i32);
+
+define i32 @sha256sig0_i32(i32 %a) nounwind {
+; RV32ZKNH-LABEL: sha256sig0_i32
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha256sig0 a0, a0
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha256sig0.i32(i32 %a)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha256sig1.i32(i32);
+
+define i32 @sha256sig1_i32(i32 %a) nounwind {
+; RV32ZKNH-LABEL: sha256sig1_i32
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha256sig1 a0, a0
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha256sig1.i32(i32 %a)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha256sum0.i32(i32);
+
+define i32 @sha256sum0_i32(i32 %a) nounwind {
+; RV32ZKNH-LABEL: sha256sum0_i32
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha256sum0 a0, a0
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha256sum0.i32(i32 %a)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha256sum1.i32(i32);
+
+define i32 @sha256sum1_i32(i32 %a) nounwind {
+; RV32ZKNH-LABEL: sha256sum1_i32
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha256sum1 a0, a0
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha256sum1.i32(i32 %a)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha512sig0l(i32, i32);
+
+define i32 @sha512sig0l(i32 %a, i32 %b) nounwind {
+; RV32ZKNH-LABEL: sha512sig0l
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha512sig0l a0, a0, a1
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha512sig0l(i32 %a, i32 %b)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha512sig0h(i32, i32);
+
+define i32 @sha512sig0h(i32 %a, i32 %b) nounwind {
+; RV32ZKNH-LABEL: sha512sig0h
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha512sig0h a0, a0, a1
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha512sig0h(i32 %a, i32 %b)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha512sig1l(i32, i32);
+
+define i32 @sha512sig1l(i32 %a, i32 %b) nounwind {
+; RV32ZKNH-LABEL: sha512sig1l
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha512sig1l a0, a0, a1
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha512sig1l(i32 %a, i32 %b)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha512sig1h(i32, i32);
+
+define i32 @sha512sig1h(i32 %a, i32 %b) nounwind {
+; RV32ZKNH-LABEL: sha512sig1h
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha512sig1h a0, a0, a1
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha512sig1h(i32 %a, i32 %b)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha512sum0r(i32, i32);
+
+define i32 @sha512sum0r(i32 %a, i32 %b) nounwind {
+; RV32ZKNH-LABEL: sha512sum0r
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha512sum0r a0, a0, a1
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha512sum0r(i32 %a, i32 %b)
+    ret i32 %val
+}
+
+declare i32 @llvm.riscv.sha512sum1r(i32, i32);
+
+define i32 @sha512sum1r(i32 %a, i32 %b) nounwind {
+; RV32ZKNH-LABEL: sha512sum1r
+; RV32ZKNH: # %bb.0:
+; RV32ZKNH-NEXT: sha512sum1r a0, a0, a1
+; RV32ZKNH-NEXT: ret
+    %val = call i32 @llvm.riscv.sha512sum1r(i32 %a, i32 %b)
+    ret i32 %val
+}

llvm/test/CodeGen/RISCV/rv32zksed-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zksed -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZKSED
+
+declare i32 @llvm.riscv.sm4ks.i32(i32, i32, i8);
+
+define i32 @sm4ks_i32(i32 %a, i32 %b) nounwind {
+; RV32ZKSED-LABEL: sm4ks_i32:
+; RV32ZKSED:       # %bb.0:
+; RV32ZKSED-NEXT:    addi sp, sp, -16
+; RV32ZKSED-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZKSED-NEXT:    li a2, 2
+; RV32ZKSED-NEXT:    call llvm.riscv.sm4ks.i32@plt
+; RV32ZKSED-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZKSED-NEXT:    addi sp, sp, 16
+; RV32ZKSED-NEXT:    ret
+  %val = call i32 @llvm.riscv.sm4ks.i32(i32 %a, i32 %b, i8 2)
+  ret i32 %val
+}
+
+declare i32 @llvm.riscv.sm4ed.i32(i32, i32, i8);
+
+define i32 @sm4ed_i32(i32 %a, i32 %b) nounwind {
+; RV32ZKSED-LABEL: sm4ed_i32:
+; RV32ZKSED:       # %bb.0:
+; RV32ZKSED-NEXT:    addi sp, sp, -16
+; RV32ZKSED-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZKSED-NEXT:    li a2, 3
+; RV32ZKSED-NEXT:    call llvm.riscv.sm4ed.i32@plt
+; RV32ZKSED-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZKSED-NEXT:    addi sp, sp, 16
+; RV32ZKSED-NEXT:    ret
+  %val = call i32 @llvm.riscv.sm4ed.i32(i32 %a, i32 %b, i8 3)
+  ret i32 %val
+}

llvm/test/CodeGen/RISCV/rv32zksh-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zksh -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZKSH
+
+declare i32 @llvm.riscv.sm3p0.i32(i32);
+
+define i32 @sm3p0_i32(i32 %a) nounwind {
+; RV32ZKSH-LABEL: sm3p0_i32:
+; RV32ZKSH:       # %bb.0:
+; RV32ZKSH-NEXT:    sm3p0 a0, a0
+; RV32ZKSH-NEXT:    ret
+  %val = call i32 @llvm.riscv.sm3p0.i32(i32 %a)
+  ret i32 %val
+}
+
+declare i32 @llvm.riscv.sm3p1.i32(i32);
+
+define i32 @sm3p1_i32(i32 %a) nounwind {
+; RV32ZKSH-LABEL: sm3p1_i32:
+; RV32ZKSH:       # %bb.0:
+; RV32ZKSH-NEXT:    sm3p1 a0, a0
+; RV32ZKSH-NEXT:    ret
+  %val = call i32 @llvm.riscv.sm3p1.i32(i32 %a)
+  ret i32 %val
+}

llvm/test/CodeGen/RISCV/rv64zbc-intrinsic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -mattr=+zbc -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64ZBC
 
-declare i64 @llvm.riscv.clmul.i64(i64 %a, i64 %b)
-
-define i64 @clmul64(i64 %a, i64 %b) nounwind {
-; RV64ZBC-LABEL: clmul64:
-; RV64ZBC:       # %bb.0:
-; RV64ZBC-NEXT:    clmul a0, a0, a1
-; RV64ZBC-NEXT:    ret
-  %tmp = call i64 @llvm.riscv.clmul.i64(i64 %a, i64 %b)
- ret i64 %tmp
-}
-
-declare i64 @llvm.riscv.clmulh.i64(i64 %a, i64 %b)
-
-define i64 @clmul64h(i64 %a, i64 %b) nounwind {
-; RV64ZBC-LABEL: clmul64h:
-; RV64ZBC:       # %bb.0:
-; RV64ZBC-NEXT:    clmulh a0, a0, a1
-; RV64ZBC-NEXT:    ret
-  %tmp = call i64 @llvm.riscv.clmulh.i64(i64 %a, i64 %b)
- ret i64 %tmp
-}
-
 declare i64 @llvm.riscv.clmulr.i64(i64 %a, i64 %b)
 
 define i64 @clmul64r(i64 %a, i64 %b) nounwind {
 ; RV64ZBC-LABEL: clmul64r:
 ; RV64ZBC:       # %bb.0:
 ; RV64ZBC-NEXT:    clmulr a0, a0, a1
 ; RV64ZBC-NEXT:    ret
   %tmp = call i64 @llvm.riscv.clmulr.i64(i64 %a, i64 %b)
  ret i64 %tmp
 }

llvm/test/CodeGen/RISCV/rv64zbc-zbkc-intrinsic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -mattr=+zbc -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64ZBC
+; RUN: llc -mtriple=riscv64 -mattr=+zbkc -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZBKC
 
 declare i64 @llvm.riscv.clmul.i64(i64 %a, i64 %b)
 
 define i64 @clmul64(i64 %a, i64 %b) nounwind {
 ; RV64ZBC-LABEL: clmul64:
 ; RV64ZBC:       # %bb.0:
 ; RV64ZBC-NEXT:    clmul a0, a0, a1
 ; RV64ZBC-NEXT:    ret
+;
+; RV64ZBKC-LABEL: clmul64:
+; RV64ZBKC:       # %bb.0:
+; RV64ZBKC-NEXT:    clmul a0, a0, a1
+; RV64ZBKC-NEXT:    ret
   %tmp = call i64 @llvm.riscv.clmul.i64(i64 %a, i64 %b)
  ret i64 %tmp
 }
 
 declare i64 @llvm.riscv.clmulh.i64(i64 %a, i64 %b)
 
 define i64 @clmul64h(i64 %a, i64 %b) nounwind {
 ; RV64ZBC-LABEL: clmul64h:
 ; RV64ZBC:       # %bb.0:
 ; RV64ZBC-NEXT:    clmulh a0, a0, a1
 ; RV64ZBC-NEXT:    ret
+;
+; RV64ZBKC-LABEL: clmul64h:
+; RV64ZBKC:       # %bb.0:
+; RV64ZBKC-NEXT:    clmulh a0, a0, a1
+; RV64ZBKC-NEXT:    ret
   %tmp = call i64 @llvm.riscv.clmulh.i64(i64 %a, i64 %b)
  ret i64 %tmp
 }
 
-declare i64 @llvm.riscv.clmulr.i64(i64 %a, i64 %b)
-
-define i64 @clmul64r(i64 %a, i64 %b) nounwind {
-; RV64ZBC-LABEL: clmul64r:
-; RV64ZBC:       # %bb.0:
-; RV64ZBC-NEXT:    clmulr a0, a0, a1
-; RV64ZBC-NEXT:    ret
-  %tmp = call i64 @llvm.riscv.clmulr.i64(i64 %a, i64 %b)
- ret i64 %tmp
-}

llvm/test/CodeGen/RISCV/rv64zbkb-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zbkb -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZBKB
+
+declare i64 @llvm.riscv.brev8(i64)
+
+define i64 @brev8(i64 %a) nounwind {
+; RV64ZBKB-LABEL: brev8:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    brev8 a0, a0
+; RV64ZBKB-NEXT:    ret
+  %val = call i64 @llvm.riscv.brev8(i64 %a)
+  ret i64 %val
+}
+
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+
+define signext i32 @ror_i32(i32 signext %a, i32 signext %b) nounwind {
+; RV64ZBKB-LABEL: ror_i32:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    rorw a0, a0, a1
+; RV64ZBKB-NEXT:    ret
+  %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 %b)
+  ret i32 %1
+}
+
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+
+define signext i32 @rol_i32(i32 signext %a, i32 signext %b) nounwind {
+; RV64ZBKB-LABEL: rol_i32:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    rolw a0, a0, a1
+; RV64ZBKB-NEXT:    ret
+  %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 %b)
+  ret i32 %1
+}
+
+define signext i32 @grev16_i32_fshl(i32 signext %a) nounwind {
+; RV64ZBKB-LABEL: grev16_i32_fshl:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    roriw a0, a0, 16
+; RV64ZBKB-NEXT:    ret
+  %or = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 16)
+  ret i32 %or
+}
+
+declare i64 @llvm.riscv.rev8(i64)
+
+define i64 @rev8_i64(i64 %a) {
+; RV64ZBKB-LABEL: rev8_i64:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    rev8 a0, a0
+; RV64ZBKB-NEXT:    ret
+  %1 = call i64 @llvm.riscv.rev8(i64 %a)
+  ret i64 %1
+}

llvm/test/CodeGen/RISCV/rv64zbkb.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zbkb -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZBKB
+
+define i64 @pack_i64(i64 %a, i64 %b) nounwind {
+; RV64ZBKB-LABEL: pack_i64:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    slli a0, a0, 32
+; RV64ZBKB-NEXT:    srli a0, a0, 32
+; RV64ZBKB-NEXT:    slli a1, a1, 32
+; RV64ZBKB-NEXT:    or a0, a1, a0
+; RV64ZBKB-NEXT:    ret
+  %shl = and i64 %a, 4294967295
+  %shl1 = shl i64 %b, 32
+  %or = or i64 %shl1, %shl
+  ret i64 %or
+}
+
+define signext i32 @packh_i32(i32 signext %a, i32 signext %b) nounwind {
+; RV64ZBKB-LABEL: packh_i32:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a0, a0, a1
+; RV64ZBKB-NEXT:    ret
+  %and = and i32 %a, 255
+  %and1 = shl i32 %b, 8
+  %shl = and i32 %and1, 65280
+  %or = or i32 %shl, %and
+  ret i32 %or
+}
+
+define i64 @packh_i64(i64 %a, i64 %b) nounwind {
+; RV64ZBKB-LABEL: packh_i64:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a0, a0, a1
+; RV64ZBKB-NEXT:    ret
+  %and = and i64 %a, 255
+  %and1 = shl i64 %b, 8
+  %shl = and i64 %and1, 65280
+  %or = or i64 %shl, %and
+  ret i64 %or
+}
+
+define signext i32 @pack_i32(i32 signext %a, i32 signext %b) nounwind {
+; RV64ZBKB-LABEL: pack_i32:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packw a0, a0, a1
+; RV64ZBKB-NEXT:    ret
+  %shl = and i32 %a, 65535
+  %shl1 = shl i32 %b, 16
+  %or = or i32 %shl1, %shl
+  ret i32 %or
+}

llvm/test/CodeGen/RISCV/rv64zbkx-intrinsic.ll

+; RUN: llc -mtriple=riscv64 -mattr=+zbkx -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZBKX
+
+declare i64 @llvm.riscv.xperm8.i64(i64 %a, i64 %b)
+
+define i64 @xperm8(i64 %a, i64 %b) nounwind {
+; RV64ZBKX-LABEL: xperm8:
+; RV64ZBKX:       # %bb.0:
+; RV64ZBKX-NEXT:    xperm8 a0, a0, a1
+; RV64ZBKX-NEXT:    ret
+  %tmp = call i64 @llvm.riscv.xperm8.i64(i64 %a, i64 %b)
+ ret i64 %tmp
+}
+
+declare i64 @llvm.riscv.xperm4.i64(i64 %a, i64 %b)
+
+define i64 @xperm4(i64 %a, i64 %b) nounwind {
+; RV64ZBKX-LABEL: xperm4:
+; RV64ZBKX:       # %bb.0:
+; RV64ZBKX-NEXT:    xperm4 a0, a0, a1
+; RV64ZBKX-NEXT:    ret
+  %tmp = call i64 @llvm.riscv.xperm4.i64(i64 %a, i64 %b)
+ ret i64 %tmp
+}

llvm/test/CodeGen/RISCV/rv64zknd-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zknd -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZKND
+
+declare i64 @llvm.riscv.aes64ds(i64, i64);
+
+define i64 @aes64ds(i64 %a, i64 %b) nounwind {
+; RV64ZKND-LABEL: aes64ds
+; RV64ZKND: # %bb.0:
+; RV64ZKND-NEXT: aes64ds a0, a0, a1
+; RV64ZKND-NEXT: ret
+    %val = call i64 @llvm.riscv.aes64ds(i64 %a, i64 %b)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.aes64dsm(i64, i64);
+
+define i64 @aes64dsm(i64 %a, i64 %b) nounwind {
+; RV64ZKND-LABEL: aes64dsm
+; RV64ZKND: # %bb.0:
+; RV64ZKND-NEXT: aes64dsm a0, a0, a1
+; RV64ZKND-NEXT: ret
+    %val = call i64 @llvm.riscv.aes64dsm(i64 %a, i64 %b)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.aes64im(i64);
+
+define i64 @aes64im(i64 %a) nounwind {
+; RV64ZKND-LABEL: aes64im
+; RV64ZKND: # %bb.0:
+; RV64ZKND-NEXT: aes64im a0, a0
+; RV64ZKND-NEXT: ret
+    %val = call i64 @llvm.riscv.aes64im(i64 %a)
+    ret i64 %val
+}

llvm/test/CodeGen/RISCV/rv64zknd-zkne-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zknd -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZKND-ZKNE
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zkne -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZKND-ZKNE
+
+declare i64 @llvm.riscv.aes64ks2(i64, i64);
+
+define i64 @aes64ks2(i64 %a, i64 %b) nounwind {
+; RV64ZKND-ZKNE-LABEL: aes64ks2
+; RV64ZKND-ZKNE: # %bb.0:
+; RV64ZKND-ZKNE-NEXT: aes64ks2 a0, a0, a1
+; RV64ZKND-ZKNE-NEXT: ret
+    %val = call i64 @llvm.riscv.aes64ks2(i64 %a, i64 %b)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.aes64ks1i(i64, i32);
+
+define i64 @aes64ks1i(i64 %a) nounwind {
+; RV64ZKND-ZKNE-LABEL: aes64ks1i
+; RV64ZKND-ZKNE: # %bb.0:
+; RV64ZKND-ZKNE-NEXT: aes64ks1i a0, a0, 10
+; RV64ZKND-ZKNE-NEXT: ret
+    %val = call i64 @llvm.riscv.aes64ks1i(i64 %a, i32 10)
+    ret i64 %val
+}

llvm/test/CodeGen/RISCV/rv64zkne-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zkne -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZKNE
+
+declare i64 @llvm.riscv.aes64es(i64, i64);
+
+define i64 @aes64es(i64 %a, i64 %b) nounwind {
+; RV64ZKNE-LABEL: aes64es
+; RV64ZKNE: # %bb.0:
+; RV64ZKNE-NEXT: aes64es a0, a0, a1
+; RV64ZKNE-NEXT: ret
+    %val = call i64 @llvm.riscv.aes64es(i64 %a, i64 %b)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.aes64esm(i64, i64);
+
+define i64 @aes64esm(i64 %a, i64 %b) nounwind {
+; RV64ZKNE-LABEL: aes64esm
+; RV64ZKNE: # %bb.0:
+; RV64ZKNE-NEXT: aes64esm a0, a0, a1
+; RV64ZKNE-NEXT: ret
+    %val = call i64 @llvm.riscv.aes64esm(i64 %a, i64 %b)
+    ret i64 %val
+}

llvm/test/CodeGen/RISCV/rv64zknh-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zknh -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZKNH
+
+
+declare i64 @llvm.riscv.sha256sig0.i64(i64);
+
+define i64 @sha256sig0_i64(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha256sig0_i64
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha256sig0 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha256sig0.i64(i64 %a)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.sha256sig1.i64(i64);
+
+define i64 @sha256sig1_i64(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha256sig1_i64
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha256sig1 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha256sig1.i64(i64 %a)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.sha256sum0.i64(i64);
+
+define i64 @sha256sum0_i64(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha256sum0_i64
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha256sum0 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha256sum0.i64(i64 %a)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.sha256sum1.i64(i64);
+
+define i64 @sha256sum1_i64(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha256sum1_i64
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha256sum1 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha256sum1.i64(i64 %a)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.sha512sig0(i64);
+
+define i64 @sha512sig0(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha512sig0
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha512sig0 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha512sig0(i64 %a)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.sha512sig1(i64);
+
+define i64 @sha512sig1(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha512sig1
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha512sig1 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha512sig1(i64 %a)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.sha512sum0(i64);
+
+define i64 @sha512sum0(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha512sum0
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha512sum0 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha512sum0(i64 %a)
+    ret i64 %val
+}
+
+declare i64 @llvm.riscv.sha512sum1(i64);
+
+define i64 @sha512sum1(i64 %a) nounwind {
+; RV64ZKNH-LABEL: sha512sum1
+; RV64ZKNH: # %bb.0:
+; RV64ZKNH-NEXT: sha512sum1 a0, a0
+; RV64ZKNH-NEXT: ret
+    %val = call i64 @llvm.riscv.sha512sum1(i64 %a)
+    ret i64 %val
+}

llvm/test/CodeGen/RISCV/rv64zksed-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zksed -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZKSED
+
+declare i64 @llvm.riscv.sm4ks.i64(i32, i32, i32);
+
+define i64 @sm4ks_i64(i32 %a, i32 %b) nounwind {
+; RV64ZKSED-LABEL: sm4ks_i64:
+; RV64ZKSED:       # %bb.0:
+; RV64ZKSED-NEXT:    addi sp, sp, -16
+; RV64ZKSED-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ZKSED-NEXT:    li a2, 0
+; RV64ZKSED-NEXT:    call llvm.riscv.sm4ks.i64@plt
+; RV64ZKSED-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ZKSED-NEXT:    addi sp, sp, 16
+; RV64ZKSED-NEXT:    ret
+  %val = call i64 @llvm.riscv.sm4ks.i64(i32 %a, i32 %b, i32 0)
+  ret i64 %val
+}
+
+declare i64 @llvm.riscv.sm4ed.i64(i32, i32, i32);
+
+define i64 @sm4ed_i64(i32 %a, i32 %b) nounwind {
+; RV64ZKSED-LABEL: sm4ed_i64:
+; RV64ZKSED:       # %bb.0:
+; RV64ZKSED-NEXT:    addi sp, sp, -16
+; RV64ZKSED-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ZKSED-NEXT:    li a2, 1
+; RV64ZKSED-NEXT:    call llvm.riscv.sm4ed.i64@plt
+; RV64ZKSED-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ZKSED-NEXT:    addi sp, sp, 16
+; RV64ZKSED-NEXT:    ret
+  %val = call i64 @llvm.riscv.sm4ed.i64(i32 %a, i32 %b, i32 1)
+  ret i64 %val
+}

llvm/test/CodeGen/RISCV/rv64zksh-intrinsic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+zksh -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZKSH
+
+declare i64 @llvm.riscv.sm3p0.i64(i64);
+
+define i64 @sm3p0_i64(i64 %a) nounwind {
+; RV64ZKSH-LABEL: sm3p0_i64:
+; RV64ZKSH:       # %bb.0:
+; RV64ZKSH-NEXT:    sm3p0 a0, a0
+; RV64ZKSH-NEXT:    ret
+  %val = call i64 @llvm.riscv.sm3p0.i64(i64 %a)
+  ret i64 %val
+}
+
+declare i64 @llvm.riscv.sm3p1.i64(i64);
+
+define i64 @sm3p1_i64(i64 %a) nounwind {
+; RV64ZKSH-LABEL: sm3p1_i64:
+; RV64ZKSH:       # %bb.0:
+; RV64ZKSH-NEXT:    sm3p1 a0, a0
+; RV64ZKSH-NEXT:    ret
+  %val = call i64 @llvm.riscv.sm3p1.i64(i64 %a)
+  ret i64 %val
+}