[RISCV] Support Zbpbo extension v0.9.11

[sunshaoce]

Summary

The Zbpbo extension is part of the P extension, and its instructions overlap with the B extension.
Details: https://github.com/riscv/riscv-p-spec/blob/master/P-ext-proposal.adoc#zbpbo

[sunshaoce]

Add more tests

[sunshaoce]

Fully supported Zbpbo v0.9.11

[sunshaoce]

llvm/lib/Support/RISCVISAInfo.cpp

361

Modifications here have been submitted separately in D128644.

[sunshaoce]

Rebased on D128644

[craig.topper]

clang/include/clang/Basic/BuiltinsRISCV.def

21

What's preventing __builtin_riscv_clz_32 from being used on rv64 with Zbpbo? I think you need to add it to isRISCV32Builtin in SemaChecking.cpp but you'll also need to check that Zbb isn't enabled.

[sunshaoce]

Address @craig.topper's comment. Thanks!

[sunshaoce]

Add alias instructions

[sunshaoce]

Rebase

[sunshaoce]

Rebase on D99158

[sunshaoce]

Rebase

[Jim]

llvm/lib/Target/RISCV/RISCVInstrInfoZb.td

471

FSR and FSRI only exist on RV32 for Zbpbo.

492

CLZ only exist on RV32 for Zbpbo.

clang/include/clang/Basic/BuiltinsRISCV.def

 
 #if defined(BUILTIN) && !defined(TARGET_BUILTIN)
 #   define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
 #endif
 
 // Zbb extension
 TARGET_BUILTIN(__builtin_riscv_orc_b_32, "ZiZi", "nc", "zbb")
 TARGET_BUILTIN(__builtin_riscv_orc_b_64, "WiWi", "nc", "zbb,64bit")
-TARGET_BUILTIN(__builtin_riscv_clz_32, "ZiZi", "nc", "zbb")
+TARGET_BUILTIN(__builtin_riscv_clz_32, "ZiZi", "nc", "zbb|experimental-zbpbo")
 TARGET_BUILTIN(__builtin_riscv_clz_64, "WiWi", "nc", "zbb,64bit")
 TARGET_BUILTIN(__builtin_riscv_ctz_32, "ZiZi", "nc", "zbb")
 TARGET_BUILTIN(__builtin_riscv_ctz_64, "WiWi", "nc", "zbb,64bit")
 
 // Zbc or Zbkc extension
 TARGET_BUILTIN(__builtin_riscv_clmul, "LiLiLi", "nc", "zbc|zbkc")
 TARGET_BUILTIN(__builtin_riscv_clmulh, "LiLiLi", "nc", "zbc|zbkc")
 TARGET_BUILTIN(__builtin_riscv_clmulr, "LiLiLi", "nc", "zbc")

 TARGET_BUILTIN(__builtin_riscv_bdecompress_64, "WiWiWi", "nc",
                "experimental-zbe,64bit")
 
 // Zbf extension
 TARGET_BUILTIN(__builtin_riscv_bfp_32, "ZiZiZi", "nc", "experimental-zbf")
 TARGET_BUILTIN(__builtin_riscv_bfp_64, "WiWiWi", "nc", "experimental-zbf,64bit")
 
 // Zbp extension
-TARGET_BUILTIN(__builtin_riscv_grev_32, "ZiZiZi", "nc", "experimental-zbp")
-TARGET_BUILTIN(__builtin_riscv_grev_64, "WiWiWi", "nc", "experimental-zbp,64bit")
+TARGET_BUILTIN(__builtin_riscv_grev_32, "ZiZiZi", "nc",
+               "experimental-zbp|experimental-zbpbo")
+TARGET_BUILTIN(__builtin_riscv_grev_64, "WiWiWi", "nc",
+               "experimental-zbp|experimental-zbpbo,64bit")
 TARGET_BUILTIN(__builtin_riscv_gorc_32, "ZiZiZi", "nc", "experimental-zbp")
 TARGET_BUILTIN(__builtin_riscv_gorc_64, "WiWiWi", "nc", "experimental-zbp,64bit")
 TARGET_BUILTIN(__builtin_riscv_shfl_32, "ZiZiZi", "nc", "experimental-zbp")
 TARGET_BUILTIN(__builtin_riscv_shfl_64, "WiWiWi", "nc", "experimental-zbp,64bit")
 TARGET_BUILTIN(__builtin_riscv_unshfl_32, "ZiZiZi", "nc", "experimental-zbp")
 TARGET_BUILTIN(__builtin_riscv_unshfl_64, "WiWiWi", "nc", "experimental-zbp,64bit")
 TARGET_BUILTIN(__builtin_riscv_xperm_n, "LiLiLi", "nc", "experimental-zbp")
 TARGET_BUILTIN(__builtin_riscv_xperm_b, "LiLiLi", "nc", "experimental-zbp")
 TARGET_BUILTIN(__builtin_riscv_xperm_h, "LiLiLi", "nc", "experimental-zbp")
 TARGET_BUILTIN(__builtin_riscv_xperm_w, "WiWiWi", "nc", "experimental-zbp,64bit")
 
 // Zbr extension
 TARGET_BUILTIN(__builtin_riscv_crc32_b, "LiLi", "nc", "experimental-zbr")
 TARGET_BUILTIN(__builtin_riscv_crc32_h, "LiLi", "nc", "experimental-zbr")
 TARGET_BUILTIN(__builtin_riscv_crc32_w, "LiLi", "nc", "experimental-zbr")
 TARGET_BUILTIN(__builtin_riscv_crc32c_b, "LiLi", "nc", "experimental-zbr")
 TARGET_BUILTIN(__builtin_riscv_crc32c_h, "LiLi", "nc", "experimental-zbr")
 TARGET_BUILTIN(__builtin_riscv_crc32c_w, "LiLi", "nc", "experimental-zbr")
 TARGET_BUILTIN(__builtin_riscv_crc32_d, "LiLi", "nc", "experimental-zbr,64bit")
 TARGET_BUILTIN(__builtin_riscv_crc32c_d, "LiLi", "nc", "experimental-zbr,64bit")
 
 // Zbt extension
 TARGET_BUILTIN(__builtin_riscv_fsl_32, "LiLiLiLi", "nc", "experimental-zbt")
-TARGET_BUILTIN(__builtin_riscv_fsr_32, "LiLiLiLi", "nc", "experimental-zbt")
+TARGET_BUILTIN(__builtin_riscv_fsr_32, "LiLiLiLi", "nc",
+               "experimental-zbt|experimental-zbpbo")
 TARGET_BUILTIN(__builtin_riscv_fsl_64, "WiWiWiWi", "nc", "experimental-zbt,64bit")
-TARGET_BUILTIN(__builtin_riscv_fsr_64, "WiWiWiWi", "nc", "experimental-zbt,64bit")
+TARGET_BUILTIN(__builtin_riscv_fsr_64, "WiWiWiWi", "nc",
+               "experimental-zbt|experimental-zbpbo,64bit")
 
 // Zbkb extension
 TARGET_BUILTIN(__builtin_riscv_brev8, "LiLi", "nc", "zbkb")
 TARGET_BUILTIN(__builtin_riscv_zip_32, "ZiZi", "nc", "zbkb")
 TARGET_BUILTIN(__builtin_riscv_unzip_32, "ZiZi", "nc", "zbkb")
 
 // Zknd extension
 TARGET_BUILTIN(__builtin_riscv_aes32dsi_32, "ZiZiZiIUc", "nc", "zknd")

clang/lib/Sema/SemaChecking.cpp

bool Sema::CheckRISCVLMUL(CallExpr *TheCall, unsigned ArgNum) {
   int64_t Val = Result.getSExtValue();
   if ((Val >= 0 && Val <= 3) || (Val >= 5 && Val <= 7))
     return false;
 
   return Diag(TheCall->getBeginLoc(), diag::err_riscv_builtin_invalid_lmul)
          << Arg->getSourceRange();
 }
 
-static bool isRISCV32Builtin(unsigned BuiltinID) {
+static bool isRISCV32Builtin(unsigned BuiltinID, const TargetInfo &TI) {
   // These builtins only work on riscv32 targets.
   switch (BuiltinID) {
   case RISCV::BI__builtin_riscv_zip_32:
   case RISCV::BI__builtin_riscv_unzip_32:
   case RISCV::BI__builtin_riscv_aes32dsi_32:
   case RISCV::BI__builtin_riscv_aes32dsmi_32:
   case RISCV::BI__builtin_riscv_aes32esi_32:
   case RISCV::BI__builtin_riscv_aes32esmi_32:
   case RISCV::BI__builtin_riscv_sha512sig0h_32:
   case RISCV::BI__builtin_riscv_sha512sig0l_32:
   case RISCV::BI__builtin_riscv_sha512sig1h_32:
   case RISCV::BI__builtin_riscv_sha512sig1l_32:
   case RISCV::BI__builtin_riscv_sha512sum0r_32:
   case RISCV::BI__builtin_riscv_sha512sum1r_32:
     return true;
+  case RISCV::BI__builtin_riscv_clz_32:
+    if (!TI.hasFeature("zbb"))
+      return true;
   }
 
   return false;
 }
 
 bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI,
                                          unsigned BuiltinID,
                                          CallExpr *TheCall) {
   // CodeGenFunction can also detect this, but this gives a better error
   // message.
   bool FeatureMissing = false;
   SmallVector<StringRef> ReqFeatures;
   StringRef Features = Context.BuiltinInfo.getRequiredFeatures(BuiltinID);
   Features.split(ReqFeatures, ',');
 
   // Check for 32-bit only builtins on a 64-bit target.
   const llvm::Triple &TT = TI.getTriple();
-  if (TT.getArch() != llvm::Triple::riscv32 && isRISCV32Builtin(BuiltinID))
+  if (TT.getArch() != llvm::Triple::riscv32 && isRISCV32Builtin(BuiltinID, TI))
     return Diag(TheCall->getCallee()->getBeginLoc(),
                 diag::err_32_bit_builtin_64_bit_tgt);
 
   // Check if each required feature is included
   for (StringRef F : ReqFeatures) {
     SmallVector<StringRef> ReqOpFeatures;
     F.split(ReqOpFeatures, '|');
     bool HasFeature = false;

clang/test/CodeGen/RISCV/rvp-intrinsics/riscv32-zbpbo.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -no-opaque-pointers -triple riscv32 -target-feature +experimental-zbpbo -emit-llvm %s -o - \
+// RUN:     | FileCheck %s  -check-prefix=RV32ZBPBO
+
+// RV32ZBPBO-LABEL: @clz_32(
+// RV32ZBPBO-NEXT:  entry:
+// RV32ZBPBO-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// RV32ZBPBO-NEXT:    store i32 [[A:%.*]], i32* [[A_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 false)
+// RV32ZBPBO-NEXT:    ret i32 [[TMP1]]
+//
+int clz_32(int a) {
+  return __builtin_riscv_clz_32(a);
+}
+
+// RV32ZBPBO-LABEL: @fsr(
+// RV32ZBPBO-NEXT:  entry:
+// RV32ZBPBO-NEXT:    [[RS1_ADDR:%.*]] = alloca i32, align 4
+// RV32ZBPBO-NEXT:    [[RS2_ADDR:%.*]] = alloca i32, align 4
+// RV32ZBPBO-NEXT:    [[RS3_ADDR:%.*]] = alloca i32, align 4
+// RV32ZBPBO-NEXT:    store i32 [[RS1:%.*]], i32* [[RS1_ADDR]], align 4
+// RV32ZBPBO-NEXT:    store i32 [[RS2:%.*]], i32* [[RS2_ADDR]], align 4
+// RV32ZBPBO-NEXT:    store i32 [[RS3:%.*]], i32* [[RS3_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP0:%.*]] = load i32, i32* [[RS1_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP1:%.*]] = load i32, i32* [[RS2_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP2:%.*]] = load i32, i32* [[RS3_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP3:%.*]] = call i32 @llvm.riscv.fsr.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
+// RV32ZBPBO-NEXT:    ret i32 [[TMP3]]
+//
+int fsr(int rs1, int rs2, int rs3) {
+  return __builtin_riscv_fsr_32(rs1, rs2, rs3);
+}
+
+// RV32ZBPBO-LABEL: @fsri(
+// RV32ZBPBO-NEXT:  entry:
+// RV32ZBPBO-NEXT:    [[RS1_ADDR:%.*]] = alloca i32, align 4
+// RV32ZBPBO-NEXT:    [[RS2_ADDR:%.*]] = alloca i32, align 4
+// RV32ZBPBO-NEXT:    store i32 [[RS1:%.*]], i32* [[RS1_ADDR]], align 4
+// RV32ZBPBO-NEXT:    store i32 [[RS2:%.*]], i32* [[RS2_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP0:%.*]] = load i32, i32* [[RS1_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP1:%.*]] = load i32, i32* [[RS2_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.fsr.i32(i32 [[TMP0]], i32 [[TMP1]], i32 15)
+// RV32ZBPBO-NEXT:    ret i32 [[TMP2]]
+//
+int fsri(int rs1, int rs2) {
+  return __builtin_riscv_fsr_32(rs1, rs2, 15);
+}
+
+// RV32ZBPBO-LABEL: @grevi(
+// RV32ZBPBO-NEXT:  entry:
+// RV32ZBPBO-NEXT:    [[RS1_ADDR:%.*]] = alloca i32, align 4
+// RV32ZBPBO-NEXT:    store i32 [[RS1:%.*]], i32* [[RS1_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP0:%.*]] = load i32, i32* [[RS1_ADDR]], align 4
+// RV32ZBPBO-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.grev.i32(i32 [[TMP0]], i32 13)
+// RV32ZBPBO-NEXT:    ret i32 [[TMP1]]
+//
+long grevi(long rs1) {
+  return __builtin_riscv_grev_32(rs1, 13);
+}

clang/test/CodeGen/RISCV/rvp-intrinsics/riscv64-zbpbo.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -no-opaque-pointers -triple riscv64 -target-feature +experimental-zbpbo -emit-llvm %s -o - \
+// RUN:     | FileCheck %s  -check-prefix=RV64ZBPBO
+
+// RV64ZBPBO-LABEL: @fsrw(
+// RV64ZBPBO-NEXT:  entry:
+// RV64ZBPBO-NEXT:    [[RS1_ADDR:%.*]] = alloca i64, align 8
+// RV64ZBPBO-NEXT:    [[RS2_ADDR:%.*]] = alloca i64, align 8
+// RV64ZBPBO-NEXT:    [[RS3_ADDR:%.*]] = alloca i64, align 8
+// RV64ZBPBO-NEXT:    store i64 [[RS1:%.*]], i64* [[RS1_ADDR]], align 8
+// RV64ZBPBO-NEXT:    store i64 [[RS2:%.*]], i64* [[RS2_ADDR]], align 8
+// RV64ZBPBO-NEXT:    store i64 [[RS3:%.*]], i64* [[RS3_ADDR]], align 8
+// RV64ZBPBO-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RS1_ADDR]], align 8
+// RV64ZBPBO-NEXT:    [[TMP1:%.*]] = load i64, i64* [[RS2_ADDR]], align 8
+// RV64ZBPBO-NEXT:    [[TMP2:%.*]] = load i64, i64* [[RS3_ADDR]], align 8
+// RV64ZBPBO-NEXT:    [[TMP3:%.*]] = call i64 @llvm.riscv.fsr.i64(i64 [[TMP0]], i64 [[TMP1]], i64 [[TMP2]])
+// RV64ZBPBO-NEXT:    ret i64 [[TMP3]]
+//
+long fsrw(long rs1, long rs2, long rs3) {
+  return __builtin_riscv_fsr_64(rs1, rs2, rs3);
+}
+
+// RV64ZBPBO-LABEL: @grevi(
+// RV64ZBPBO-NEXT:  entry:
+// RV64ZBPBO-NEXT:    [[RS1_ADDR:%.*]] = alloca i64, align 8
+// RV64ZBPBO-NEXT:    store i64 [[RS1:%.*]], i64* [[RS1_ADDR]], align 8
+// RV64ZBPBO-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RS1_ADDR]], align 8
+// RV64ZBPBO-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.grev.i64(i64 [[TMP0]], i64 13)
+// RV64ZBPBO-NEXT:    ret i64 [[TMP1]]
+//
+long grevi(long rs1) {
+  return __builtin_riscv_grev_64(rs1, 13);
+}

llvm/lib/Support/RISCVISAInfo.cpp

 
 static const RISCVSupportedExtension SupportedExperimentalExtensions[] = {
     {"zbe", RISCVExtensionVersion{0, 93}},
     {"zbf", RISCVExtensionVersion{0, 93}},
     {"zbm", RISCVExtensionVersion{0, 93}},
     {"zbp", RISCVExtensionVersion{0, 93}},
     {"zbr", RISCVExtensionVersion{0, 93}},
     {"zbt", RISCVExtensionVersion{0, 93}},
+    {"zbpbo", RISCVExtensionVersion{0, 911}},
     {"zvfh", RISCVExtensionVersion{0, 1}},
 
     {"p", RISCVExtensionVersion{0, 911}},
     {"zbpbo", RISCVExtensionVersion{0, 911}},
     {"zpn", RISCVExtensionVersion{0, 911}},
     {"zpsfoperand", RISCVExtensionVersion{0, 911}},
 };
 
static Error getExtensionVersion(StringRef Ext, StringRef In, unsigned &Major,
   Major = 0;
   Minor = 0;
   ConsumeLength = 0;
   MajorStr = In.take_while(isDigit);
   In = In.substr(MajorStr.size());
 
   if (!MajorStr.empty() && In.consume_front("p")) {
     MinorStr = In.take_while(isDigit);
     In = In.substr(MajorStr.size() + MinorStr.size() - 1);
 
     // Expected 'p' to be followed by minor version number.
     if (MinorStr.empty()) {
       return createStringError(
           errc::invalid_argument,
           "minor version number missing after 'p' for extension '" + Ext + "'");
     }
   }

llvm/lib/Target/RISCV/RISCV.td

: Predicate<"Subtarget->hasStdExtZbkc()">,
                              "'Zbkc' (Carry-less multiply instructions for Cryptography)">;
 
 def HasStdExtZbcOrZbkc
     : Predicate<"Subtarget->hasStdExtZbc() || Subtarget->hasStdExtZbkc()">,
                 AssemblerPredicate<(any_of FeatureStdExtZbc, FeatureStdExtZbkc),
                                    "'Zbc' (Carry-Less Multiplication) or "
                                    "'Zbkc' (Carry-less multiply instructions for Cryptography)">;
 
+def FeatureStdExtZbpbo
+    : SubtargetFeature<"experimental-zbpbo", "HasStdExtZbpbo", "true",
+                       "'Zbpbo' ('B' & 'P' Overlay Instructions)">;
+def HasStdExtZbpbo
+    : Predicate<"Subtarget->hasStdExtZbpbo()">,
+                             AssemblerPredicate<(all_of FeatureStdExtZbpbo),
+                             "'Zbpbo' ('B' & 'P' Overlay Instructions)">;
+
+def HasStdExtZbbOrZbpbo
+    : Predicate<"Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbpbo()">,
+                AssemblerPredicate<(any_of FeatureStdExtZbb, FeatureStdExtZbpbo),
+                                   "'Zbb' (Basic Bit-Manipulation) or "
+                                   "'Zbpbo' ('B' & 'P' Overlay Instructions)">;
+
+def HasStdExtZbpOrZbpbo
+    : Predicate<"Subtarget->hasStdExtZbp() || Subtarget->hasStdExtZbpbo()">,
+                AssemblerPredicate<(any_of FeatureStdExtZbp, FeatureStdExtZbpbo),
+                                   "'Zbp' (Permutation 'Zb' Instructions) or "
+                                   "'Zbpbo' ('B' & 'P' Overlay Instructions)">;
+
+def HasStdExtZbpOrZbkbOrZbpbo
+    : Predicate<"Subtarget->hasStdExtZbp() || Subtarget->hasStdExtZbkb()"
+                 "|| Subtarget->hasStdExtZbpbo()">,
+                AssemblerPredicate<(any_of FeatureStdExtZbp, FeatureStdExtZbkb,
+                                           FeatureStdExtZbpbo),
+                                   "'Zbp' (Permutation 'Zb' Instructions) or "
+                                   "'Zbkb' (Bitmanip instructions for Cryptography) or "
+                                   "'Zbpbo' ('B' & 'P' Overlay Instructions)">;
+
+def HasStdExtZbtOrZbpbo
+    : Predicate<"Subtarget->hasStdExtZbt() || Subtarget->hasStdExtZbpbo()">,
+                AssemblerPredicate<(any_of FeatureStdExtZbt, FeatureStdExtZbpbo),
+                                   "'Zbt' (Ternary 'Zb' Instructions) or "
+                                   "'Zbpbo' ('B' & 'P' Overlay Instructions)">;
+
 def FeatureStdExtZknd
     : SubtargetFeature<"zknd", "HasStdExtZknd", "true",
                        "'Zknd' (NIST Suite: AES Decryption)">;
 def HasStdExtZknd : Predicate<"Subtarget->hasStdExtZknd()">,
                              AssemblerPredicate<(all_of FeatureStdExtZknd),
                              "'Zknd' (NIST Suite: AES Decryption)">;
 
 def FeatureStdExtZkne

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

if (Subtarget.is64Bit())
           {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF},
           MVT::i32, Custom);
   } else {
     setOperationAction({ISD::CTTZ, ISD::CTLZ, ISD::CTPOP}, XLenVT, Expand);
 
     if (Subtarget.is64Bit())
       setOperationAction(ISD::ABS, MVT::i32, Custom);
   }
+  if (Subtarget.hasStdExtZbpbo()) {
+    setOperationAction({ISD::SMIN, ISD::SMAX}, XLenVT, Legal);
+    setOperationAction(ISD::CTLZ, MVT::i32, Legal);
+    setOperationAction({ISD::BITREVERSE, ISD::BSWAP}, MVT::i16, Custom);
+  }
 
   if (Subtarget.hasStdExtZbt()) {
     setOperationAction({ISD::FSHL, ISD::FSHR}, XLenVT, Custom);
     setOperationAction(ISD::SELECT, XLenVT, Legal);
 
     if (Subtarget.is64Bit())
       setOperationAction({ISD::FSHL, ISD::FSHR}, MVT::i32, Custom);
   } else {
case RISCVISD::SHFL: {
     break;
   }
   case ISD::BSWAP:
   case ISD::BITREVERSE: {
     MVT VT = N->getSimpleValueType(0);
     MVT XLenVT = Subtarget.getXLenVT();
     assert((VT == MVT::i8 || VT == MVT::i16 ||
             (VT == MVT::i32 && Subtarget.is64Bit())) &&
-           Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
+           (Subtarget.hasStdExtZbp() || Subtarget.hasStdExtZbpbo()) &&
+           "Unexpected custom legalisation");
     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
     unsigned Imm = VT.getSizeInBits() - 1;
     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
     if (N->getOpcode() == ISD::BSWAP)
       Imm &= ~0x7U;
     SDValue GREVI = DAG.getNode(RISCVISD::GREV, DL, XLenVT, NewOp0,
                                 DAG.getConstant(Imm, DL, XLenVT));
     // ReplaceNodeResults requires we maintain the same type for the return
bool IsWInstruction =
       N->getOpcode() == RISCVISD::RORW || N->getOpcode() == RISCVISD::ROLW;
   assert((N->getOpcode() == ISD::ROTR || N->getOpcode() == ISD::ROTL ||
           IsWInstruction) &&
          "Unexpected opcode!");
   SDValue Src = N->getOperand(0);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
-  if (!Subtarget.hasStdExtZbp() || Src.getOpcode() != RISCVISD::GREV)
+  if (!(Subtarget.hasStdExtZbp() || Subtarget.hasStdExtZbpbo()) ||
+      Src.getOpcode() != RISCVISD::GREV)
     return SDValue();
 
   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
       !isa<ConstantSDNode>(Src.getOperand(1)))
     return SDValue();
 
   unsigned BitWidth = IsWInstruction ? 32 : VT.getSizeInBits();
   assert(isPowerOf2_32(BitWidth) && "Expected a power of 2");

llvm/lib/Target/RISCV/RISCVInstrInfoZb.td

def BEXTI : RVBShift_ri<0b01001, 0b101, OPC_OP_IMM, "bexti">,
             Sched<[WriteSingleBitImm, ReadSingleBitImm]>;
 } // Predicates = [HasStdExtZbs]
 
 let Predicates = [HasStdExtZbp] in {
 def GORC : ALU_rr<0b0010100, 0b101, "gorc">,
            Sched<[WriteORC, ReadORC, ReadORC]>;
 def GREV : ALU_rr<0b0110100, 0b101, "grev">,
            Sched<[WriteREV, ReadREV, ReadREV]>;
+} // Predicates = [HasStdExtZbp]
 
+let Predicates = [HasStdExtZbpOrZbpbo] in
 def GREVI : RVBShift_ri<0b01101, 0b101, OPC_OP_IMM, "grevi">,
             Sched<[WriteREVImm, ReadREVImm]>;
+
+let Predicates = [HasStdExtZbp] in {
 def GORCI : RVBShift_ri<0b00101, 0b101, OPC_OP_IMM, "gorci">,
             Sched<[WriteORCImm, ReadORCImm]>;
 
 def SHFL   : ALU_rr<0b0000100, 0b001, "shfl">,
              Sched<[WriteSHFL, ReadSHFL, ReadSHFL]>;
 def UNSHFL : ALU_rr<0b0000100, 0b101, "unshfl">,
              Sched<[WriteUNSHFL, ReadUNSHFL, ReadUNSHFL]>;
 

 // the draft bit manipulation specification they were included in. However, we
 // use the mnemonics given to them in the ratified Zbkx extension.
 let Predicates = [HasStdExtZbpOrZbkx] in {
 def XPERM4 : ALU_rr<0b0010100, 0b010, "xperm4">, Sched<[]>;
 def XPERM8 : ALU_rr<0b0010100, 0b100, "xperm8">, Sched<[]>;
 } // Predicates = [HasStdExtZbpOrZbkx]
 
 let Predicates = [HasStdExtZbt] in {
-def CMIX : RVBTernaryR<0b11, 0b001, OPC_OP, "cmix", "$rd, $rs2, $rs1, $rs3">,
-           Sched<[WriteCMix, ReadCMix, ReadCMix, ReadCMix]>;
 def CMOV : RVBTernaryR<0b11, 0b101, OPC_OP, "cmov", "$rd, $rs2, $rs1, $rs3">,
            Sched<[WriteCMov, ReadCMov, ReadCMov, ReadCMov]>;
 def FSL  : RVBTernaryR<0b10, 0b001, OPC_OP, "fsl", "$rd, $rs1, $rs3, $rs2">,
            Sched<[WriteFSReg, ReadFSReg, ReadFSReg, ReadFSReg]>;
+} // Predicates = [HasStdExtZbt]
+
+let Predicates = [HasStdExtZbtOrZbpbo] in {
+def CMIX : RVBTernaryR<0b11, 0b001, OPC_OP, "cmix", "$rd, $rs2, $rs1, $rs3">,
+           Sched<[WriteCMix, ReadCMix, ReadCMix, ReadCMix]>;
 def FSR  : RVBTernaryR<0b10, 0b101, OPC_OP, "fsr", "$rd, $rs1, $rs3, $rs2">,
            Sched<[WriteFSReg, ReadFSReg, ReadFSReg, ReadFSReg]>;
 def FSRI : RVBTernaryImm6<0b101, OPC_OP_IMM, "fsri",
                           "$rd, $rs1, $rs3, $shamt">,
            Sched<[WriteFSRImm, ReadFSRImm, ReadFSRImm]>;
-} // Predicates = [HasStdExtZbt]
+} // Predicates = [HasStdExtZbtOrZbpbo]
 
 let Predicates = [HasStdExtZbt, IsRV64] in {
 def FSLW  : RVBTernaryR<0b10, 0b001, OPC_OP_32,
                         "fslw", "$rd, $rs1, $rs3, $rs2">,
             Sched<[WriteFSReg32, ReadFSReg32, ReadFSReg32, ReadFSReg32]>;
-def FSRW  : RVBTernaryR<0b10, 0b101, OPC_OP_32, "fsrw",
-                        "$rd, $rs1, $rs3, $rs2">,
-            Sched<[WriteFSReg32, ReadFSReg32, ReadFSReg32, ReadFSReg32]>;
 def FSRIW : RVBTernaryImm5<0b10, 0b101, OPC_OP_IMM_32,
                            "fsriw", "$rd, $rs1, $rs3, $shamt">,
             Sched<[WriteFSRImm32, ReadFSRImm32, ReadFSRImm32]>;
 } // Predicates = [HasStdExtZbt, IsRV64]
 
-let Predicates = [HasStdExtZbb] in {
+let Predicates = [HasStdExtZbtOrZbpbo, IsRV64] in
+def FSRW  : RVBTernaryR<0b10, 0b101, OPC_OP_32, "fsrw",
+                        "$rd, $rs1, $rs3, $rs2">,
+            Sched<[WriteFSReg32, ReadFSReg32, ReadFSReg32, ReadFSReg32]>;
+
+let Predicates = [HasStdExtZbbOrZbpbo] in
 def CLZ  : RVBUnary<0b0110000, 0b00000, 0b001, OPC_OP_IMM, "clz">,
            Sched<[WriteCLZ, ReadCLZ]>;
+
+let Predicates = [HasStdExtZbb] in {
 def CTZ  : RVBUnary<0b0110000, 0b00001, 0b001, OPC_OP_IMM, "ctz">,
            Sched<[WriteCTZ, ReadCTZ]>;
 def CPOP : RVBUnary<0b0110000, 0b00010, 0b001, OPC_OP_IMM, "cpop">,
            Sched<[WriteCPOP, ReadCPOP]>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
 def CLZW   : RVBUnary<0b0110000, 0b00000, 0b001, OPC_OP_IMM_32, "clzw">,

 
 let Predicates = [HasStdExtZbcOrZbkc] in {
 def CLMUL  : ALU_rr<0b0000101, 0b001, "clmul", /*Commutable*/1>,
              Sched<[WriteCLMUL, ReadCLMUL, ReadCLMUL]>;
 def CLMULH : ALU_rr<0b0000101, 0b011, "clmulh", /*Commutable*/1>,
              Sched<[WriteCLMUL, ReadCLMUL, ReadCLMUL]>;
 } // Predicates = [HasStdExtZbcOrZbkc]
 
-let Predicates = [HasStdExtZbb] in {
+let Predicates = [HasStdExtZbbOrZbpbo] in {
 def MIN  : ALU_rr<0b0000101, 0b100, "min", /*Commutable*/1>,
            Sched<[WriteIALU, ReadIALU, ReadIALU]>;
-def MINU : ALU_rr<0b0000101, 0b101, "minu", /*Commutable*/1>,
-           Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 def MAX  : ALU_rr<0b0000101, 0b110, "max", /*Commutable*/1>,
            Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+} // Predicates = [HasStdExtZbbOrZbpbo]
+
+let Predicates = [HasStdExtZbb] in {
+def MINU : ALU_rr<0b0000101, 0b101, "minu", /*Commutable*/1>,
+           Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 def MAXU : ALU_rr<0b0000101, 0b111, "maxu", /*Commutable*/1>,
            Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbe] in {
 // NOTE: These mnemonics are from the 0.94 spec. There is a name conflict with
 // bext in the 0.93 spec.
 def BDECOMPRESS : ALU_rr<0b0100100, 0b110, "bdecompress">,
                   Sched<[WriteDecompress, ReadDecompress, ReadDecompress]>;
 def BCOMPRESS   : ALU_rr<0b0000100, 0b110, "bcompress">,
                   Sched<[WriteCompress, ReadCompress, ReadCompress]>;
 } // Predicates = [HasStdExtZbe]
 
 let Predicates = [HasStdExtZbe, IsRV64] in {
 // NOTE: These mnemonics are from the 0.94 spec. There is a name conflict with
 // bextw in the 0.93 spec.
 def BDECOMPRESSW : ALUW_rr<0b0100100, 0b110, "bdecompressw">,
                    Sched<[WriteDecompress32, ReadDecompress32, ReadDecompress32]>;
 def BCOMPRESSW   : ALUW_rr<0b0000100, 0b110, "bcompressw">,
                    Sched<[WriteCompress32, ReadCompress32, ReadCompress32]>;
 } // Predicates = [HasStdExtZbe, IsRV64]
 
-let Predicates = [HasStdExtZbpOrZbkb] in {
+let Predicates = [HasStdExtZbpOrZbkbOrZbpbo] in
 def PACK  : ALU_rr<0b0000100, 0b100, "pack">,
             Sched<[WritePACK, ReadPACK, ReadPACK]>;
+
+let Predicates = [HasStdExtZbpOrZbkb] in
 def PACKH : ALU_rr<0b0000100, 0b111, "packh">,
             Sched<[WritePACK, ReadPACK, ReadPACK]>;
-} // Predicates = [HasStdExtZbpOrZbkb]
 
 let Predicates = [HasStdExtZbpOrZbkb, IsRV64] in 
 def PACKW  : ALUW_rr<0b0000100, 0b100, "packw">,
              Sched<[WritePACK32, ReadPACK32, ReadPACK32]>;
 
-let Predicates = [HasStdExtZbp] in 
+let Predicates = [HasStdExtZbpOrZbpbo] in 
 def PACKU : ALU_rr<0b0100100, 0b100, "packu">,
             Sched<[WritePACKU, ReadPACKU, ReadPACKU]>;
 
 let Predicates = [HasStdExtZbp, IsRV64] in 
 def PACKUW : ALUW_rr<0b0100100, 0b100, "packuw">,
              Sched<[WritePACKU32, ReadPACKU32, ReadPACKU32]>;
 
 let Predicates = [HasStdExtZbm, IsRV64] in {

 } // Predicates = [HasStdExtZba, IsRV64]
 
 let Predicates = [HasStdExtZbp] in {
 def : InstAlias<"rev.p $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b00001)>;
 def : InstAlias<"rev2.n $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b00010)>;
 def : InstAlias<"rev.n $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b00011)>;
 def : InstAlias<"rev4.b $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b00100)>;
 def : InstAlias<"rev2.b $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b00110)>;
-def : InstAlias<"rev8.h $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b01000)>;
 def : InstAlias<"rev4.h $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b01100)>;
 def : InstAlias<"rev2.h $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b01110)>;
 def : InstAlias<"rev.h $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b01111)>;
 def : InstAlias<"rev.b $rd, $rs",  (BREV8 GPR:$rd, GPR:$rs)>;
 
 def : InstAlias<"zip.n $rd, $rs",    (SHFLI   GPR:$rd, GPR:$rs, 0b0001)>;
 def : InstAlias<"unzip.n $rd, $rs",  (UNSHFLI GPR:$rd, GPR:$rs, 0b0001)>;
 def : InstAlias<"zip2.b $rd, $rs",   (SHFLI   GPR:$rd, GPR:$rs, 0b0010)>;

 def : InstAlias<"orc2.b $rd, $rs", (GORCI GPR:$rd, GPR:$rs, 0b00110)>;
 // orc.b is considered an instruction rather than an alias.
 def : InstAlias<"orc8.h $rd, $rs", (GORCI GPR:$rd, GPR:$rs, 0b01000)>;
 def : InstAlias<"orc4.h $rd, $rs", (GORCI GPR:$rd, GPR:$rs, 0b01100)>;
 def : InstAlias<"orc2.h $rd, $rs", (GORCI GPR:$rd, GPR:$rs, 0b01110)>;
 def : InstAlias<"orc.h $rd, $rs",  (GORCI GPR:$rd, GPR:$rs, 0b01111)>;
 } // Predicates = [HasStdExtZbp]
 
+let Predicates = [HasStdExtZbpOrZbpbo] in
+def : InstAlias<"rev8.h $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b01000)>;
+
 let Predicates = [HasStdExtZbp, IsRV32] in {
 def : InstAlias<"rev16 $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b10000)>;
 // rev8 is considered an instruction rather than an alias.
 def : InstAlias<"rev4 $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b11100)>;
 def : InstAlias<"rev2 $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b11110)>;
-def : InstAlias<"rev $rd, $rs",   (GREVI GPR:$rd, GPR:$rs, 0b11111)>;
 
 def : InstAlias<"zip8 $rd, $rs",   (SHFLI   GPR:$rd, GPR:$rs, 0b1000)>;
 def : InstAlias<"unzip8 $rd, $rs", (UNSHFLI GPR:$rd, GPR:$rs, 0b1000)>;
 def : InstAlias<"zip4 $rd, $rs",   (SHFLI   GPR:$rd, GPR:$rs, 0b1100)>;
 def : InstAlias<"unzip4 $rd, $rs", (UNSHFLI GPR:$rd, GPR:$rs, 0b1100)>;
 def : InstAlias<"zip2 $rd, $rs",   (SHFLI   GPR:$rd, GPR:$rs, 0b1110)>;
 def : InstAlias<"unzip2 $rd, $rs", (UNSHFLI GPR:$rd, GPR:$rs, 0b1110)>;
 // zip and unzip are considered instructions rather than an alias.
 
 def : InstAlias<"orc16 $rd, $rs", (GORCI GPR:$rd, GPR:$rs, 0b10000)>;
 def : InstAlias<"orc8 $rd, $rs",  (GORCI GPR:$rd, GPR:$rs, 0b11000)>;
 def : InstAlias<"orc4 $rd, $rs",  (GORCI GPR:$rd, GPR:$rs, 0b11100)>;
 def : InstAlias<"orc2 $rd, $rs",  (GORCI GPR:$rd, GPR:$rs, 0b11110)>;
 def : InstAlias<"orc $rd, $rs",   (GORCI GPR:$rd, GPR:$rs, 0b11111)>;
 } // Predicates = [HasStdExtZbp, IsRV32]
 
+let Predicates = [HasStdExtZbpOrZbpbo, IsRV32] in
+def : InstAlias<"rev $rd, $rs",   (GREVI GPR:$rd, GPR:$rs, 0b11111)>;
+let Predicates = [HasStdExtZbpOrZbpbo, IsRV64] in
+def : InstAlias<"rev $rd, $rs",   (GREVI GPR:$rd, GPR:$rs, 0b111111)>;
+
 let Predicates = [HasStdExtZbp, IsRV64] in {
 def : InstAlias<"rev16.w $rd, $rs", (GREVI GPR:$rd, GPR:$rs, 0b010000)>;
 def : InstAlias<"rev8.w $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b011000)>;
 def : InstAlias<"rev4.w $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b011100)>;
 def : InstAlias<"rev2.w $rd, $rs",  (GREVI GPR:$rd, GPR:$rs, 0b011110)>;
 def : InstAlias<"rev.w $rd, $rs",   (GREVI GPR:$rd, GPR:$rs, 0b011111)>;
 def : InstAlias<"rev32 $rd, $rs",   (GREVI GPR:$rd, GPR:$rs, 0b100000)>;
 def : InstAlias<"rev16 $rd, $rs",   (GREVI GPR:$rd, GPR:$rs, 0b110000)>;

 def : InstAlias<"bclr $rd, $rs1, $shamt",
                 (BCLRI GPR:$rd, GPR:$rs1, uimmlog2xlen:$shamt), 0>;
 def : InstAlias<"binv $rd, $rs1, $shamt",
                 (BINVI GPR:$rd, GPR:$rs1, uimmlog2xlen:$shamt), 0>;
 def : InstAlias<"bext $rd, $rs1, $shamt",
                 (BEXTI GPR:$rd, GPR:$rs1, uimmlog2xlen:$shamt), 0>;
 } // Predicates = [HasStdExtZbs]
 
+let Predicates = [HasStdExtZbpbo, IsRV32] in {
+def : InstAlias<"pkbb16 $rd, $rs1, $rs2",
+                (PACK GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+def : InstAlias<"pktt16 $rd, $rs1, $rs2",
+                (PACKU GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+} // Predicates = [HasStdExtZbpbo]
+
+let Predicates = [HasStdExtZbpbo, IsRV64] in {
+def : InstAlias<"pkbb32 $rd, $rs1, $rs2",
+                (PACK GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+def : InstAlias<"pktt32 $rd, $rs1, $rs2",
+                (PACKU GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+} // Predicates = [HasStdExtZbpbo]
+
 //===----------------------------------------------------------------------===//
 // Codegen patterns
 //===----------------------------------------------------------------------===//
 
 let Predicates = [HasStdExtZbbOrZbpOrZbkb] in {
 def : Pat<(XLenVT (and GPR:$rs1, (not GPR:$rs2))), (ANDN GPR:$rs1, GPR:$rs2)>;
 def : Pat<(XLenVT (or  GPR:$rs1, (not GPR:$rs2))), (ORN  GPR:$rs1, GPR:$rs2)>;
 def : Pat<(XLenVT (xor GPR:$rs1, (not GPR:$rs2))), (XNOR GPR:$rs1, GPR:$rs2)>;

 } // Predicates = [HasStdExtZbpOrZbkb]
 
 let Predicates = [HasStdExtZbpOrZbkb, IsRV32] in {
 // We treat zip and unzip as separate instructions, so match it directly.
 def : Pat<(i32 (riscv_shfl GPR:$rs1, 15)), (ZIP_RV32 GPR:$rs1)>;
 def : Pat<(i32 (riscv_unshfl GPR:$rs1, 15)), (UNZIP_RV32 GPR:$rs1)>;
 } // Predicates = [HasStdExtZbpOrZbkb, IsRV32]
 
-let Predicates = [HasStdExtZbp] in {
+let Predicates = [HasStdExtZbpOrZbpbo] in {
 def : PatGprGpr<riscv_grev, GREV>;
-def : PatGprGpr<riscv_gorc, GORC>;
 def : PatGprImm<riscv_grev, GREVI, uimmlog2xlen>;
+def : PatGprGpr<riscv_shfl, SHFL>;
+} // Predicates = [HasStdExtZbpOrZbpbo]
+
+let Predicates = [HasStdExtZbp] in {
+def : PatGprGpr<riscv_gorc, GORC>;
 def : PatGprImm<riscv_gorc, GORCI, uimmlog2xlen>;
 
-def : PatGprGpr<riscv_shfl, SHFL>;
 def : PatGprGpr<riscv_unshfl, UNSHFL>;
 def : PatGprImm<riscv_shfl, SHFLI, shfl_uimm>;
 def : PatGprImm<riscv_unshfl, UNSHFLI, shfl_uimm>;
 
 def : PatGprGpr<int_riscv_xperm_n, XPERM4>;
 def : PatGprGpr<int_riscv_xperm_b, XPERM8>;
 def : PatGprGpr<int_riscv_xperm_h, XPERM_H>;
 } // Predicates = [HasStdExtZbp]

 def : Pat<(i32 (riscv_grev GPR:$rs1, 24)), (REV8_RV32 GPR:$rs1)>;
 } // Predicates = [HasStdExtZbp, IsRV32]
 
 let Predicates = [HasStdExtZbp, IsRV64] in {
 // We treat rev8 as a separate instruction, so match it directly.
 def : Pat<(i64 (riscv_grev GPR:$rs1, 56)), (REV8_RV64 GPR:$rs1)>;
 } // Predicates = [HasStdExtZbp, IsRV64]
 
-let Predicates = [HasStdExtZbt] in {
+let Predicates = [HasStdExtZbtOrZbpbo] in {
 def : Pat<(XLenVT (or (and (not GPR:$rs2), GPR:$rs3), (and GPR:$rs2, GPR:$rs1))),
           (CMIX GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 def : Pat<(XLenVT (xor (and (xor GPR:$rs1, GPR:$rs3), GPR:$rs2), GPR:$rs3)),
           (CMIX GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
+} // Predicates = [HasStdExtZbtOrZbpbo]
 
+let Predicates = [HasStdExtZbt] in {
 def : Pat<(XLenVT (select (XLenVT (setne (XLenVT GPR:$rs2), 0)), GPR:$rs1, GPR:$rs3)),
           (CMOV GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 def : Pat<(XLenVT (select (XLenVT (seteq (XLenVT GPR:$rs2), 0)), GPR:$rs3, GPR:$rs1)),
           (CMOV GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 def : Pat<(XLenVT (select (XLenVT (setne (XLenVT GPR:$x), simm12_plus1:$y)), GPR:$rs1, GPR:$rs3)),
           (CMOV GPR:$rs1, (ADDI GPR:$x, (NegImm simm12_plus1:$y)), GPR:$rs3)>;
 def : Pat<(XLenVT (select (XLenVT (seteq (XLenVT GPR:$x), simm12_plus1:$y)), GPR:$rs3, GPR:$rs1)),
           (CMOV GPR:$rs1, (ADDI GPR:$x, (NegImm simm12_plus1:$y)), GPR:$rs3)>;
def : Pat<(select (XLenVT (setgt (XLenVT GPR:$x), simm12_minus1_nonzero:$imm)), (XLenVT GPR:$rs3), GPR:$rs1),
           (CMOV GPR:$rs1, (SLTI GPR:$x, (ImmPlus1 simm12_minus1_nonzero:$imm)), GPR:$rs3)>;
 def : Pat<(select (XLenVT (setugt (XLenVT GPR:$x), simm12_minus1_nonzero:$imm)), (XLenVT GPR:$rs3), GPR:$rs1),
           (CMOV GPR:$rs1, (SLTIU GPR:$x, (ImmPlus1 simm12_minus1_nonzero:$imm)), GPR:$rs3)>;
 
 def : Pat<(XLenVT (select (XLenVT GPR:$rs2), GPR:$rs1, GPR:$rs3)),
           (CMOV GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 } // Predicates = [HasStdExtZbt]
 
-let Predicates = [HasStdExtZbt] in {
+let Predicates = [HasStdExtZbt] in
 def : Pat<(XLenVT (riscv_fsl GPR:$rs1, GPR:$rs3, (XLenVT GPR:$rs2))),
           (FSL GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
+
+let Predicates = [HasStdExtZbtOrZbpbo] in {
 def : Pat<(XLenVT (riscv_fsr GPR:$rs1, GPR:$rs3, (XLenVT GPR:$rs2))),
           (FSR GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 def : Pat<(XLenVT (riscv_fsr GPR:$rs1, GPR:$rs3, uimmlog2xlen:$shamt)),
           (FSRI GPR:$rs1, GPR:$rs3, uimmlog2xlen:$shamt)>;
 // We can use FSRI for FSL by immediate if we subtract the immediate from
 // XLen and swap the operands.
 def : Pat<(XLenVT (riscv_fsl GPR:$rs3, GPR:$rs1, uimmlog2xlen:$shamt)),
           (FSRI GPR:$rs1, GPR:$rs3, (ImmSubFromXLen uimmlog2xlen:$shamt))>;
-} // Predicates = [HasStdExtZbt]
+} // Predicates = [HasStdExtZbtOrZbpbo]
+
+let Predicates = [HasStdExtZbtOrZbpbo, IsRV64] in
+def : Pat<(riscv_fsrw GPR:$rs1, GPR:$rs3, GPR:$rs2),
+          (FSRW GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 
 let Predicates = [HasStdExtZbt, IsRV64] in {
 def : Pat<(riscv_fslw GPR:$rs1, GPR:$rs3, GPR:$rs2),
           (FSLW GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
-def : Pat<(riscv_fsrw GPR:$rs1, GPR:$rs3, GPR:$rs2),
-          (FSRW GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
 def : Pat<(riscv_fsrw GPR:$rs1, GPR:$rs3, uimm5:$shamt),
           (FSRIW GPR:$rs1, GPR:$rs3, uimm5:$shamt)>;
 // We can use FSRIW for FSLW by immediate if we subtract the immediate from
 // 32 and swap the operands.
 def : Pat<(riscv_fslw GPR:$rs3, GPR:$rs1, uimm5:$shamt),
           (FSRIW GPR:$rs1, GPR:$rs3, (ImmSubFrom32 uimm5:$shamt))>;
 } // Predicates = [HasStdExtZbt, IsRV64]
 
-let Predicates = [HasStdExtZbb] in {
+let Predicates = [HasStdExtZbbOrZbpbo] in
 def : PatGpr<ctlz, CLZ>;
+
+let Predicates = [HasStdExtZbb] in {
 def : PatGpr<cttz, CTZ>;
 def : PatGpr<ctpop, CPOP>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
 def : PatGpr<riscv_clzw, CLZW>;
 def : PatGpr<riscv_ctzw, CTZW>;
 def : Pat<(i64 (ctpop (i64 (zexti32 (i64 GPR:$rs1))))), (CPOPW GPR:$rs1)>;
 } // Predicates = [HasStdExtZbb, IsRV64]
 
 let Predicates = [HasStdExtZbb] in {
 def : Pat<(sext_inreg GPR:$rs1, i8), (SEXT_B GPR:$rs1)>;
 def : Pat<(sext_inreg GPR:$rs1, i16), (SEXT_H GPR:$rs1)>;
 } // Predicates = [HasStdExtZbb]
 
-let Predicates = [HasStdExtZbb] in {
+let Predicates = [HasStdExtZbbOrZbpbo] in {
 def : PatGprGpr<smin, MIN>;
 def : PatGprGpr<smax, MAX>;
+} // Predicates = [HasStdExtZbbOrZbpbo]
+
+let Predicates = [HasStdExtZbb] in {
 def : PatGprGpr<umin, MINU>;
 def : PatGprGpr<umax, MAXU>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbbOrZbkb, IsRV32] in {
 def : Pat<(i32 (bswap GPR:$rs1)), (REV8_RV32 GPR:$rs1)>;
 } // Predicates = [HasStdExtZbbOrZbkb, IsRV32]
 
 let Predicates = [HasStdExtZbbOrZbkb, IsRV64] in {
 def : Pat<(i64 (bswap GPR:$rs1)), (REV8_RV64 GPR:$rs1)>;
 } // Predicates = [HasStdExtZbbOrZbkb, IsRV64]
 
 let Predicates = [HasStdExtZbpOrZbkb] in {
 def : Pat<(XLenVT (or (and (shl GPR:$rs2, (XLenVT 8)), 0xFFFF),
                       (and GPR:$rs1, 0x00FF))),
           (PACKH GPR:$rs1, GPR:$rs2)>;
 def : Pat<(XLenVT (or (shl (and GPR:$rs2, 0x00FF), (XLenVT 8)),
                       (and GPR:$rs1, 0x00FF))),
           (PACKH GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtZbpOrZbkb]
 
-let Predicates = [HasStdExtZbpOrZbkb, IsRV32] in
+let Predicates = [HasStdExtZbpOrZbkbOrZbpbo, IsRV32] in
 def : Pat<(i32 (or (and GPR:$rs1, 0x0000FFFF), (shl GPR:$rs2, (i32 16)))),
           (PACK GPR:$rs1, GPR:$rs2)>;
 
-let Predicates = [HasStdExtZbpOrZbkb, IsRV64] in {
+let Predicates = [HasStdExtZbpOrZbkbOrZbpbo, IsRV64] in
 def : Pat<(i64 (or (and GPR:$rs1, 0x00000000FFFFFFFF), (shl GPR:$rs2, (i64 32)))),
           (PACK GPR:$rs1, GPR:$rs2)>;
 
+let Predicates = [HasStdExtZbpOrZbkb, IsRV64] in {
 def : Pat<(i64 (sext_inreg (or (shl GPR:$rs2, (i64 16)),
                                (and GPR:$rs1, 0x000000000000FFFF)),
                            i32)),
           (PACKW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i64 (or (sext_inreg (shl GPR:$rs2, (i64 16)), i32),
                    (and GPR:$rs1, 0x000000000000FFFF))),
           (PACKW GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtZbpOrZbkb, IsRV64]
 
-let Predicates = [HasStdExtZbp, IsRV32] in
+let Predicates = [HasStdExtZbpOrZbpbo, IsRV32] in
 def : Pat<(i32 (or (and GPR:$rs2, 0xFFFF0000), (srl GPR:$rs1, (i32 16)))),
           (PACKU GPR:$rs1, GPR:$rs2)>;
 
-let Predicates = [HasStdExtZbp, IsRV64] in {
+let Predicates = [HasStdExtZbpOrZbpbo, IsRV64] in
 def : Pat<(i64 (or (and GPR:$rs2, 0xFFFFFFFF00000000), (srl GPR:$rs1, (i64 32)))),
           (PACKU GPR:$rs1, GPR:$rs2)>;
 
+let Predicates = [HasStdExtZbp, IsRV64] in
 def : Pat<(i64 (or (and (assertsexti32 GPR:$rs2), 0xFFFFFFFFFFFF0000),
                    (srl (and GPR:$rs1, 0xFFFFFFFF), (i64 16)))),
           (PACKUW GPR:$rs1, GPR:$rs2)>;
-} // Predicates = [HasStdExtZbp, IsRV64]
 
 let Predicates = [HasStdExtZbbOrZbp, IsRV32] in
 def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (ZEXT_H_RV32 GPR:$rs)>;
 let Predicates = [HasStdExtZbbOrZbp, IsRV64] in
 def : Pat<(i64 (and GPR:$rs, 0xFFFF)), (ZEXT_H_RV64 GPR:$rs)>;
 
 let Predicates = [HasStdExtZba] in {
 def : Pat<(add (shl GPR:$rs1, (XLenVT 1)), non_imm12:$rs2),

llvm/lib/Target/RISCV/RISCVSubtarget.h

private:
   bool HasStdExtZbc = false;
   bool HasStdExtZbe = false;
   bool HasStdExtZbf = false;
   bool HasStdExtZbm = false;
   bool HasStdExtZbp = false;
   bool HasStdExtZbr = false;
   bool HasStdExtZbs = false;
   bool HasStdExtZbt = false;
+  bool HasStdExtZbpbo = false;
   bool HasStdExtV = false;
   bool HasStdExtZve32x = false;
   bool HasStdExtZve32f = false;
   bool HasStdExtZve64x = false;
   bool HasStdExtZve64f = false;
   bool HasStdExtZve64d = false;
   bool HasStdExtZvfh = false;
   bool HasStdExtZfhmin = false;
public:
   bool hasStdExtZihintpause() const { return HasStdExtZihintpause; }
   bool hasStdExtZba() const { return HasStdExtZba; }
   bool hasStdExtZbb() const { return HasStdExtZbb; }
   bool hasStdExtZbc() const { return HasStdExtZbc; }
   bool hasStdExtZbe() const { return HasStdExtZbe; }
   bool hasStdExtZbf() const { return HasStdExtZbf; }
   bool hasStdExtZbm() const { return HasStdExtZbm; }
   bool hasStdExtZbp() const { return HasStdExtZbp; }
+  bool hasStdExtZbpbo() const { return HasStdExtZbpbo; }
   bool hasStdExtZbr() const { return HasStdExtZbr; }
   bool hasStdExtZbs() const { return HasStdExtZbs; }
   bool hasStdExtZbt() const { return HasStdExtZbt; }
   bool hasStdExtZvl() const { return ZvlLen != 0; }
   bool hasStdExtZvfh() const { return HasStdExtZvfh; }
   bool hasStdExtZfhmin() const { return HasStdExtZfhmin; }
   bool hasStdExtZfh() const { return HasStdExtZfh; }
   bool hasStdExtZfinx() const { return HasStdExtZfinx; }

llvm/test/CodeGen/RISCV/attributes.ll

 ; RUN: llc -mtriple=riscv32 -mattr=+zk %s -o - | FileCheck --check-prefix=RV32ZK %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zkn,+zkr,+zkt %s -o - | FileCheck --check-prefix=RV32COMBINEINTOZK %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zbkb,+zbkc,+zbkx,+zkne,+zknd,+zknh %s -o - | FileCheck --check-prefix=RV32COMBINEINTOZKN %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zbkb,+zbkc,+zbkx,+zksed,+zksh %s -o - | FileCheck --check-prefix=RV32COMBINEINTOZKS %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zicbom %s -o - | FileCheck --check-prefix=RV32ZICBOM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zicboz %s -o - | FileCheck --check-prefix=RV32ZICBOZ %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zicbop %s -o - | FileCheck --check-prefix=RV32ZICBOP %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-p %s -o - | FileCheck --check-prefix=RV32P %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbpbo %s -o - | FileCheck --check-prefix=RV32ZBPBO %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zpn %s -o - | FileCheck --check-prefix=RV32ZPN %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zpsfoperand %s -o - | FileCheck --check-prefix=RV32ZPSFOPERAND %s
 ; RUN: llc -mtriple=riscv64 -mattr=+m %s -o - | FileCheck --check-prefix=RV64M %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a %s -o - | FileCheck --check-prefix=RV64A %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f %s -o - | FileCheck --check-prefix=RV64F %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d %s -o - | FileCheck --check-prefix=RV64D %s
 ; RUN: llc -mtriple=riscv64 -mattr=+c %s -o - | FileCheck --check-prefix=RV64C %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zihintpause %s -o - | FileCheck --check-prefix=RV64ZIHINTPAUSE %s

 ; RUN: llc -mtriple=riscv64 -mattr=+zk %s -o - | FileCheck --check-prefix=RV64ZK %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zkn,+zkr,+zkt %s -o - | FileCheck --check-prefix=RV64COMBINEINTOZK %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zbkb,+zbkc,+zbkx,+zkne,+zknd,+zknh %s -o - | FileCheck --check-prefix=RV64COMBINEINTOZKN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zbkb,+zbkc,+zbkx,+zksed,+zksh %s -o - | FileCheck --check-prefix=RV64COMBINEINTOZKS %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zicbom %s -o - | FileCheck --check-prefix=RV64ZICBOM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zicboz %s -o - | FileCheck --check-prefix=RV64ZICBOZ %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zicbop %s -o - | FileCheck --check-prefix=RV64ZICBOP %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-p %s -o - | FileCheck --check-prefix=RV64P %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbpbo %s -o - | FileCheck --check-prefix=RV64ZBPBO %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zpn %s -o - | FileCheck --check-prefix=RV64ZPN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zpsfoperand %s -o - | FileCheck --check-prefix=RV64ZPSFOPERAND %s
 
 ; RV32M: .attribute 5, "rv32i2p0_m2p0"
 ; RV32A: .attribute 5, "rv32i2p0_a2p0"
 ; RV32F: .attribute 5, "rv32i2p0_f2p0"
 ; RV32D: .attribute 5, "rv32i2p0_f2p0_d2p0"
 ; RV32C: .attribute 5, "rv32i2p0_c2p0"

 ; RV32ZK: .attribute 5, "rv32i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zk1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0_zkr1p0_zkt1p0"
 ; RV32COMBINEINTOZK: .attribute 5, "rv32i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zk1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0_zkr1p0_zkt1p0"
 ; RV32COMBINEINTOZKN: .attribute 5, "rv32i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0"
 ; RV32COMBINEINTOZKS: .attribute 5, "rv32i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zks1p0_zksed1p0_zksh1p0"
 ; RV32ZICBOM: .attribute 5, "rv32i2p0_zicbom1p0"
 ; RV32ZICBOZ: .attribute 5, "rv32i2p0_zicboz1p0"
 ; RV32ZICBOP: .attribute 5, "rv32i2p0_zicbop1p0"
 ; RV32P: .attribute 5, "rv32i2p0_p0p911_zbpbo0p911_zpn0p911_zpsfoperand0p911"
+; RV32ZBPBO: .attribute 5, "rv32i2p0_zbpbo0p911"
 ; RV32ZPN: .attribute 5, "rv32i2p0_zpn0p911"
 ; RV32ZPSFOPERAND: .attribute 5, "rv32i2p0_zpsfoperand0p911"
 
 ; RV64M: .attribute 5, "rv64i2p0_m2p0"
 ; RV64A: .attribute 5, "rv64i2p0_a2p0"
 ; RV64F: .attribute 5, "rv64i2p0_f2p0"
 ; RV64D: .attribute 5, "rv64i2p0_f2p0_d2p0"
 ; RV64C: .attribute 5, "rv64i2p0_c2p0"

 ; RV64ZK: .attribute 5, "rv64i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zk1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0_zkr1p0_zkt1p0"
 ; RV64COMBINEINTOZK: .attribute 5, "rv64i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zk1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0_zkr1p0_zkt1p0"
 ; RV64COMBINEINTOZKN: .attribute 5, "rv64i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zkn1p0_zknd1p0_zkne1p0_zknh1p0"
 ; RV64COMBINEINTOZKS: .attribute 5, "rv64i2p0_zbkb1p0_zbkc1p0_zbkx1p0_zks1p0_zksed1p0_zksh1p0"
 ; RV64ZICBOM: .attribute 5, "rv64i2p0_zicbom1p0"
 ; RV64ZICBOZ: .attribute 5, "rv64i2p0_zicboz1p0"
 ; RV64ZICBOP: .attribute 5, "rv64i2p0_zicbop1p0"
 ; RV64P: .attribute 5, "rv64i2p0_p0p911_zbpbo0p911_zpn0p911_zpsfoperand0p911"
+; RV64ZBPBO: .attribute 5, "rv64i2p0_zbpbo0p911"
 ; RV64ZPN: .attribute 5, "rv64i2p0_zpn0p911"
 ; RV64ZPSFOPERAND: .attribute 5, "rv64i2p0_zpsfoperand0p911"
 
 define i32 @addi(i32 %a) {
   %1 = add i32 %a, 1
   ret i32 %1
 }

llvm/test/CodeGen/RISCV/rv32zbpbo-intrinsics.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbpbo -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZBPBO
+
+declare i32 @llvm.riscv.fsr.i32(i32, i32, i32)
+
+define i32 @fsr_i32(i32 %a, i32 %b, i32 %c) nounwind {
+; RV32ZBPBO-LABEL: fsr_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    fsr a0, a0, a1, a2
+; RV32ZBPBO-NEXT:    ret
+  %1 = call i32 @llvm.riscv.fsr.i32(i32 %a, i32 %b, i32 %c)
+  ret i32 %1
+}
+
+define i32 @fsri_i32(i32 %a, i32 %b) nounwind {
+; RV32ZBPBO-LABEL: fsri_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    fsri a0, a0, a1, 15
+; RV32ZBPBO-NEXT:    ret
+  %1 = call i32 @llvm.riscv.fsr.i32(i32 %a, i32 %b, i32 15)
+  ret i32 %1
+}
+
+declare i32 @llvm.riscv.grev.i32(i32 %a, i32 %b)
+
+define i32 @revi32(i32 %a) nounwind {
+; RV32ZBPBO-LABEL: revi32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    rev a0, a0
+; RV32ZBPBO-NEXT:    ret
+  %tmp = call i32 @llvm.riscv.grev.i32(i32 %a, i32 31)
+  ret i32 %tmp
+}

llvm/test/CodeGen/RISCV/rv32zbpbo.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbpbo -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32ZBPBO
+
+declare i32 @llvm.ctlz.i32(i32, i1)
+
+define i32 @ctlz_i32(i32 %a) nounwind {
+; CHECK-LABEL: ctlz_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    beqz a0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %cond.false
+; CHECK-NEXT:    srli a1, a0, 1
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    srli a1, a0, 4
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    srli a1, a0, 8
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    srli a1, a0, 16
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    not a0, a0
+; CHECK-NEXT:    srli a1, a0, 1
+; CHECK-NEXT:    lui a2, 349525
+; CHECK-NEXT:    addi a2, a2, 1365
+; CHECK-NEXT:    and a1, a1, a2
+; CHECK-NEXT:    sub a0, a0, a1
+; CHECK-NEXT:    lui a1, 209715
+; CHECK-NEXT:    addi a1, a1, 819
+; CHECK-NEXT:    and a2, a0, a1
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    add a0, a2, a0
+; CHECK-NEXT:    srli a1, a0, 4
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    lui a1, 61681
+; CHECK-NEXT:    addi a1, a1, -241
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    lui a1, 4112
+; CHECK-NEXT:    addi a1, a1, 257
+; CHECK-NEXT:    call __mulsi3@plt
+; CHECK-NEXT:    srli a0, a0, 24
+; CHECK-NEXT:    j .LBB0_3
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:  .LBB0_3: # %cond.end
+; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+; RV32I-LABEL: ctlz_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    beqz a0, .LBB0_2
+; RV32I-NEXT:  # %bb.1: # %cond.false
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 8
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    not a0, a0
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    lui a1, 209715
+; RV32I-NEXT:    addi a1, a1, 819
+; RV32I-NEXT:    and a2, a0, a1
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    add a0, a2, a0
+; RV32I-NEXT:    srli a1, a0, 4
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lui a1, 61681
+; RV32I-NEXT:    addi a1, a1, -241
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi a1, a1, 257
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    li a0, 32
+; RV32I-NEXT:    ret
+;
+; RV32ZBPBO-LABEL: ctlz_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    beqz a0, .LBB0_2
+; RV32ZBPBO-NEXT:  # %bb.1: # %cond.false
+; RV32ZBPBO-NEXT:    clz a0, a0
+; RV32ZBPBO-NEXT:    ret
+; RV32ZBPBO-NEXT:  .LBB0_2:
+; RV32ZBPBO-NEXT:    li a0, 32
+; RV32ZBPBO-NEXT:    ret
+  %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+  ret i32 %1
+}
+
+define i32 @pack_i32(i32 %a, i32 %b) nounwind {
+; RV32I-LABEL: pack_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBPBO-LABEL: pack_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    pkbb16 a0, a0, a1
+; RV32ZBPBO-NEXT:    ret
+  %shl = and i32 %a, 65535
+  %shl1 = shl i32 %b, 16
+  %or = or i32 %shl1, %shl
+  ret i32 %or
+}
+
+define i32 @packu_i32(i32 %a, i32 %b) nounwind {
+; RV32I-LABEL: packu_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    lui a2, 1048560
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBPBO-LABEL: packu_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    pktt16 a0, a0, a1
+; RV32ZBPBO-NEXT:    ret
+  %shr = lshr i32 %a, 16
+  %shr1 = and i32 %b, -65536
+  %or = or i32 %shr1, %shr
+  ret i32 %or
+}
+
+define i32 @max_i32(i32 %a, i32 %b) nounwind {
+; RV32I-LABEL: max_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    blt a1, a0, .LBB3_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    ret
+;
+; RV32ZBPBO-LABEL: max_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    max a0, a0, a1
+; RV32ZBPBO-NEXT:    ret
+  %cmp = icmp sgt i32 %a, %b
+  %cond = select i1 %cmp, i32 %a, i32 %b
+  ret i32 %cond
+}
+
+define i32 @min_i32(i32 %a, i32 %b) nounwind {
+; RV32I-LABEL: min_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    blt a0, a1, .LBB4_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    ret
+;
+; RV32ZBPBO-LABEL: min_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    min a0, a0, a1
+; RV32ZBPBO-NEXT:    ret
+  %cmp = icmp slt i32 %a, %b
+  %cond = select i1 %cmp, i32 %a, i32 %b
+  ret i32 %cond
+}
+
+declare i16 @llvm.bswap.i16(i16)
+
+define zeroext i16 @bswap_i16(i16 zeroext %a) nounwind {
+; RV32I-LABEL: bswap_i16:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a1, a0, 8
+; RV32I-NEXT:    slli a0, a0, 8
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a0, a0, 16
+; RV32I-NEXT:    srli a0, a0, 16
+; RV32I-NEXT:    ret
+;
+; RV32ZBPBO-LABEL: bswap_i16:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    rev8.h a0, a0
+; RV32ZBPBO-NEXT:    ret
+  %1 = tail call i16 @llvm.bswap.i16(i16 %a)
+  ret i16 %1
+}
+
+define i32 @cmix_i32(i32 %a, i32 %b, i32 %c) nounwind {
+; RV32I-LABEL: cmix_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    not a1, a1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBPBO-LABEL: cmix_i32:
+; RV32ZBPBO:       # %bb.0:
+; RV32ZBPBO-NEXT:    cmix a0, a1, a0, a2
+; RV32ZBPBO-NEXT:    ret
+  %and = and i32 %b, %a
+  %neg = xor i32 %b, -1
+  %and1 = and i32 %neg, %c
+  %or = or i32 %and1, %and
+  ret i32 %or
+}

llvm/test/CodeGen/RISCV/rv64zbpbo-intrinsics.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbpbo -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZBPBO
+
+declare i32 @llvm.riscv.fsr.i32(i32, i32, i32)
+
+define i32 @fsr_i32(i32 %a, i32 %b, i32 %c) nounwind {
+; RV64ZBPBO-LABEL: fsr_i32:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    fsrw a0, a0, a1, a2
+; RV64ZBPBO-NEXT:    ret
+  %1 = call i32 @llvm.riscv.fsr.i32(i32 %a, i32 %b, i32 %c)
+  ret i32 %1
+}
+
+declare i64 @llvm.riscv.grev.i64(i64 %a, i64 %b)
+
+define i64 @revi64(i64 %a) nounwind {
+; RV64ZBPBO-LABEL: revi64:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    rev a0, a0
+; RV64ZBPBO-NEXT:    ret
+  %tmp = call i64 @llvm.riscv.grev.i64(i64 %a, i64 63)
+  ret i64 %tmp
+}

llvm/test/CodeGen/RISCV/rv64zbpbo.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbpbo -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ZBPBO
+
+define i64 @pack_i64(i64 %a, i64 %b) nounwind {
+; RV64I-LABEL: pack_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBPBO-LABEL: pack_i64:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    pkbb32 a0, a0, a1
+; RV64ZBPBO-NEXT:    ret
+  %shl = and i64 %a, 4294967295
+  %shl1 = shl i64 %b, 32
+  %or = or i64 %shl1, %shl
+  ret i64 %or
+}
+
+define i64 @packu_i64(i64 %a, i64 %b) nounwind {
+; RV64I-LABEL: packu_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBPBO-LABEL: packu_i64:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    pktt32 a0, a0, a1
+; RV64ZBPBO-NEXT:    ret
+  %shr = lshr i64 %a, 32
+  %shr1 = and i64 %b, -4294967296
+  %or = or i64 %shr1, %shr
+  ret i64 %or
+}
+
+define i64 @max_i64(i64 %a, i64 %b) nounwind {
+; RV64I-LABEL: max_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    blt a1, a0, .LBB2_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:  .LBB2_2:
+; RV64I-NEXT:    ret
+;
+; RV64ZBPBO-LABEL: max_i64:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    max a0, a0, a1
+; RV64ZBPBO-NEXT:    ret
+  %cmp = icmp sgt i64 %a, %b
+  %cond = select i1 %cmp, i64 %a, i64 %b
+  ret i64 %cond
+}
+
+define i64 @min_i64(i64 %a, i64 %b) nounwind {
+; RV64I-LABEL: min_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    blt a0, a1, .LBB3_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:  .LBB3_2:
+; RV64I-NEXT:    ret
+;
+; RV64ZBPBO-LABEL: min_i64:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    min a0, a0, a1
+; RV64ZBPBO-NEXT:    ret
+  %cmp = icmp slt i64 %a, %b
+  %cond = select i1 %cmp, i64 %a, i64 %b
+  ret i64 %cond
+}
+
+declare i16 @llvm.bswap.i16(i16)
+
+define zeroext i16 @bswap_i16(i16 zeroext %a) nounwind {
+; RV64I-LABEL: bswap_i16:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srli a1, a0, 8
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 48
+; RV64I-NEXT:    ret
+;
+; RV64ZBPBO-LABEL: bswap_i16:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    rev8.h a0, a0
+; RV64ZBPBO-NEXT:    ret
+  %1 = tail call i16 @llvm.bswap.i16(i16 %a)
+  ret i16 %1
+}
+
+define i64 @cmix_i64(i64 %a, i64 %b, i64 %c) nounwind {
+; RV64I-LABEL: cmix_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    not a1, a1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBPBO-LABEL: cmix_i64:
+; RV64ZBPBO:       # %bb.0:
+; RV64ZBPBO-NEXT:    cmix a0, a1, a0, a2
+; RV64ZBPBO-NEXT:    ret
+  %and = and i64 %b, %a
+  %neg = xor i64 %b, -1
+  %and1 = and i64 %neg, %c
+  %or = or i64 %and1, %and
+  ret i64 %or
+}

llvm/test/MC/RISCV/rv32zbpbo-aliases-valid.s

+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zbpbo -riscv-no-aliases \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ-NOALIAS %s
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zbpbo \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-zbpbo < %s \
+# RUN:     | llvm-objdump -d -r -M no-aliases --mattr=+experimental-zbpbo - \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ-NOALIAS %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-zbpbo < %s \
+# RUN:     | llvm-objdump -d -r --mattr=+experimental-zbpbo - \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ %s
+
+# CHECK-S-OBJ-NOALIAS: grevi t0, t1, 8
+# CHECK-S-OBJ: rev8.h t0, t1
+rev8.h x5, x6
+
+# CHECK-S-OBJ-NOALIAS: grevi t0, t1, 31
+# CHECK-S-OBJ: rev t0, t1
+rev x5, x6
+
+# CHECK-S-OBJ-NOALIAS: pack t0, t1, t2
+# CHECK-S-OBJ: pkbb16 t0, t1, t2
+pkbb16 x5, x6, x7
+
+# CHECK-S-OBJ-NOALIAS: packu t0, t1, t2
+# CHECK-S-OBJ: pktt16 t0, t1, t2
+pktt16 x5, x6, x7

llvm/test/MC/RISCV/rv32zbpbo-valid.s

+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zbpbo -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-zbpbo < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-zbpbo -M no-aliases -d -r - \
+# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+
+# CHECK-ASM-AND-OBJ: clz t0, t1
+# CHECK-ASM: encoding: [0x93,0x12,0x03,0x60]
+clz t0, t1
+
+# CHECK-ASM-AND-OBJ: pack t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x42,0x73,0x08]
+pack t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: packu t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x42,0x73,0x48]
+packu t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: fsr t0, t1, t2, t3
+# CHECK-ASM: encoding: [0xb3,0x52,0xc3,0x3d]
+fsr t0, t1, t2, t3
+
+# CHECK-ASM-AND-OBJ: fsri t0, t1, t2, 0
+# CHECK-ASM: encoding: [0x93,0x52,0x03,0x3c]
+fsri t0, t1, t2, 0
+
+# CHECK-ASM-AND-OBJ: max t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x62,0x73,0x0a]
+max t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: min t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x42,0x73,0x0a]
+min t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: cmix t0, t1, t2, t3
+# CHECK-ASM: encoding: [0xb3,0x92,0x63,0xe6]
+cmix t0, t1, t2, t3

llvm/test/MC/RISCV/rv64zbpbo-aliases-valid.s

+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zbpbo -riscv-no-aliases \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ-NOALIAS %s
+# RUN: llvm-mc %s  -triple=riscv64 -mattr=+experimental-zbpbo \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-zbpbo < %s \
+# RUN:     | llvm-objdump -d -r -M no-aliases --mattr=+experimental-zbpbo - \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ-NOALIAS %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-zbpbo < %s \
+# RUN:     | llvm-objdump -d -r --mattr=+experimental-zbpbo - \
+# RUN:     | FileCheck -check-prefixes=CHECK-S-OBJ %s
+
+# CHECK-S-OBJ-NOALIAS: grevi t0, t1, 8
+# CHECK-S-OBJ: rev8.h t0, t1
+rev8.h x5, x6
+
+# CHECK-S-OBJ-NOALIAS: grevi t0, t1, 63
+# CHECK-S-OBJ: rev t0, t1
+rev x5, x6

llvm/test/MC/RISCV/rv64zbpbo-valid.s

+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zbpbo -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zbpbo < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-zbpbo -M no-aliases -d -r - \
+# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+
+# CHECK-ASM-AND-OBJ: pack t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x42,0x73,0x08]
+pack t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: packu t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x42,0x73,0x48]
+packu t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: fsrw t0, t1, t2, t3
+# CHECK-ASM: encoding: [0xbb,0x52,0xc3,0x3d]
+fsrw t0, t1, t2, t3
+
+# CHECK-ASM-AND-OBJ: max t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x62,0x73,0x0a]
+max t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: min t0, t1, t2
+# CHECK-ASM: encoding: [0xb3,0x42,0x73,0x0a]
+min t0, t1, t2
+
+# CHECK-ASM-AND-OBJ: cmix t0, t1, t2, t3
+# CHECK-ASM: encoding: [0xb3,0x92,0x63,0xe6]
+cmix t0, t1, t2, t3