[AArch64][SVEIntrinsicOpts] Convect cntb/h/w/d to vscale intrinsic or constant.

[junparser]

Summary

As is mentioned above

[junparser]

clang-format.

[junparser]

update clang test.

[junparser]

@sdesmalen @david-arm @paulwalker-arm kindly ping~

[david-arm]

Hi @junparser, the patch looks sensible to me! I just had a couple of minor comments if that's ok.

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

658

nit: Do you need to fix the clang-tidy warning here?

662

Could you potentially fold these two cases into one somehow? Maybe with a switch-case statement? I'm just imagining a situation where we might want other patterns too like vl32, vl64, etc.

[junparser]

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

662

There is no other special pattern except vl16. But I do think switch-case is more straightforward

[junparser]

Address comments.

[david-arm]

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

662

OK, thanks for making this a switch statement. I was just thinking that in the developer manual we say we also support vl1-vl256 so at some point we may add more enums in LLVM too.

671

I was actually wondering if we could commonise this code somehow. Perhaps by setting a MinNumElts variable in the case statements, i.e.

unsigned MinNumElts;
...
case AArch64SVEPredPattern::vl8:
  MinNumElts = Pattern;
  break;
case AArch64SVEPredPattern::vl16:
  MinNumElts = 16;
  break;
}

if (NumElts < MinNumElts) return None;

return Optional<Instruction *>(IC.replaceInstUsesWith(
                             II, ConstantInt::get(II.getType(), NumElts)));

[junparser]

address comments.

[david-arm]

LGTM! Thanks a lot for making the changes. :)

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntb.c

 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null
 #include <arm_sve.h>
 
 uint64_t test_svcntb()
 {
   // CHECK-LABEL: test_svcntb
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 4
+  // CHECK: ret i64 %[[RET]]
   return svcntb();
 }
 
 uint64_t test_svcntb_pat()
 {
   // CHECK-LABEL: test_svcntb_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntb_pat(SV_POW2);
 }
 
 uint64_t test_svcntb_pat_1()
 {
   // CHECK-LABEL: test_svcntb_pat_1
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 1)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 1
   return svcntb_pat(SV_VL1);
 }
 
 uint64_t test_svcntb_pat_2()
 {
   // CHECK-LABEL: test_svcntb_pat_2
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 2)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 2
   return svcntb_pat(SV_VL2);
 }
 
 uint64_t test_svcntb_pat_3()
 {
   // CHECK-LABEL: test_svcntb_pat_3
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 3)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 3
   return svcntb_pat(SV_VL3);
 }
 
 uint64_t test_svcntb_pat_4()
 {
   // CHECK-LABEL: test_svcntb_pat_4
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 4)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 4
   return svcntb_pat(SV_VL4);
 }
 
 uint64_t test_svcntb_pat_5()
 {
   // CHECK-LABEL: test_svcntb_pat_5
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 5)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 5
   return svcntb_pat(SV_VL5);
 }
 
 uint64_t test_svcntb_pat_6()
 {
   // CHECK-LABEL: test_svcntb_pat_6
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 6)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 6
   return svcntb_pat(SV_VL6);
 }
 
 uint64_t test_svcntb_pat_7()
 {
   // CHECK-LABEL: test_svcntb_pat_7
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 7)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 7
   return svcntb_pat(SV_VL7);
 }
 
 uint64_t test_svcntb_pat_8()
 {
   // CHECK-LABEL: test_svcntb_pat_8
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 8)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 8
   return svcntb_pat(SV_VL8);
 }
 
 uint64_t test_svcntb_pat_9()
 {
   // CHECK-LABEL: test_svcntb_pat_9
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 9)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 16
   return svcntb_pat(SV_VL16);
 }
 
 uint64_t test_svcntb_pat_10()
 {
   // CHECK-LABEL: test_svcntb_pat_10
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 10)
   // CHECK: ret i64 %[[INTRINSIC]]
uint64_t test_svcntb_pat_15()
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 30)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntb_pat(SV_MUL3);
 }
 
 uint64_t test_svcntb_pat_16()
 {
   // CHECK-LABEL: test_svcntb_pat_16
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 4
+  // CHECK: ret i64 %[[RET]]
   return svcntb_pat(SV_ALL);
 }

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntd.c

 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null
 #include <arm_sve.h>
 
 uint64_t test_svcntd()
 {
   // CHECK-LABEL: test_svcntd
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 1
+  // CHECK: ret i64 %[[RET]]
   return svcntd();
 }
 
 uint64_t test_svcntd_pat()
 {
   // CHECK-LABEL: test_svcntd_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntd_pat(SV_POW2);
 }
 
 uint64_t test_svcntd_pat_1()
 {
   // CHECK-LABEL: test_svcntd_pat_1
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 1)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 1
   return svcntd_pat(SV_VL1);
 }
 
 uint64_t test_svcntd_pat_2()
 {
   // CHECK-LABEL: test_svcntd_pat_2
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 2)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 2
   return svcntd_pat(SV_VL2);
 }
 
 uint64_t test_svcntd_pat_3()
 {
   // CHECK-LABEL: test_svcntd_pat_3
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 3)
   // CHECK: ret i64 %[[INTRINSIC]]
uint64_t test_svcntd_pat_15()
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 30)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntd_pat(SV_MUL3);
 }
 
 uint64_t test_svcntd_pat_16()
 {
   // CHECK-LABEL: test_svcntd_pat_16
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 1
+  // CHECK: ret i64 %[[RET]]
   return svcntd_pat(SV_ALL);
 }

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnth.c

 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null
 #include <arm_sve.h>
 
 uint64_t test_svcnth()
 {
   // CHECK-LABEL: test_svcnth
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 3
+  // CHECK: ret i64 %[[RET]]
   return svcnth();
 }
 
 uint64_t test_svcnth_pat()
 {
   // CHECK-LABEL: test_svcnth_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcnth_pat(SV_POW2);
 }
 
 uint64_t test_svcnth_pat_1()
 {
   // CHECK-LABEL: test_svcnth_pat_1
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 1)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 1
   return svcnth_pat(SV_VL1);
 }
 
 uint64_t test_svcnth_pat_2()
 {
   // CHECK-LABEL: test_svcnth_pat_2
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 2)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 2
   return svcnth_pat(SV_VL2);
 }
 
 uint64_t test_svcnth_pat_3()
 {
   // CHECK-LABEL: test_svcnth_pat_3
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 3)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 3
   return svcnth_pat(SV_VL3);
 }
 
 uint64_t test_svcnth_pat_4()
 {
   // CHECK-LABEL: test_svcnth_pat_4
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 4)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 4
   return svcnth_pat(SV_VL4);
 }
 
 uint64_t test_svcnth_pat_5()
 {
   // CHECK-LABEL: test_svcnth_pat_5
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 5)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 5
   return svcnth_pat(SV_VL5);
 }
 
 uint64_t test_svcnth_pat_6()
 {
   // CHECK-LABEL: test_svcnth_pat_6
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 6)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 6
   return svcnth_pat(SV_VL6);
 }
 
 uint64_t test_svcnth_pat_7()
 {
   // CHECK-LABEL: test_svcnth_pat_7
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 7)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 7
   return svcnth_pat(SV_VL7);
 }
 
 uint64_t test_svcnth_pat_8()
 {
   // CHECK-LABEL: test_svcnth_pat_8
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 8)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 8
   return svcnth_pat(SV_VL8);
 }
 
 uint64_t test_svcnth_pat_9()
 {
   // CHECK-LABEL: test_svcnth_pat_9
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 9)
   // CHECK: ret i64 %[[INTRINSIC]]
uint64_t test_svcnth_pat_15()
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 30)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcnth_pat(SV_MUL3);
 }
 
 uint64_t test_svcnth_pat_16()
 {
   // CHECK-LABEL: test_svcnth_pat_16
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 3
+  // CHECK: ret i64 %[[RET]]
   return svcnth_pat(SV_ALL);
 }

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntw.c

 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null
 #include <arm_sve.h>
 
 uint64_t test_svcntw()
 {
   // CHECK-LABEL: test_svcntw
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 2
+  // CHECK: ret i64 %[[RET]]
   return svcntw();
 }
 
 uint64_t test_svcntw_pat()
 {
   // CHECK-LABEL: test_svcntw_pat
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 0)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntw_pat(SV_POW2);
 }
 
 uint64_t test_svcntw_pat_1()
 {
   // CHECK-LABEL: test_svcntw_pat_1
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 1)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 1
   return svcntw_pat(SV_VL1);
 }
 
 uint64_t test_svcntw_pat_2()
 {
   // CHECK-LABEL: test_svcntw_pat_2
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 2)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 2
   return svcntw_pat(SV_VL2);
 }
 
 uint64_t test_svcntw_pat_3()
 {
   // CHECK-LABEL: test_svcntw_pat_3
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 3)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 3
   return svcntw_pat(SV_VL3);
 }
 
 uint64_t test_svcntw_pat_4()
 {
   // CHECK-LABEL: test_svcntw_pat_4
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 4)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: ret i64 4
   return svcntw_pat(SV_VL4);
 }
 
 uint64_t test_svcntw_pat_5()
 {
   // CHECK-LABEL: test_svcntw_pat_5
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 5)
   // CHECK: ret i64 %[[INTRINSIC]]
uint64_t test_svcntw_pat_15()
   // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 30)
   // CHECK: ret i64 %[[INTRINSIC]]
   return svcntw_pat(SV_MUL3);
 }
 
 uint64_t test_svcntw_pat_16()
 {
   // CHECK-LABEL: test_svcntw_pat_16
-  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 31)
-  // CHECK: ret i64 %[[INTRINSIC]]
+  // CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
+  // CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 2
+  // CHECK: ret i64 %[[RET]]
   return svcntw_pat(SV_ALL);
 }

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

static Optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
   auto *PTrue = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
                                         {II.getType()}, {AllPat});
   auto *RDFFR =
       Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue});
   RDFFR->takeName(&II);
   return IC.replaceInstUsesWith(II, RDFFR);
 }
 
+static Optional<Instruction *>
+instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
+  const auto Pattern = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue();
+
+  if (Pattern == AArch64SVEPredPattern::all) {
+    LLVMContext &Ctx = II.getContext();
+    IRBuilder<> Builder(Ctx);
+    Builder.SetInsertPoint(&II);
+
+    Constant *StepVal = ConstantInt::get(II.getType(), NumElts);
+    auto *VScale = Builder.CreateVScale(StepVal);
+    VScale->takeName(&II);
+    return IC.replaceInstUsesWith(II, VScale);
+  }
+
+  unsigned MinNumElts = 0;
+  switch (Pattern) {
+  default:
+    return None;
+  case AArch64SVEPredPattern::vl1:
+  case AArch64SVEPredPattern::vl2:
+  case AArch64SVEPredPattern::vl3:
+  case AArch64SVEPredPattern::vl4:
+  case AArch64SVEPredPattern::vl5:
+  case AArch64SVEPredPattern::vl6:
+  case AArch64SVEPredPattern::vl7:
+  case AArch64SVEPredPattern::vl8:
+    MinNumElts = Pattern;
+    break;
+  case AArch64SVEPredPattern::vl16:
+    MinNumElts = 16;
+    break;
+  }
+
+  return NumElts >= MinNumElts
+             ? Optional<Instruction *>(IC.replaceInstUsesWith(
+                   II, ConstantInt::get(II.getType(), MinNumElts)))
+             : None;
+}
+
 Optional<Instruction *>
 AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
                                      IntrinsicInst &II) const {
   Intrinsic::ID IID = II.getIntrinsicID();
   switch (IID) {
   default:
     break;
   case Intrinsic::aarch64_sve_convert_from_svbool:
     return instCombineConvertFromSVBool(IC, II);
   case Intrinsic::aarch64_sve_dup:
     return instCombineSVEDup(IC, II);
   case Intrinsic::aarch64_sve_cmpne:
   case Intrinsic::aarch64_sve_cmpne_wide:
     return instCombineSVECmpNE(IC, II);
   case Intrinsic::aarch64_sve_rdffr:
     return instCombineRDFFR(IC, II);
   case Intrinsic::aarch64_sve_lasta:
   case Intrinsic::aarch64_sve_lastb:
     return instCombineSVELast(IC, II);
+  case Intrinsic::aarch64_sve_cntd:
+    return instCombineSVECntElts(IC, II, 2);
+  case Intrinsic::aarch64_sve_cntw:
+    return instCombineSVECntElts(IC, II, 4);
+  case Intrinsic::aarch64_sve_cnth:
+    return instCombineSVECntElts(IC, II, 8);
+  case Intrinsic::aarch64_sve_cntb:
+    return instCombineSVECntElts(IC, II, 16);
   }
 
   return None;
 }
 
 bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
                                            ArrayRef<const Value *> Args) {
 

llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; CNTB
+;
+
+define i64 @cntb_vl1() {
+; CHECK-LABEL: @cntb_vl1(
+; CHECK-NEXT:    ret i64 1
+;
+  %out = call i64 @llvm.aarch64.sve.cntb(i32 1)
+  ret i64 %out
+}
+
+define i64 @cntb_vl2() {
+; CHECK-LABEL: @cntb_vl2(
+; CHECK-NEXT:    ret i64 2
+;
+  %out = call i64 @llvm.aarch64.sve.cntb(i32 2)
+  ret i64 %out
+}
+
+define i64 @cntb_vl4() {
+; CHECK-LABEL: @cntb_vl4(
+; CHECK-NEXT:    ret i64 4
+;
+  %out = call i64 @llvm.aarch64.sve.cntb(i32 4)
+  ret i64 %out
+}
+
+define i64 @cntb_mul3() {
+; CHECK-LABEL: @cntb_mul3(
+; CHECK-NEXT:    ret i64 24
+;
+  %cnt = call i64 @llvm.aarch64.sve.cntb(i32 8)
+  %out = mul i64 %cnt, 3
+  ret i64 %out
+}
+
+define i64 @cntb_mul4() {
+; CHECK-LABEL: @cntb_mul4(
+; CHECK-NEXT:    ret i64 64
+;
+  %cnt = call i64 @llvm.aarch64.sve.cntb(i32 9)
+  %out = mul i64 %cnt, 4
+  ret i64 %out
+}
+
+define i64 @cntb_all() {
+; CHECK-LABEL: @cntb_all(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sve.cntb(i32 31)
+  ret i64 %out
+}
+
+;
+; CNTH
+;
+
+define i64 @cnth_vl1() {
+; CHECK-LABEL: @cnth_vl1(
+; CHECK-NEXT:    ret i64 1
+;
+  %out = call i64 @llvm.aarch64.sve.cnth(i32 1)
+  ret i64 %out
+}
+
+define i64 @cnth_vl2() {
+; CHECK-LABEL: @cnth_vl2(
+; CHECK-NEXT:    ret i64 2
+;
+  %out = call i64 @llvm.aarch64.sve.cnth(i32 2)
+  ret i64 %out
+}
+
+define i64 @cnth_vl4() {
+; CHECK-LABEL: @cnth_vl4(
+; CHECK-NEXT:    ret i64 4
+;
+  %out = call i64 @llvm.aarch64.sve.cnth(i32 4)
+  ret i64 %out
+}
+
+define i64 @cnth_mul3() {
+; CHECK-LABEL: @cnth_mul3(
+; CHECK-NEXT:    ret i64 24
+;
+  %cnt = call i64 @llvm.aarch64.sve.cnth(i32 8)
+  %out = mul i64 %cnt, 3
+  ret i64 %out
+}
+
+define i64 @cnth_mul4() {
+; CHECK-LABEL: @cnth_mul4(
+; CHECK-NEXT:    [[CNT:%.*]] = call i64 @llvm.aarch64.sve.cnth(i32 9)
+; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[CNT]], 2
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %cnt = call i64 @llvm.aarch64.sve.cnth(i32 9)
+  %out = mul i64 %cnt, 4
+  ret i64 %out
+}
+
+define i64 @cnth_all() {
+; CHECK-LABEL: @cnth_all(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 3
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sve.cnth(i32 31)
+  ret i64 %out
+}
+
+;
+; CNTW
+;
+
+define i64 @cntw_vl1() {
+; CHECK-LABEL: @cntw_vl1(
+; CHECK-NEXT:    ret i64 1
+;
+  %out = call i64 @llvm.aarch64.sve.cntw(i32 1)
+  ret i64 %out
+}
+
+define i64 @cntw_vl2() {
+; CHECK-LABEL: @cntw_vl2(
+; CHECK-NEXT:    ret i64 2
+;
+  %out = call i64 @llvm.aarch64.sve.cntw(i32 2)
+  ret i64 %out
+}
+
+define i64 @cntw_vl4() {
+; CHECK-LABEL: @cntw_vl4(
+; CHECK-NEXT:    ret i64 4
+;
+  %out = call i64 @llvm.aarch64.sve.cntw(i32 4)
+  ret i64 %out
+}
+
+define i64 @cntw_mul3() {
+; CHECK-LABEL: @cntw_mul3(
+; CHECK-NEXT:    [[CNT:%.*]] = call i64 @llvm.aarch64.sve.cntw(i32 8)
+; CHECK-NEXT:    [[OUT:%.*]] = mul i64 [[CNT]], 3
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %cnt = call i64 @llvm.aarch64.sve.cntw(i32 8)
+  %out = mul i64 %cnt, 3
+  ret i64 %out
+}
+
+define i64 @cntw_mul4() {
+; CHECK-LABEL: @cntw_mul4(
+; CHECK-NEXT:    [[CNT:%.*]] = call i64 @llvm.aarch64.sve.cntw(i32 9)
+; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[CNT]], 2
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %cnt = call i64 @llvm.aarch64.sve.cntw(i32 9)
+  %out = mul i64 %cnt, 4
+  ret i64 %out
+}
+
+define i64 @cntw_all() {
+; CHECK-LABEL: @cntw_all(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sve.cntw(i32 31)
+  ret i64 %out
+}
+
+
+;
+; CNTD
+;
+
+define i64 @cntd_vl1() {
+; CHECK-LABEL: @cntd_vl1(
+; CHECK-NEXT:    ret i64 1
+;
+  %out = call i64 @llvm.aarch64.sve.cntd(i32 1)
+  ret i64 %out
+}
+
+define i64 @cntd_vl2() {
+; CHECK-LABEL: @cntd_vl2(
+; CHECK-NEXT:    ret i64 2
+;
+  %out = call i64 @llvm.aarch64.sve.cntd(i32 2)
+  ret i64 %out
+}
+
+define i64 @cntd_vl4() {
+; CHECK-LABEL: @cntd_vl4(
+; CHECK-NEXT:    [[OUT:%.*]] = call i64 @llvm.aarch64.sve.cntd(i32 4)
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sve.cntd(i32 4)
+  ret i64 %out
+}
+
+define i64 @cntd_mul3() {
+; CHECK-LABEL: @cntd_mul3(
+; CHECK-NEXT:    [[CNT:%.*]] = call i64 @llvm.aarch64.sve.cntd(i32 8)
+; CHECK-NEXT:    [[OUT:%.*]] = mul i64 [[CNT]], 3
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %cnt = call i64 @llvm.aarch64.sve.cntd(i32 8)
+  %out = mul i64 %cnt, 3
+  ret i64 %out
+}
+
+define i64 @cntd_mul4() {
+; CHECK-LABEL: @cntd_mul4(
+; CHECK-NEXT:    [[CNT:%.*]] = call i64 @llvm.aarch64.sve.cntd(i32 9)
+; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[CNT]], 2
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %cnt = call i64 @llvm.aarch64.sve.cntd(i32 9)
+  %out = mul i64 %cnt, 4
+  ret i64 %out
+}
+
+define i64 @cntd_all() {
+; CHECK-LABEL: @cntd_all(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 1
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sve.cntd(i32 31)
+  ret i64 %out
+}
+
+
+declare i64 @llvm.aarch64.sve.cntb(i32 %pattern)
+declare i64 @llvm.aarch64.sve.cnth(i32 %pattern)
+declare i64 @llvm.aarch64.sve.cntw(i32 %pattern)
+declare i64 @llvm.aarch64.sve.cntd(i32 %pattern)
+