[AArch64][GlobalISel] Legalize ctpop for v2s64, v2s32, v4s32, v4s16, v8s16

[jroelofs]

Summary

https://llvm.godbolt.org/z/nTTK6M5qe

[paquette]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

1157

doesn't SmallVector need an initial size?

1177

I think CTPOP is only used here? Maybe it's a little cleaner to move it and related stuff down near it.

1184

I don't think you should have to do this here, since we're before register bank selection?

1190

Logic here is getting a little complex. Maybe this?

if (Ty.isScalar() && Size == 64)
  MIRBuilder.buildZExt(Dst, UADD);
else
  UADD->getOperand(0).setReg(Dst);

1198

This shouldn't be necessary since we haven't selected regbanks yet.

[jroelofs]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

1157

Sean Silva recently added a default to it: https://reviews.llvm.org/D92522

1177

I wanted to keep HAdds near where it was used.

1184

MachineVerifier seems to expect them just after legalization:

*** Bad machine code: Virtual register does not match instruction constraint ***
- function:    s32_lower
- basic block: %bb.0  (0x7fba998570c8)
- instruction: %ctpop:_(s32) = UADDLVv8i8v %4:_(<8 x s8>)
- operand 0:   %ctpop:_
Expect register class FPR16 but got nothing

1190

👍

[paquette]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

1184

ah, yeah I guess it's cause it's not a generic instr. :/

Maybe it would make sense to add a G_UADDLV generic instruction to avoid emitting target instrs.

IIRC we import tablegen patterns for the uaddlv intrinsic allowing us to select it easily. Maybe if we had a generic instruction, we could do something similar.

[jroelofs]

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

1184

maybe I ought to just use the intrinsics then

[paquette]

LGTM

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_ROTR)
       })
       .lower();
   getActionDefinitionsBuilder(G_ROTL).lower();
 
   getActionDefinitionsBuilder({G_SBFX, G_UBFX})
       .customFor({{s32, s32}, {s64, s64}});
 
   // TODO: Custom legalization for s128
-  // TODO: v2s64, v2s32, v4s32, v4s16, v8s16
   // TODO: Use generic lowering when custom lowering is not possible.
   auto always = [=](const LegalityQuery &Q) { return true; };
   getActionDefinitionsBuilder(G_CTPOP)
       .legalFor({{v8s8, v8s8}, {v16s8, v16s8}})
       .clampScalar(0, s32, s128)
       .widenScalarToNextPow2(0)
       .minScalarEltSameAsIf(always, 1, 0)
       .maxScalarEltSameAsIf(always, 1, 0)
-      .customFor({{s32, s32}, {s64, s64}});
+      .customFor({{s32, s32},
+                  {s64, s64},
+                  {v2s64, v2s64},
+                  {v2s32, v2s32},
+                  {v4s32, v4s32},
+                  {v4s16, v4s16},
+                  {v8s16, v8s16}});
 
   getLegacyLegalizerInfo().computeTables();
   verify(*ST.getInstrInfo());
 }
 
 bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
                                           MachineInstr &MI) const {
   MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
   // While there is no integer popcount instruction, it can
   // be more efficiently lowered to the following sequence that uses
   // AdvSIMD registers/instructions as long as the copies to/from
   // the AdvSIMD registers are cheap.
   //  FMOV    D0, X0        // copy 64-bit int to vector, high bits zero'd
   //  CNT     V0.8B, V0.8B  // 8xbyte pop-counts
   //  ADDV    B0, V0.8B     // sum 8xbyte pop-counts
   //  UMOV    X0, V0.B[0]   // copy byte result back to integer reg
+  //
+  // For 128 bit vector popcounts, we lower to the following sequence:
+  //  cnt.16b   v0, v0  // v8s16, v4s32, v2s64
+  //  uaddlp.8h v0, v0  // v8s16, v4s32, v2s64
+  //  uaddlp.4s v0, v0  //        v4s32, v2s64
+  //  uaddlp.2d v0, v0  //               v2s64
+  //
+  // For 64 bit vector popcounts, we lower to the following sequence:
+  //  cnt.8b    v0, v0  // v4s16, v2s32
+  //  uaddlp.4h v0, v0  // v4s16, v2s32
+  //  uaddlp.2s v0, v0  //        v2s32
+
   if (!ST->hasNEON() ||
       MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat))
     return false;
   MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
   Register Dst = MI.getOperand(0).getReg();
   Register Val = MI.getOperand(1).getReg();
   LLT Ty = MRI.getType(Val);
 
-  // TODO: Handle vector types.
-  assert(!Ty.isVector() && "Vector types not handled yet!");
   assert(Ty == MRI.getType(Dst) &&
          "Expected src and dst to have the same type!");
-  // TODO: Handle s128.
   unsigned Size = Ty.getSizeInBits();
+
+  if (Ty.isScalar()) {
+    // TODO: Handle s128.
     assert((Size == 32 || Size == 64) && "Expected only 32 or 64 bit scalars!");
-  if (Size == 32)
+    if (Size == 32) {
       Val = MIRBuilder.buildZExt(LLT::scalar(64), Val).getReg(0);
-  const LLT V8S8 = LLT::fixed_vector(8, LLT::scalar(8));
-  Val = MIRBuilder.buildBitcast(V8S8, Val).getReg(0);
-  auto CTPOP = MIRBuilder.buildCTPOP(V8S8, Val);
-  auto UADDLV =
-      MIRBuilder
-          .buildIntrinsic(Intrinsic::aarch64_neon_uaddlv, {LLT::scalar(32)},
-                          /*HasSideEffects = */ false)
-          .addUse(CTPOP.getReg(0));
+    }
+  }
+
+  // s32,s64,v4s16,v2s32 -> v8i8
+  // v8s16,v4s32,v2s64 -> v16i8
+  LLT CTPOPTy = Size == 128 ? LLT::fixed_vector(16, 8) : LLT::fixed_vector(8, 8);
+  Val = MIRBuilder.buildBitcast(CTPOPTy, Val).getReg(0);
+  auto CTPOP = MIRBuilder.buildCTPOP(CTPOPTy, Val);
+
+  SmallVector<std::pair<LLT, unsigned>> HAdds;
+  if (Ty.isScalar()) {
+    HAdds.emplace_back(LLT::scalar(32), AArch64::UADDLVv8i8v);
+  } else if (Ty == LLT::fixed_vector(8, 16)) {
+    HAdds.emplace_back(LLT::fixed_vector(8, 16), AArch64::UADDLPv16i8_v8i16);
+  } else if (Ty == LLT::fixed_vector(4, 32)) {
+    HAdds.emplace_back(LLT::fixed_vector(8, 16), AArch64::UADDLPv16i8_v8i16);
+    HAdds.emplace_back(LLT::fixed_vector(4, 32), AArch64::UADDLPv8i16_v4i32);
+  } else if (Ty == LLT::fixed_vector(2, 64)) {
+    HAdds.emplace_back(LLT::fixed_vector(8, 16), AArch64::UADDLPv16i8_v8i16);
+    HAdds.emplace_back(LLT::fixed_vector(4, 32), AArch64::UADDLPv8i16_v4i32);
+    HAdds.emplace_back(LLT::fixed_vector(2, 64), AArch64::UADDLPv4i32_v2i64);
+  } else if (Ty == LLT::fixed_vector(4, 16)) {
+    HAdds.emplace_back(LLT::fixed_vector(4, 16), AArch64::UADDLPv8i8_v4i16);
+  } else if (Ty == LLT::fixed_vector(2, 32)) {
+    HAdds.emplace_back(LLT::fixed_vector(4, 16), AArch64::UADDLPv8i8_v4i16);
+    HAdds.emplace_back(LLT::fixed_vector(2, 32), AArch64::UADDLPv4i16_v2i32);
+  } else
+    llvm_unreachable("unexpected vector shape");
+
+  Register HSum = CTPOP.getReg(0);
+  MachineInstrBuilder UADD;
+  for (auto &HAdd : HAdds) {
+    LLT HTy;
+    unsigned Opc;
+    std::tie(HTy, Opc) = HAdd;
+    UADD = MIRBuilder.buildInstr(Opc, {HTy}, {HSum});
+    constrainSelectedInstRegOperands(*UADD, *ST->getInstrInfo(),
+                                     *MRI.getTargetRegisterInfo(),
+                                     *ST->getRegBankInfo());
+    HSum = UADD.getReg(0);
+  }
+
+  if (Ty.isScalar()) {
     if (Size == 64)
-    MIRBuilder.buildZExt(Dst, UADDLV);
+      MIRBuilder.buildZExt(Dst, UADD);
     else
-    UADDLV->getOperand(0).setReg(Dst);
+      UADD->getOperand(0).setReg(Dst);
+  } else {
+    UADD->getOperand(0).setReg(Dst);
+  }
+  constrainSelectedInstRegOperands(*UADD, *ST->getInstrInfo(),
+                                   *MRI.getTargetRegisterInfo(),
+                                   *ST->getRegBankInfo());
   MI.eraseFromParent();
   return true;
 }
 
 bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
     MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
   MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
   LLT s64 = LLT::scalar(64);

llvm/test/CodeGen/AArch64/GlobalISel/legalize-ctpop.mir

 body:             |
   bb.0:
     liveins: $w0
     ; CHECK-LABEL: name: s32_lower
     ; CHECK: liveins: $w0
     ; CHECK: %copy:_(s32) = COPY $w0
     ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT %copy(s32)
     ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST [[ZEXT]](s64)
-    ; CHECK: [[CTPOP:%[0-9]+]]:_(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
-    ; CHECK: %ctpop:_(s32) = G_INTRINSIC intrinsic(@llvm.aarch64.neon.uaddlv), [[CTPOP]](<8 x s8>)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: %ctpop:fpr16(s32) = UADDLVv8i8v [[CTPOP]](<8 x s8>)
     ; CHECK: $w0 = COPY %ctpop(s32)
     ; CHECK: RET_ReallyLR implicit $w0
     %copy:_(s32) = COPY $w0
     %ctpop:_(s32) = G_CTPOP %copy(s32)
     $w0 = COPY %ctpop(s32)
     RET_ReallyLR implicit $w0
 
 ...
 ---
 name:            s64_lower
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $x0
     ; CHECK-LABEL: name: s64_lower
     ; CHECK: liveins: $x0
     ; CHECK: %copy:_(s64) = COPY $x0
     ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST %copy(s64)
-    ; CHECK: [[CTPOP:%[0-9]+]]:_(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
-    ; CHECK: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.aarch64.neon.uaddlv), [[CTPOP]](<8 x s8>)
-    ; CHECK: %ctpop:_(s64) = G_ZEXT [[INT]](s32)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: [[UADDLVv8i8v:%[0-9]+]]:fpr16(s32) = UADDLVv8i8v [[CTPOP]](<8 x s8>)
+    ; CHECK: %ctpop:_(s64) = G_ZEXT [[UADDLVv8i8v]](s32)
     ; CHECK: $x0 = COPY %ctpop(s64)
     ; CHECK: RET_ReallyLR implicit $x0
     %copy:_(s64) = COPY $x0
     %ctpop:_(s64) = G_CTPOP %copy(s64)
     $x0 = COPY %ctpop(s64)
     RET_ReallyLR implicit $x0
 
 ...
 ---
 name:            widen_s16
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $w0
 
     ; CHECK-LABEL: name: widen_s16
     ; CHECK: liveins: $w0
     ; CHECK: %copy:_(s32) = COPY $w0
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT %copy(s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST [[AND]](s64)
-    ; CHECK: [[CTPOP:%[0-9]+]]:_(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
-    ; CHECK: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.aarch64.neon.uaddlv), [[CTPOP]](<8 x s8>)
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[INT]](s32)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: [[UADDLVv8i8v:%[0-9]+]]:fpr16(s32) = UADDLVv8i8v [[CTPOP]](<8 x s8>)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[UADDLVv8i8v]](s32)
     ; CHECK: %ext:_(s32) = COPY [[COPY]](s32)
     ; CHECK: $w0 = COPY %ext(s32)
     ; CHECK: RET_ReallyLR implicit $w0
     %copy:_(s32) = COPY $w0
     %trunc:_(s16) = G_TRUNC %copy(s32)
     %ctpop:_(s16) = G_CTPOP %trunc(s16)
     %ext:_(s32) = G_ANYEXT %ctpop(s16)
     $w0 = COPY %ext(s32)
bb.0:
 
     ; CHECK-LABEL: name: widen_s8
     ; CHECK: liveins: $w0
     ; CHECK: %copy:_(s32) = COPY $w0
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT %copy(s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST [[AND]](s64)
-    ; CHECK: [[CTPOP:%[0-9]+]]:_(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
-    ; CHECK: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.aarch64.neon.uaddlv), [[CTPOP]](<8 x s8>)
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[INT]](s32)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: [[UADDLVv8i8v:%[0-9]+]]:fpr16(s32) = UADDLVv8i8v [[CTPOP]](<8 x s8>)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[UADDLVv8i8v]](s32)
     ; CHECK: %ext:_(s32) = COPY [[COPY]](s32)
     ; CHECK: $w0 = COPY %ext(s32)
     ; CHECK: RET_ReallyLR implicit $w0
     %copy:_(s32) = COPY $w0
     %trunc:_(s8) = G_TRUNC %copy(s32)
     %ctpop:_(s8) = G_CTPOP %trunc(s8)
     %ext:_(s32) = G_ANYEXT %ctpop(s8)
     $w0 = COPY %ext(s32)
bb.0:
 
     ; CHECK-LABEL: name: widen_s3
     ; CHECK: liveins: $w0
     ; CHECK: %copy:_(s32) = COPY $w0
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT %copy(s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST [[AND]](s64)
-    ; CHECK: [[CTPOP:%[0-9]+]]:_(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
-    ; CHECK: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.aarch64.neon.uaddlv), [[CTPOP]](<8 x s8>)
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[INT]](s32)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: [[UADDLVv8i8v:%[0-9]+]]:fpr16(s32) = UADDLVv8i8v [[CTPOP]](<8 x s8>)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[UADDLVv8i8v]](s32)
     ; CHECK: %ext:_(s32) = COPY [[COPY]](s32)
     ; CHECK: $w0 = COPY %ext(s32)
     ; CHECK: RET_ReallyLR implicit $w0
     %copy:_(s32) = COPY $w0
     %trunc:_(s3) = G_TRUNC %copy(s32)
     %ctpop:_(s3) = G_CTPOP %trunc(s3)
     %ext:_(s32) = G_ANYEXT %ctpop(s3)
     $w0 = COPY %ext(s32)
     RET_ReallyLR implicit $w0
 
 ...
 ---
 name:            different_sizes
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $w0
     ; CHECK-LABEL: name: different_sizes
     ; CHECK: liveins: $w0
     ; CHECK: %copy:_(s32) = COPY $w0
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT %copy(s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST [[AND]](s64)
-    ; CHECK: [[CTPOP:%[0-9]+]]:_(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
-    ; CHECK: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.aarch64.neon.uaddlv), [[CTPOP]](<8 x s8>)
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[INT]](s32)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: [[UADDLVv8i8v:%[0-9]+]]:fpr16(s32) = UADDLVv8i8v [[CTPOP]](<8 x s8>)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[UADDLVv8i8v]](s32)
     ; CHECK: %ext:_(s32) = COPY [[COPY]](s32)
     ; CHECK: $w0 = COPY %ext(s32)
     ; CHECK: RET_ReallyLR implicit $w0
     %copy:_(s32) = COPY $w0
     %trunc:_(s8) = G_TRUNC %copy(s32)
     %ctpop:_(s16) = G_CTPOP %trunc(s8)
     %ext:_(s32) = G_ANYEXT %ctpop(s16)
     $w0 = COPY %ext(s32)
     RET_ReallyLR implicit $w0
+
+...
+---
+name:            custom_8x16
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: custom_8x16
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<16 x s8>) = G_BITCAST [[COPY]](<8 x s16>)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr128(<16 x s8>) = G_CTPOP [[BITCAST]](<16 x s8>)
+    ; CHECK: [[UADDLPv16i8_v8i16_:%[0-9]+]]:fpr128(<8 x s16>) = UADDLPv16i8_v8i16 [[CTPOP]](<16 x s8>)
+    ; CHECK: $q0 = COPY [[UADDLPv16i8_v8i16_]](<8 x s16>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_CTPOP %0(<8 x s16>)
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            custom_4x32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: custom_4x32
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<16 x s8>) = G_BITCAST [[COPY]](<4 x s32>)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr128(<16 x s8>) = G_CTPOP [[BITCAST]](<16 x s8>)
+    ; CHECK: [[UADDLPv16i8_v8i16_:%[0-9]+]]:fpr128(<8 x s16>) = UADDLPv16i8_v8i16 [[CTPOP]](<16 x s8>)
+    ; CHECK: [[UADDLPv8i16_v4i32_:%[0-9]+]]:fpr128(<4 x s32>) = UADDLPv8i16_v4i32 [[UADDLPv16i8_v8i16_]](<8 x s16>)
+    ; CHECK: $q0 = COPY [[UADDLPv8i16_v4i32_]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_CTPOP %0(<4 x s32>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            custom_2x64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: custom_2x64
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<16 x s8>) = G_BITCAST [[COPY]](<2 x s64>)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr128(<16 x s8>) = G_CTPOP [[BITCAST]](<16 x s8>)
+    ; CHECK: [[UADDLPv16i8_v8i16_:%[0-9]+]]:fpr128(<8 x s16>) = UADDLPv16i8_v8i16 [[CTPOP]](<16 x s8>)
+    ; CHECK: [[UADDLPv8i16_v4i32_:%[0-9]+]]:fpr128(<4 x s32>) = UADDLPv8i16_v4i32 [[UADDLPv16i8_v8i16_]](<8 x s16>)
+    ; CHECK: [[UADDLPv4i32_v2i64_:%[0-9]+]]:fpr128(<2 x s64>) = UADDLPv4i32_v2i64 [[UADDLPv8i16_v4i32_]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[UADDLPv4i32_v2i64_]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_CTPOP %0(<2 x s64>)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            custom_4x16
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: custom_4x16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST [[COPY]](<4 x s16>)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: [[UADDLPv8i8_v4i16_:%[0-9]+]]:fpr64(<4 x s16>) = UADDLPv8i8_v4i16 [[CTPOP]](<8 x s8>)
+    ; CHECK: $d0 = COPY [[UADDLPv8i8_v4i16_]](<4 x s16>)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_CTPOP %0(<4 x s16>)
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            custom_2x32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: custom_2x32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<8 x s8>) = G_BITCAST [[COPY]](<2 x s32>)
+    ; CHECK: [[CTPOP:%[0-9]+]]:fpr64(<8 x s8>) = G_CTPOP [[BITCAST]](<8 x s8>)
+    ; CHECK: [[UADDLPv8i8_v4i16_:%[0-9]+]]:fpr64(<4 x s16>) = UADDLPv8i8_v4i16 [[CTPOP]](<8 x s8>)
+    ; CHECK: [[UADDLPv4i16_v2i32_:%[0-9]+]]:fpr64(<2 x s32>) = UADDLPv4i16_v2i32 [[UADDLPv8i8_v4i16_]](<4 x s16>)
+    ; CHECK: $d0 = COPY [[UADDLPv4i16_v2i32_]](<2 x s32>)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_CTPOP %0(<2 x s32>)
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0