AMDGPU: Other sizes of popcnt are fast

[arsenm]

Summary

We can chain bcnt instructions together, so any width popcnt is pretty fast.

[• tstellarAMD]

LGTM.

[arsenm]

r269950

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

AMDGPUTTIImpl(AMDGPUTTIImpl &&Arg)
         TLI(std::move(Arg.TLI)) {}
 
   bool hasBranchDivergence() { return true; }
 
   void getUnrollingPreferences(Loop *L, TTI::UnrollingPreferences &UP);
 
   TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) {
     assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
-    return ST->hasBCNT(TyWidth) ? TTI::PSK_FastHardware : TTI::PSK_Software;
+    return TTI::PSK_FastHardware;
   }
 
   unsigned getNumberOfRegisters(bool Vector);
   unsigned getRegisterBitWidth(bool Vector);
   unsigned getMaxInterleaveFactor(unsigned VF);
 
   int getArithmeticInstrCost(
     unsigned Opcode, Type *Ty,

test/CodeGen/AMDGPU/ctpop64.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
 
 declare i64 @llvm.ctpop.i64(i64) nounwind readnone
 declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) nounwind readnone
 declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>) nounwind readnone
 declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>) nounwind readnone
 declare <16 x i64> @llvm.ctpop.v16i64(<16 x i64>) nounwind readnone
 
+declare i65 @llvm.ctpop.i65(i65) nounwind readnone
+declare i128 @llvm.ctpop.i128(i128) nounwind readnone
+
 ; FUNC-LABEL: {{^}}s_ctpop_i64:
 ; SI: s_load_dwordx2 [[SVAL:s\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
 ; VI: s_load_dwordx2 [[SVAL:s\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
 ; GCN: s_bcnt1_i32_b64 [[SRESULT:s[0-9]+]], [[SVAL]]
 ; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]]
 ; GCN: buffer_store_dword [[VRESULT]],
 ; GCN: s_endpgm
 define void @s_ctpop_i64(i32 addrspace(1)* noalias %out, i64 %val) nounwind {
else:
   %tmp4 = load i64, i64 addrspace(1)* %tmp3
   br label %endif
 
 endif:
   %tmp5 = phi i64 [%tmp2, %if], [%tmp4, %else]
   store i64 %tmp5, i64 addrspace(1)* %out
   ret void
 }
+
+; FUNC-LABEL: {{^}}s_ctpop_i128:
+; GCN: s_bcnt1_i32_b64 [[SRESULT0:s[0-9]+]],
+; GCN: s_bcnt1_i32_b64 [[SRESULT1:s[0-9]+]],
+; GCN: s_add_i32 s{{[0-9]+}}, [[SRESULT0]], [[SRESULT1]]
+; GCN: s_endpgm
+define void @s_ctpop_i128(i32 addrspace(1)* noalias %out, i128 %val) nounwind {
+  %ctpop = call i128 @llvm.ctpop.i128(i128 %val) nounwind readnone
+  %truncctpop = trunc i128 %ctpop to i32
+  store i32 %truncctpop, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_ctpop_i65:
+; GCN: s_bcnt1_i32_b64
+; GCN: s_and_b32
+; GCN: s_bcnt1_i32_b64
+; GCN: s_add_i32
+; GCN: s_endpgm
+define void @s_ctpop_i65(i32 addrspace(1)* noalias %out, i65 %val) nounwind {
+  %ctpop = call i65 @llvm.ctpop.i65(i65 %val) nounwind readnone
+  %truncctpop = trunc i65 %ctpop to i32
+  store i32 %truncctpop, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FIXME: Should not have extra add
+
+; FUNC-LABEL: {{^}}v_ctpop_i128:
+; GCN-DAG: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; GCN-DAG: buffer_load_dwordx2 v{{\[}}[[VAL2:[0-9]+]]:[[VAL3:[0-9]+]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8{{$}}
+
+; GCN: v_bcnt_u32_b32_e64 [[MIDRESULT0:v[0-9]+]], v[[VAL2]], 0
+; GCN: v_bcnt_u32_b32{{_e32|_e64}} [[MIDRESULT1:v[0-9]+]], v[[VAL3]], [[MIDRESULT0]]
+
+; GCN: v_bcnt_u32_b32_e64 [[MIDRESULT2:v[0-9]+]], v[[VAL0]], 0
+; GCN: v_bcnt_u32_b32{{_e32|_e64}} [[MIDRESULT3:v[0-9]+]], v[[VAL1]], [[MIDRESULT2]]
+
+; GCN: v_add_i32_e32 [[RESULT:v[0-9]+]], vcc, [[MIDRESULT2]], [[MIDRESULT1]]
+
+; GCN: buffer_store_dword [[RESULT]],
+; GCN: s_endpgm
+define void @v_ctpop_i128(i32 addrspace(1)* noalias %out, i128 addrspace(1)* noalias %in) nounwind {
+  %val = load i128, i128 addrspace(1)* %in, align 8
+  %ctpop = call i128 @llvm.ctpop.i128(i128 %val) nounwind readnone
+  %truncctpop = trunc i128 %ctpop to i32
+  store i32 %truncctpop, i32 addrspace(1)* %out, align 4
+  ret void
+}

test/Transforms/LoopIdiom/AMDGPU/popcnt.ll

-; RUN: opt -loop-idiom -mtriple=r600-- -mcpu=SI -S < %s | FileCheck %s
+; RUN: opt -loop-idiom -mtriple=amdgcn-- -S < %s | FileCheck %s
 
 ; Mostly copied from x86 version.
 
 ;To recognize this pattern:
 ;int popcount(unsigned long long a) {
 ;    int c = 0;
 ;    while (a) {
 ;        c++;
while.body:                                       ; preds = %entry, %while.body
   %tobool = icmp eq i32 %and, 0
   br i1 %tobool, label %while.end, label %while.body
 
 while.end:                                        ; preds = %while.body, %entry
   %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
   ret i32 %c.0.lcssa
 }
 
+; CHECK-LABEL: @popcount_i128
+; CHECK: entry
+; CHECK: llvm.ctpop.i128
+; CHECK: ret
+define i32 @popcount_i128(i128 %a) nounwind uwtable readnone ssp {
+entry:
+  %tobool3 = icmp eq i128 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i128 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i128 %a.addr.04, -1
+  %and = and i128 %sub, %a.addr.04
+  %tobool = icmp eq i128 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
 ; To recognize this pattern:
 ;int popcount(unsigned long long a, int mydata1, int mydata2) {
 ;    int c = 0;
 ;    while (a) {
 ;        c++;
 ;        a &= a - 1;
 ;        mydata1 *= c;
 ;        mydata2 *= (int)a;