[AMDGPU] Promote uniform i16 bitreverse intrinsic to i32

[arsenm]

lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

288

This should be able to handle any bitwidth < 32. We will want this for the < i16 types that will want to be legalized to i16 as well

295–298

You should just define these when declared instead of resetting them below

430–431

This should be a switch since other intrinsics are likely, and it can just return the result of visitBitreverseIntrinsicInst directly

test/CodeGen/AMDGPU/amdgpu-codegenprepare-i16-to-i32.ll

433

Should have a test with smaller bitwidth

434–439

There should be check lines for the non-VI case for the unprompted case

[kzhuravl]

Address review feedback

[kzhuravl]

lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

288

I am working on handling smaller types in instructions and bitereverse in a separate patch, it is almost ready (I need to clean up and update doxygen). smaller types patch fixes a bunch of test failures on Wei's fp16 branch.

test/CodeGen/AMDGPU/amdgpu-codegenprepare-i16-to-i32.ll

433

In a separate patch.

[kzhuravl]

Bring up to date with trunk.

[arsenm]

LGTM

[kzhuravl]

r283415

lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

class AMDGPUCodeGenPrepare : public FunctionPass,
   /// \brief Promotes uniform 16 bit binary operation \p I to equivalent 32 bit
   /// binary operation by sign or zero extending operands to 32 bits, replacing
   /// 16 bit operation with equivalent 32 bit operation, and truncating the
   /// result of 32 bit operation back to 16 bits. 16 bit division operation is
   /// not promoted.
   ///
   /// \returns True if 16 bit binary operation is promoted to equivalent 32 bit
   /// binary operation, false otherwise.
-  bool promoteUniformI16OpToI32Op(BinaryOperator &I) const;
+  bool promoteUniformI16OpToI32(BinaryOperator &I) const;
 
   /// \brief Promotes uniform 16 bit 'icmp' operation \p I to 32 bit 'icmp'
   /// operation by sign or zero extending operands to 32 bits, and replacing 16
   /// bit operation with 32 bit operation.
   ///
   /// \returns True.
-  bool promoteUniformI16OpToI32Op(ICmpInst &I) const;
+  bool promoteUniformI16OpToI32(ICmpInst &I) const;
 
   /// \brief Promotes uniform 16 bit 'select' operation \p I to 32 bit 'select'
   /// operation by sign or zero extending operands to 32 bits, replacing 16 bit
   /// operation with 32 bit operation, and truncating the result of 32 bit
   /// operation back to 16 bits.
   ///
   /// \returns True.
-  bool promoteUniformI16OpToI32Op(SelectInst &I) const;
+  bool promoteUniformI16OpToI32(SelectInst &I) const;
+
+  /// \brief Promotes uniform 16 bit 'bitreverse' intrinsic \p I to 32 bit
+  /// 'bitreverse' intrinsic by zero extending operand to 32 bits, replacing 16
+  /// bit intrinsic with 32 bit intrinsic, shifting the result of 32 bit
+  /// intrinsic 16 bits to the right with zero fill, and truncating the result
+  /// of shift operation back to 16 bits.
+  ///
+  /// \returns True.
+  bool promoteUniformI16BitreverseIntrinsicToI32(IntrinsicInst &I) const;
 
 public:
   static char ID;
   AMDGPUCodeGenPrepare(const TargetMachine *TM = nullptr) :
     FunctionPass(ID),
     TM(static_cast<const GCNTargetMachine *>(TM)),
     ST(nullptr),
     DA(nullptr),
     Mod(nullptr),
     HasUnsafeFPMath(false) { }
 
   bool visitFDiv(BinaryOperator &I);
 
   bool visitInstruction(Instruction &I) { return false; }
   bool visitBinaryOperator(BinaryOperator &I);
   bool visitICmpInst(ICmpInst &I);
   bool visitSelectInst(SelectInst &I);
 
+  bool visitIntrinsicInst(IntrinsicInst &I);
+  bool visitBitreverseIntrinsicInst(IntrinsicInst &I);
+
   bool doInitialization(Module &M) override;
   bool runOnFunction(Function &F) override;
 
   const char *getPassName() const override {
     return "AMDGPU IR optimizations";
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
return I.getOpcode() == Instruction::SDiv ||
       I.getOpcode() == Instruction::SRem;
 }
 
 bool AMDGPUCodeGenPrepare::isSigned(const SelectInst &I) const {
   return isa<ICmpInst>(I.getOperand(0)) ?
       cast<ICmpInst>(I.getOperand(0))->isSigned() : false;
 }
 
-bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32Op(BinaryOperator &I) const {
-  assert(isI16Ty(I.getType()) && "Op must be 16 bits");
+bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32(BinaryOperator &I) const {
+  assert(isI16Ty(I.getType()) && "I must be 16 bits");
 
   if (I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv)
     return false;
 
   IRBuilder<> Builder(&I);
   Builder.SetCurrentDebugLocation(I.getDebugLoc());
 
   Type *I32Ty = getI32Ty(Builder, I.getType());
bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32(BinaryOperator &I) const {
   TruncRes = Builder.CreateTrunc(ExtRes, getI16Ty(Builder, ExtRes->getType()));
 
   I.replaceAllUsesWith(TruncRes);
   I.eraseFromParent();
 
   return true;
 }
 
-bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32Op(ICmpInst &I) const {
+bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32(ICmpInst &I) const {
   assert(isI16Ty(I.getOperand(0)->getType()) && "Op0 must be 16 bits");
   assert(isI16Ty(I.getOperand(1)->getType()) && "Op1 must be 16 bits");
 
   IRBuilder<> Builder(&I);
   Builder.SetCurrentDebugLocation(I.getDebugLoc());
 
   Type *I32TyOp0 = getI32Ty(Builder, I.getOperand(0)->getType());
   Type *I32TyOp1 = getI32Ty(Builder, I.getOperand(1)->getType());
bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32(ICmpInst &I) const {
   NewICmp = Builder.CreateICmp(I.getPredicate(), ExtOp0, ExtOp1);
 
   I.replaceAllUsesWith(NewICmp);
   I.eraseFromParent();
 
   return true;
 }
 
-bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32Op(SelectInst &I) const {
-  assert(isI16Ty(I.getType()) && "Op must be 16 bits");
+bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32(SelectInst &I) const {
+  assert(isI16Ty(I.getType()) && "I must be 16 bits");
 
   IRBuilder<> Builder(&I);
   Builder.SetCurrentDebugLocation(I.getDebugLoc());
 
   Type *I32Ty = getI32Ty(Builder, I.getType());
   Value *ExtOp1 = nullptr;
   Value *ExtOp2 = nullptr;
   Value *ExtRes = nullptr;
bool AMDGPUCodeGenPrepare::promoteUniformI16OpToI32(SelectInst &I) const {
   TruncRes = Builder.CreateTrunc(ExtRes, getI16Ty(Builder, ExtRes->getType()));
 
   I.replaceAllUsesWith(TruncRes);
   I.eraseFromParent();
 
   return true;
 }
 
+bool AMDGPUCodeGenPrepare::promoteUniformI16BitreverseIntrinsicToI32(
+    IntrinsicInst &I) const {
+  assert(I.getIntrinsicID() == Intrinsic::bitreverse && "I must be bitreverse");
+  assert(isI16Ty(I.getType()) && "I must be 16 bits");
+
+  IRBuilder<> Builder(&I);
+  Builder.SetCurrentDebugLocation(I.getDebugLoc());
+
+  Type *I32Ty = getI32Ty(Builder, I.getType());
+  Function *I32 = nullptr;
+  Value *ExtOp = nullptr;
+  Value *ExtRes = nullptr;
+  Value *LShrOp = nullptr;
+  Value *TruncRes = nullptr;
+
+  ExtOp = Builder.CreateZExt(I.getOperand(0), I32Ty);
+  I32 = Intrinsic::getDeclaration(Mod, Intrinsic::bitreverse, { I32Ty });
+  ExtRes = Builder.CreateCall(I32, { ExtOp });
+  LShrOp = Builder.CreateLShr(ExtRes, 16);
+  TruncRes = Builder.CreateTrunc(LShrOp, getI16Ty(Builder, ExtRes->getType()));
+
+  I.replaceAllUsesWith(TruncRes);
+  I.eraseFromParent();
+
+  return true;
+}
+
 static bool shouldKeepFDivF32(Value *Num, bool UnsafeDiv) {
   const ConstantFP *CNum = dyn_cast<ConstantFP>(Num);
   if (!CNum)
     return false;
 
   // Reciprocal f32 is handled separately without denormals.
   return UnsafeDiv || CNum->isExactlyValue(+1.0);
 }
static bool hasUnsafeFPMath(const Function &F) {
   return Attr.getValueAsString() == "true";
 }
 
 bool AMDGPUCodeGenPrepare::visitBinaryOperator(BinaryOperator &I) {
   bool Changed = false;
 
   // TODO: Should we promote smaller types that will be legalized to i16?
   if (ST->has16BitInsts() && isI16Ty(I.getType()) && DA->isUniform(&I))
-    Changed |= promoteUniformI16OpToI32Op(I);
+    Changed |= promoteUniformI16OpToI32(I);
 
   return Changed;
 }
 
 bool AMDGPUCodeGenPrepare::visitICmpInst(ICmpInst &I) {
   bool Changed = false;
 
   // TODO: Should we promote smaller types that will be legalized to i16?
   if (ST->has16BitInsts() && isI16Ty(I.getOperand(0)->getType()) &&
           isI16Ty(I.getOperand(1)->getType()) && DA->isUniform(&I))
-    Changed |= promoteUniformI16OpToI32Op(I);
+    Changed |= promoteUniformI16OpToI32(I);
 
   return Changed;
 }
 
 bool AMDGPUCodeGenPrepare::visitSelectInst(SelectInst &I) {
   bool Changed = false;
 
   // TODO: Should we promote smaller types that will be legalized to i16?
   if (ST->has16BitInsts() && isI16Ty(I.getType()) && DA->isUniform(&I))
-    Changed |= promoteUniformI16OpToI32Op(I);
+    Changed |= promoteUniformI16OpToI32(I);
+
+  return Changed;
+}
+
+bool AMDGPUCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
+  bool Changed = false;
+
+  if (I.getIntrinsicID() == Intrinsic::bitreverse)
+    Changed |= visitBitreverseIntrinsicInst(I);
+
+  return Changed;
+}
+
+bool AMDGPUCodeGenPrepare::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
+  bool Changed = false;
+
+  // TODO: Should we promote smaller types that will be legalized to i16?
+  if (ST->has16BitInsts() && isI16Ty(I.getType()) && DA->isUniform(&I))
+    Changed |= promoteUniformI16BitreverseIntrinsicToI32(I);
 
   return Changed;
 }
 
 bool AMDGPUCodeGenPrepare::doInitialization(Module &M) {
   Mod = &M;
   return false;
 }

test/CodeGen/AMDGPU/amdgpu-codegenprepare-i16-to-i32.ll

 ; VI: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16
 ; VI: ret i16 %[[SEL_16]]
 define i16 @select_sle_i16(i16 %a, i16 %b) {
   %cmp = icmp sle i16 %a, %b
   %sel = select i1 %cmp, i16 %a, i16 %b
   ret i16 %sel
 }
 
+declare i16 @llvm.bitreverse.i16(i16)
+; VI-LABEL: @bitreverse_i16(
+; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32
+; VI: %[[R_32:[0-9]+]] = call i32 @llvm.bitreverse.i32(i32 %[[A_32]])
+; VI: %[[S_32:[0-9]+]] = lshr i32 %[[R_32]], 16
+; VI: %[[R_16:[0-9]+]] = trunc i32 %[[S_32]] to i16
+; VI: ret i16 %[[R_16]]
+define i16 @bitreverse_i16(i16 %a) {
+  %brev = call i16 @llvm.bitreverse.i16(i16 %a)
+  ret i16 %brev
+}
+
 ; VI-LABEL: @add_3xi16(
 ; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32>
 ; VI: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32>
 ; VI: %[[R_32:[0-9]+]] = add <3 x i32> %[[A_32]], %[[B_32]]
 ; VI: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16>
 ; VI: ret <3 x i16> %[[R_16]]
 define <3 x i16> @add_3xi16(<3 x i16> %a, <3 x i16> %b) {
   %r = add <3 x i16> %a, %b

 ; VI: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]]
 ; VI: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16>
 ; VI: ret <3 x i16> %[[SEL_16]]
 define <3 x i16> @select_sle_3xi16(<3 x i16> %a, <3 x i16> %b) {
   %cmp = icmp sle <3 x i16> %a, %b
   %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b
   ret <3 x i16> %sel
 }
+
+declare <3 x i16> @llvm.bitreverse.v3i16(<3 x i16>)
+; VI-LABEL: @bitreverse_3xi16(
+; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32>
+; VI: %[[R_32:[0-9]+]] = call <3 x i32> @llvm.bitreverse.v3i32(<3 x i32> %[[A_32]])
+; VI: %[[S_32:[0-9]+]] = lshr <3 x i32> %[[R_32]], <i32 16, i32 16, i32 16>
+; VI: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[S_32]] to <3 x i16>
+; VI: ret <3 x i16> %[[R_16]]
+define <3 x i16> @bitreverse_3xi16(<3 x i16> %a) {
+  %brev = call <3 x i16> @llvm.bitreverse.v3i16(<3 x i16> %a)
+  ret <3 x i16> %brev
+}