[Profile] Lower expect intrinsic properly for select instruction

[davidxl]

Summary

builtin expect passed to select instruction gets ignored. This patch fixes the issue.

[davidxl]

Make the implementation more efficient -- avoid traversing IR twice.

[spatel]

lib/Transforms/Scalar/LowerExpectIntrinsic.cpp

86 ↗

(On Diff #70133)

Neat trick. :)
But can you add a comment and/or rename BI, so it is clear that this isn't strictly for a 'BI' (BranchInst) any more?

159 ↗

(On Diff #70133)

Fix comment to indicate why we are using reverse iterator (and that we're also handling selects in this loop).

160 ↗

(On Diff #70133)

Use 'auto' to avoid 80-col ugliness?

test/Transforms/LowerExpectIntrinsic/basic.ll

276–287 ↗

(On Diff #70133)

Minimize test?

define i32 @test10(i64 %t6) {
  %t7 = call i64 @llvm.expect.i64(i64 %t6, i64 0)
  %t8 = icmp ne i64 %t7, 0
  %t9 = select i1 %t8, i32 42, i32 43
  ret i32 %t9
}

[davidxl]

Addressed review comments.

[spatel]

LGTM.

llvm/trunk/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp

static bool handleSwitchExpect(SwitchInst &SI) {
 
   SI.setMetadata(LLVMContext::MD_prof,
                  MDBuilder(CI->getContext()).createBranchWeights(Weights));
 
   SI.setCondition(ArgValue);
   return true;
 }
 
-static bool handleBranchExpect(BranchInst &BI) {
-  if (BI.isUnconditional())
-    return false;
+// Handle both BranchInst and SelectInst.
+template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
 
   // Handle non-optimized IR code like:
   //   %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
   //   %tobool = icmp ne i64 %expval, 0
   //   br i1 %tobool, label %if.then, label %if.end
   //
   // Or the following simpler case:
   //   %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
   //   br i1 %expval, label %if.then, label %if.end
 
   CallInst *CI;
 
-  ICmpInst *CmpI = dyn_cast<ICmpInst>(BI.getCondition());
+  ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition());
   if (!CmpI) {
-    CI = dyn_cast<CallInst>(BI.getCondition());
+    CI = dyn_cast<CallInst>(BSI.getCondition());
   } else {
     if (CmpI->getPredicate() != CmpInst::ICMP_NE)
       return false;
     CI = dyn_cast<CallInst>(CmpI->getOperand(0));
   }
 
   if (!CI)
     return false;
template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
 
   // If expect value is equal to 1 it means that we are more likely to take
   // branch 0, in other case more likely is branch 1.
   if (ExpectedValue->isOne())
     Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
   else
     Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
 
-  BI.setMetadata(LLVMContext::MD_prof, Node);
+  BSI.setMetadata(LLVMContext::MD_prof, Node);
 
   if (CmpI)
     CmpI->setOperand(0, ArgValue);
   else
-    BI.setCondition(ArgValue);
+    BSI.setCondition(ArgValue);
   return true;
 }
 
+static bool handleBranchExpect(BranchInst &BI) {
+  if (BI.isUnconditional())
+    return false;
+
+  return handleBrSelExpect<BranchInst>(BI);
+}
+
 static bool lowerExpectIntrinsic(Function &F) {
   bool Changed = false;
 
   for (BasicBlock &BB : F) {
     // Create "block_weights" metadata.
     if (BranchInst *BI = dyn_cast<BranchInst>(BB.getTerminator())) {
       if (handleBranchExpect(*BI))
         ExpectIntrinsicsHandled++;
     } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB.getTerminator())) {
       if (handleSwitchExpect(*SI))
         ExpectIntrinsicsHandled++;
     }
 
-    // Remove llvm.expect intrinsics.
-    for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) {
-      CallInst *CI = dyn_cast<CallInst>(BI++);
-      if (!CI)
+    // Remove llvm.expect intrinsics. Iterate backwards in order
+    // to process select instructions before the intrinsic gets
+    // removed.
+    for (auto BI = BB.rbegin(), BE = BB.rend(); BI != BE;) {
+      Instruction *Inst = &*BI++;
+      CallInst *CI = dyn_cast<CallInst>(Inst);
+      if (!CI) {
+        if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
+          if (handleBrSelExpect(*SI))
+            ExpectIntrinsicsHandled++;
+        }
         continue;
+      }
 
       Function *Fn = CI->getCalledFunction();
       if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) {
         Value *Exp = CI->getArgOperand(0);
         CI->replaceAllUsesWith(Exp);
         CI->eraseFromParent();
         Changed = true;
       }

llvm/trunk/test/Transforms/LowerExpectIntrinsic/basic.ll

if.end:                                           ; preds = %entry
   store i32 1, i32* %retval
   br label %return
 
 return:                                           ; preds = %if.end, %if.then
   %0 = load i32, i32* %retval
   ret i32 %0
 }
 
+; CHECK-LABEL: @test10(
+define i32 @test10(i64 %t6) {
+  %t7 = call i64 @llvm.expect.i64(i64 %t6, i64 0)
+  %t8 = icmp ne i64 %t7, 0
+  %t9 = select i1 %t8, i32 1, i32 2
+; CHECK: select{{.*}}, !prof !1
+  ret i32 %t9
+}
+
+
 declare i1 @llvm.expect.i1(i1, i1) nounwind readnone
 
 ; CHECK: !0 = !{!"branch_weights", i32 2000, i32 1}
 ; CHECK: !1 = !{!"branch_weights", i32 1, i32 2000}
 ; CHECK: !2 = !{!"branch_weights", i32 1, i32 2000, i32 1}
 ; CHECK: !3 = !{!"branch_weights", i32 2000, i32 1, i32 1}