Update profile during memory instrinsic optimization

[tejohnson]

Summary

Ensure that the new merge BB (which contains the rest of the original BB
after the mem op being optimized) gets a profile frequency, in case
there are additional mem ops later in the BB. Otherwise they get skipped
as the merge BB looks cold.

[davidxl]

lib/Transforms/Instrumentation/IndirectCallPromotion.cpp

955 ↗

(On Diff #96435)

This comment seems unnecessary :) Also can you move the DefaultBB setting after MergeBB update?

[tejohnson]

lib/Transforms/Instrumentation/IndirectCallPromotion.cpp

955 ↗

(On Diff #96435)

Sure. I mostly added the comment to indicate why it was important to update the MergeBB profile frequency specifically (and not the other new BBs), but can remove it.

[tejohnson]

Update per comments

[davidxl]

lgtm

llvm/trunk/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp

bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
   //      mem_op(..., size);
   //      goto merge_bb;
   // }
   // merge_bb:
 
   BasicBlock *BB = MI->getParent();
   DEBUG(dbgs() << "\n\n== Basic Block Before ==\n");
   DEBUG(dbgs() << *BB << "\n");
+  auto OrigBBFreq = BFI.getBlockFreq(BB);
 
   BasicBlock *DefaultBB = SplitBlock(BB, MI);
   BasicBlock::iterator It(*MI);
   ++It;
   assert(It != DefaultBB->end());
   BasicBlock *MergeBB = SplitBlock(DefaultBB, &(*It));
-  DefaultBB->setName("MemOP.Default");
   MergeBB->setName("MemOP.Merge");
+  BFI.setBlockFreq(MergeBB, OrigBBFreq.getFrequency());
+  DefaultBB->setName("MemOP.Default");
 
   auto &Ctx = Func.getContext();
   IRBuilder<> IRB(BB);
   BB->getTerminator()->eraseFromParent();
   Value *SizeVar = MI->getLength();
   SwitchInst *SI = IRB.CreateSwitch(SizeVar, DefaultBB, SizeIds.size());
 
   // Clear the value profile data.

llvm/trunk/test/Transforms/PGOProfile/memop_size_opt.ll

 ; RUN: opt < %s -passes=pgo-memop-opt -pgo-memop-count-threshold=90 -pgo-memop-percent-threshold=15 -S | FileCheck %s --check-prefix=MEMOP_OPT
 ; RUN: opt < %s -pgo-memop-opt -pgo-memop-count-threshold=90 -pgo-memop-percent-threshold=15 -S | FileCheck %s --check-prefix=MEMOP_OPT
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @foo(i8* %dst, i8* %src, i32* %a, i32 %n) !prof !27 {
+define void @foo(i8* %dst, i8* %src, i8* %dst2, i8* %src2, i32* %a, i32 %n) !prof !27 {
 entry:
   br label %for.cond
 
 for.cond:
   %i.0 = phi i32 [ 0, %entry ], [ %inc5, %for.inc4 ]
   %cmp = icmp slt i32 %i.0, %n
   br i1 %cmp, label %for.body, label %for.end6, !prof !28
 
 for.body:
   br label %for.cond1
 
 for.cond1:
   %j.0 = phi i32 [ 0, %for.body ], [ %inc, %for.inc ]
   %idx.ext = sext i32 %i.0 to i64
   %add.ptr = getelementptr inbounds i32, i32* %a, i64 %idx.ext
   %0 = load i32, i32* %add.ptr, align 4
   %cmp2 = icmp slt i32 %j.0, %0
   br i1 %cmp2, label %for.body3, label %for.end, !prof !29
 
 for.body3:
   %add = add nsw i32 %i.0, 1
   %conv = sext i32 %add to i64
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i32 1, i1 false), !prof !30
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 %conv, i32 1, i1 false), !prof !31
   br label %for.inc
 
-; MEMOP_OPT:  switch i64 %conv, label %[[Default_LABEL:.*]] [
+; MEMOP_OPT:  switch i64 %conv, label %[[DEFAULT_LABEL:.*]] [
 ; MEMOP_OPT:    i64 1, label %[[CASE_1_LABEL:.*]]
 ; MEMOP_OPT:  ], !prof [[SWITCH_BW:![0-9]+]] 
 ; MEMOP_OPT: [[CASE_1_LABEL]]:
 ; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 1, i32 1, i1 false)
 ; MEMOP_OPT:   br label %[[MERGE_LABEL:.*]]
-; MEMOP_OPT: [[Default_LABEL]]:
-; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i32 1, i1 false)
-; MEMOP_OPT-NOT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i32 1, i1 false), !prof
+; MEMOP_OPT: [[DEFAULT_LABEL]]:
+; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i32 1, i1 false){{[[:space:]]}}
 ; MEMOP_OPT:   br label %[[MERGE_LABEL]]
 ; MEMOP_OPT: [[MERGE_LABEL]]:
+; MEMOP_OPT:  switch i64 %conv, label %[[DEFAULT_LABEL2:.*]] [
+; MEMOP_OPT:    i64 1, label %[[CASE_1_LABEL2:.*]]
+; MEMOP_OPT:  ], !prof [[SWITCH_BW:![0-9]+]] 
+; MEMOP_OPT: [[CASE_1_LABEL2]]:
+; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 1, i32 1, i1 false)
+; MEMOP_OPT:   br label %[[MERGE_LABEL2:.*]]
+; MEMOP_OPT: [[DEFAULT_LABEL2]]:
+; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 %conv, i32 1, i1 false){{[[:space:]]}}
+; MEMOP_OPT:   br label %[[MERGE_LABEL2]]
+; MEMOP_OPT: [[MERGE_LABEL2]]:
 ; MEMOP_OPT:   br label %for.inc
 ; MEMOP_OPT: [[SWITCH_BW]] = !{!"branch_weights", i32 457, i32 99}
 
 for.inc:
   %inc = add nsw i32 %j.0, 1
   br label %for.cond1
 
 for.end:

 !23 = !{i32 999000, i64 1, i32 5}
 !24 = !{i32 999900, i64 1, i32 5}
 !25 = !{i32 999990, i64 1, i32 5}
 !26 = !{i32 999999, i64 1, i32 5}
 !27 = !{!"function_entry_count", i64 1}
 !28 = !{!"branch_weights", i32 20, i32 1}
 !29 = !{!"branch_weights", i32 556, i32 20}
 !30 = !{!"VP", i32 1, i64 556, i64 1, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22}
+!31 = !{!"VP", i32 1, i64 556, i64 1, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22}
 
 declare void @llvm.lifetime.start(i64, i8* nocapture)
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1)
 
 declare void @llvm.lifetime.end(i64, i8* nocapture)