[InstrProf] Fix promoter when using counter relocations

[ellis]

Summary

When using counter relocations, two instructions are emitted to compute
the address of the counter variable.

%BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
%Addr = inttoptr i64 %BiasAdd to i64*

When promoting a counter, these instructions might not be available in
the block, so we need to copy these instructions.

This fixes https://github.com/llvm/llvm-project/issues/55125

[phosek]

LGTM thanks for looking into this!

llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp

for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
       Instruction *InsertPos = InsertPts[i];
       // Get LiveIn value into the ExitBlock. If there are multiple
       // predecessors, the value is defined by a PHI node in this
       // block.
       Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
       Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
       Type *Ty = LiveInValue->getType();
       IRBuilder<> Builder(InsertPos);
+      if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
+        // If isRuntimeCounterRelocationEnabled() is true then the address of
+        // the store instruction is computed with two instructions in
+        // InstrProfiling::getCounterAddress(). We need to copy those
+        // instructions to this block to compute Addr correctly.
+        // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
+        // %Addr = inttoptr i64 %BiasAdd to i64*
+        auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
+        assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
+        Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
+        Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo());
+      }
       if (AtomicCounterUpdatePromoted)
         // automic update currently can only be promoted across the current
         // loop, not the whole loop nest.
         Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
                                 MaybeAlign(),
                                 AtomicOrdering::SequentiallyConsistent);
       else {
         LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");

llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll

+; RUN: opt < %s -instrprof -runtime-counter-relocation -do-counter-promotion=true -S | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@__profn_foo = private constant [3 x i8] c"foo"
+
+define void @foo(i1 %c) {
+entry:
+; CHECK: %[[BIAS:[0-9]+]] = load i64, i64* @__llvm_profile_counter_bias
+  br label %while.cond
+
+while.cond:                                       ; preds = %land.rhs, %while.cond.preheader
+; CHECK: %[[COUNT:[a-z0-9.]+]] = phi i64 [ %[[LAND_COUNT:[0-9]+]], %land.rhs ], [ 0, %entry ]
+  br i1 %c, label %while.cond.cleanup_crit_edge, label %land.rhs
+
+while.cond.cleanup_crit_edge:                     ; preds = %while.cond
+; CHECK: %[[COUNTER_PTR:[0-9]+]] = add i64 ptrtoint ([1 x i64]* @__profc_foo to i64), %[[BIAS]]
+; CHECK: %[[COUNTER_ADDR:[0-9]+]] = inttoptr i64 %[[COUNTER_PTR]] to i64*
+; CHECK: %[[COUNTER_PROMO:[a-z0-9.]+]] = load i64, i64* %[[COUNTER_ADDR]]
+; CHECK: %[[VALUE:[0-9]+]] = add i64 %[[COUNTER_PROMO]], %[[COUNT]]
+; CHECK: store i64 %[[VALUE]], i64* %[[COUNTER_ADDR]]
+  br label %cleanup
+
+land.rhs:                                         ; preds = %while.cond
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i32 1, i32 0)
+; CHECK: %[[LAND_COUNT]] = add i64 %[[COUNT]], 1
+  br label %while.cond
+
+cleanup:                                          ; preds = %while.cond.cleanup_crit_edge
+  ret void
+}
+
+declare void @llvm.instrprof.increment(i8 *, i64, i32, i32)