Index: include/llvm/Transforms/InstrProfiling.h =================================================================== --- include/llvm/Transforms/InstrProfiling.h +++ include/llvm/Transforms/InstrProfiling.h @@ -60,11 +60,21 @@ GlobalVariable *NamesVar; size_t NamesSize; + // vector of counter load/store pairs to be register promoted. + std::vector> PromotionCandidates; + // The start value of precise value profile range for memory intrinsic sizes. int64_t MemOPSizeRangeStart; // The end value of precise value profile range for memory intrinsic sizes. int64_t MemOPSizeRangeLast; + /// Lower instrumentation intrinsics in the function. Returns true if there + /// any lowering. + bool lowerIntrinsics(Function *F); + + /// Register promote counter loads and stores in loops. + void promoteCounterLoadStores(Function *F); + /// Count the number of instrumented value sites for the function. void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins); Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -116,6 +116,9 @@ // Add the 'noredzone' attribute to added runtime library calls. bool NoRedZone = false; + // Do counter register promotion + bool DoCounterPromotion = false; + // Name of the profile file to use as output std::string InstrProfileOutput; Index: lib/Passes/PassBuilder.cpp =================================================================== --- lib/Passes/PassBuilder.cpp +++ lib/Passes/PassBuilder.cpp @@ -460,10 +460,15 @@ if (RunProfileGen) { MPM.addPass(PGOInstrumentationGen()); + FunctionPassManager FPM; + FPM.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass())); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + // Add the profile lowering pass. InstrProfOptions Options; if (!ProfileGenFile.empty()) Options.InstrProfileOutput = ProfileGenFile; + Options.DoCounterPromotion = true; MPM.addPass(InstrProfiling(Options)); } Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -292,6 +292,8 @@ InstrProfOptions Options; if (!PGOInstrGen.empty()) Options.InstrProfileOutput = PGOInstrGen; + Options.DoCounterPromotion = true; + MPM.add(createLoopRotatePass()); MPM.add(createInstrProfilingLegacyPass(Options)); } if (!PGOInstrUse.empty()) Index: lib/Transforms/Instrumentation/InstrProfiling.cpp =================================================================== --- lib/Transforms/Instrumentation/InstrProfiling.cpp +++ lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -19,19 +19,21 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/Pass.h" @@ -40,7 +42,10 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Transforms/Utils/SSAUpdater.h" #include #include #include @@ -92,6 +97,30 @@ // is usually smaller than 2. cl::init(1.0)); +cl::opt AtomicCounterUpdatePromoted( + "atomic-counter-update-promoted", cl::ZeroOrMore, + cl::desc("Do counter update using atomic fetch add " + " for promoted counters only"), + cl::init(false)); + +// If the option is not specified, the default behavior about whether +// counter promotion is done depends on how instrumentaiton lowering +// pipeline is setup, i.e., the default value of true of this option +// does not mean the promotion will be done by default. Explicitly +// setting this option can override the default behavior. +cl::opt DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore, + cl::desc("Do counter register promotion"), + cl::init(true)); +cl::opt MaxNumberOfPromotions( + cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(10), + cl::desc("Max number counter promotions per loop to avoid" + " increasing register pressure too much")); + +cl::opt SpeculativeCounterPromotion( + cl::ZeroOrMore, "speculative-counter-promotion", cl::init(false), + cl::desc("Allow counter promotion for loops with multiple exiting blocks " + " or top-tested loops. ")); + class InstrProfilingLegacyPass : public ModulePass { InstrProfiling InstrProf; @@ -116,6 +145,104 @@ } }; +class PGOCounterPromoterHelper : public LoadAndStorePromoter { +public: + PGOCounterPromoterHelper(Instruction *L, Instruction *S, SSAUpdater &SSA, + Value *Init, BasicBlock *PH, + ArrayRef ExitBlocks, + ArrayRef InsertPts) + : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks), + InsertPts(InsertPts) { + assert(isa(L)); + assert(isa(S)); + SSA.AddAvailableValue(PH, Init); + } + void doExtraRewritesBeforeFinalDeletion() const override { + for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { + BasicBlock *ExitBlock = ExitBlocks[i]; + Instruction *InsertPos = InsertPts[i]; + Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock); + Value *Addr = cast(Store)->getPointerOperand(); + IRBuilder<> Builder(InsertPos); + if (AtomicCounterUpdatePromoted) + Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue, + AtomicOrdering::SequentiallyConsistent); + else { + LoadInst *OldVal = Builder.CreateLoad(Addr, "pgocount.promoted"); + auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue); + Builder.CreateStore(NewVal, Addr); + } + } + } + +private: + Instruction *Store; + ArrayRef ExitBlocks; + ArrayRef InsertPts; +}; + +class PGOCounterPromoter { +public: + PGOCounterPromoter(ArrayRef> Cands, + Loop &Loop) + : Candidates(Cands), ExitBlocks(), InsertPts(), ParentLoop(Loop) { + + ParentLoop.getExitBlocks(ExitBlocks); + + InsertPts.reserve(ExitBlocks.size()); + for (BasicBlock *ExitBlock : ExitBlocks) + InsertPts.push_back(&*ExitBlock->getFirstInsertionPt()); + } + + bool run() { + // We can't insert into a catchswitch. + bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) { + return isa(Exit->getTerminator()); + }); + + if (HasCatchSwitch) + return false; + + if (!ParentLoop.hasDedicatedExits()) + return false; + + BasicBlock *PH = ParentLoop.getLoopPreheader(); + if (!PH) + return false; + + BasicBlock *H = ParentLoop.getHeader(); + bool TopTested = + ((ParentLoop.getBlocks().size() > 1) && ParentLoop.isLoopExiting(H)); + if (!SpeculativeCounterPromotion && + (TopTested || ParentLoop.getExitingBlock() == nullptr)) + return false; + + unsigned Promoted = 0; + for (auto Cand : Candidates) { + + SmallVector NewPHIs; + SSAUpdater SSA(&NewPHIs); + Value *InitVal = ConstantInt::get(Cand.first->getType(), 0); + PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal, + PH, ExitBlocks, InsertPts); + Promoter.run(SmallVector({Cand.first, Cand.second})); + Promoted++; + if (Promoted >= MaxNumberOfPromotions) + break; + } + + DEBUG(dbgs() << Promoted << " counters promoted for loop (depth=" + << ParentLoop.getLoopDepth() << ")\n"); + return Promoted != 0; + } + +private: + ArrayRef> Candidates; + SmallVector ExitBlocks; + SmallVector InsertPts; + Loop &ParentLoop; +}; + } // end anonymous namespace PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) { @@ -147,6 +274,60 @@ return dyn_cast(Instr); } +bool InstrProfiling::lowerIntrinsics(Function *F) { + bool MadeChange = false; + PromotionCandidates.clear(); + for (BasicBlock &BB : *F) { + for (auto I = BB.begin(), E = BB.end(); I != E;) { + auto Instr = I++; + InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr); + if (Inc) { + lowerIncrement(Inc); + MadeChange = true; + } else if (auto *Ind = dyn_cast(Instr)) { + lowerValueProfileInst(Ind); + MadeChange = true; + } + } + } + + if (!MadeChange) + return false; + + promoteCounterLoadStores(F); + return true; +} + +void InstrProfiling::promoteCounterLoadStores(Function *F) { + + if ((DoCounterPromotion.getNumOccurrences() == 0 && + !Options.DoCounterPromotion) || + !DoCounterPromotion) + return; + + DominatorTree DT(*F); + LoopInfo LI(DT); + DenseMap, 8>> + LoopPromotionCandidates; + + for (auto LoadStore : PromotionCandidates) { + auto *CounterLoad = LoadStore.first; + auto *CounterStore = LoadStore.second; + BasicBlock *BB = CounterLoad->getParent(); + Loop *ParentLoop = LI.getLoopFor(BB); + if (!ParentLoop) + continue; + LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore); + } + + SmallVector Loops = LI.getLoopsInPreorder(); + + for (auto *Loop : Loops) { + PGOCounterPromoter Promoter(LoopPromotionCandidates[Loop], *Loop); + Promoter.run(); + } +} + bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) { bool MadeChange = false; @@ -179,18 +360,7 @@ } for (Function &F : M) - for (BasicBlock &BB : F) - for (auto I = BB.begin(), E = BB.end(); I != E;) { - auto Instr = I++; - InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr); - if (Inc) { - lowerIncrement(Inc); - MadeChange = true; - } else if (auto *Ind = dyn_cast(Instr)) { - lowerValueProfileInst(Ind); - MadeChange = true; - } - } + MadeChange |= lowerIntrinsics(&F); if (GlobalVariable *CoverageNamesVar = M.getNamedGlobal(getCoverageUnusedNamesVarName())) { @@ -303,9 +473,11 @@ IRBuilder<> Builder(Inc); uint64_t Index = Inc->getIndex()->getZExtValue(); Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index); - Value *Count = Builder.CreateLoad(Addr, "pgocount"); - Count = Builder.CreateAdd(Count, Inc->getStep()); - Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr)); + Value *Load = Builder.CreateLoad(Addr, "pgocount"); + auto *Count = Builder.CreateAdd(Load, Inc->getStep()); + auto *Store = Builder.CreateStore(Count, Addr); + Inc->replaceAllUsesWith(Store); + PromotionCandidates.emplace_back(cast(Load), Store); Inc->eraseFromParent(); } Index: test/Transforms/PGOProfile/counter_promo.ll =================================================================== --- test/Transforms/PGOProfile/counter_promo.ll +++ test/Transforms/PGOProfile/counter_promo.ll @@ -0,0 +1,55 @@ +; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO %s +; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO %s +@g = common global i32 0, align 4 + +define void @foo(i32 %n, i32 %N) { +; PROMO-LABEL: @foo +bb: + %tmp = add nsw i32 %n, 1 + %tmp1 = add nsw i32 %n, -1 + br label %bb2 + +bb2: ; preds = %bb9, %bb + %i.0 = phi i32 [ 0, %bb ], [ %tmp10, %bb9 ] + %tmp3 = icmp slt i32 %i.0, %tmp + br i1 %tmp3, label %bb4, label %bb5 + +bb4: ; preds = %bb2 + tail call void @bar(i32 1) + br label %bb9 + +bb5: ; preds = %bb2 + %tmp6 = icmp slt i32 %i.0, %tmp1 + br i1 %tmp6, label %bb7, label %bb8 + +bb7: ; preds = %bb5 + tail call void @bar(i32 2) + br label %bb9 + +bb8: ; preds = %bb5 + tail call void @bar(i32 3) + br label %bb9 + +bb9: ; preds = %bb8, %bb7, %bb4 + %tmp10 = add nsw i32 %i.0, 1 + %tmp11 = icmp slt i32 %tmp10, %N + br i1 %tmp11, label %bb2, label %bb12 + +bb12: ; preds = %bb9 + ret void +; PROMO: %pgocount.promoted = load {{.*}} @__profc_foo{{.*}} 0) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0) +; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1) +; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2) +; PROMO-NEXT: %pgocount3 = load {{.*}} @__profc_foo{{.*}} 3) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3) + +} + +declare void @bar(i32) Index: test/Transforms/PGOProfile/counter_promo_mexits.ll =================================================================== --- test/Transforms/PGOProfile/counter_promo_mexits.ll +++ test/Transforms/PGOProfile/counter_promo_mexits.ll @@ -0,0 +1,79 @@ +; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s +; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -speculative-counter-promotion -S | FileCheck --check-prefix=PROMO %s + +@g = common local_unnamed_addr global i32 0, align 4 + +define void @foo(i32 %arg) local_unnamed_addr { +; PROMO-LABEL: @foo +bb: + %tmp = add nsw i32 %arg, -1 + br label %bb1 +bb1: ; preds = %bb11, %bb + %tmp2 = phi i32 [ 0, %bb ], [ %tmp12, %bb11 ] + %tmp3 = icmp sgt i32 %tmp2, %arg + br i1 %tmp3, label %bb7, label %bb4 + +bb4: ; preds = %bb1 + tail call void @bar(i32 1) + %tmp5 = load i32, i32* @g, align 4 + %tmp6 = icmp sgt i32 %tmp5, 100 + br i1 %tmp6, label %bb15_0, label %bb11 + +bb7: ; preds = %bb1 + %tmp8 = icmp slt i32 %tmp2, %tmp + br i1 %tmp8, label %bb9, label %bb10 + +bb9: ; preds = %bb7 + tail call void @bar(i32 2) + br label %bb11 + +bb10: ; preds = %bb7 + tail call void @bar(i32 3) + br label %bb11 + +bb11: ; preds = %bb10, %bb9, %bb4 + %tmp12 = add nuw nsw i32 %tmp2, 1 + %tmp13 = icmp slt i32 %tmp2, 99 + br i1 %tmp13, label %bb1, label %bb14 + +bb14: ; preds = %bb11 +; PROMO-LABEL: bb14: + tail call void @bar(i32 0) + br label %bb15 +; PROMO: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0) +; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1) +; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2) +; PROMO-NEXT: %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 3) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3) + + +bb15_0: ; preds = %bb11 +; PROMO-LABEL: bb15_0: + br label %bb15 +; PROMO: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0) +; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1) +; PROMO-NEXT: %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2) +; PROMO-NEXT: %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 4) +; PROMO-NEXT: add +; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}4) + + +bb15: ; preds = %bb14, %bb4 + tail call void @bar(i32 1) + ret void +} + +declare void @bar(i32) local_unnamed_addr Index: test/profile/Linux/counter_promo_for.c =================================================================== --- test/profile/Linux/counter_promo_for.c +++ test/profile/Linux/counter_promo_for.c @@ -0,0 +1,59 @@ +// RUN: rm -fr %t.promo.prof +// RUN: rm -fr %t.nopromo.prof +// RUN: %clang_pgogen=%t.promo.prof/ -o %t.promo.gen -O2 %s +// RUN: %clang_pgogen=%t.promo.prof/ -o %t.promo.gen.ll -emit-llvm -S -O2 %s +// RUN: cat %t.promo.gen.ll | FileCheck --check-prefix=PROMO %s +// RUN: %run %t.promo.gen +// RUN: llvm-profdata merge -o %t.promo.profdata %t.promo.prof/ +// RUN: llvm-profdata show --counts --all-functions %t.promo.profdata > %t.promo.dump +// RUN: %clang_pgogen=%t.nopromo.prof/ -mllvm -do-counter-promotion=false -o %t.nopromo.gen -O2 %s +// RUN: %clang_pgogen=%t.nopromo.prof/ -mllvm -do-counter-promotion=false -o %t.nopromo.gen.ll -emit-llvm -S -O2 %s +// RUN: cat %t.nopromo.gen.ll | FileCheck --check-prefix=NOPROMO %s +// RUN: %run %t.nopromo.gen +// RUN: llvm-profdata merge -o %t.nopromo.profdata %t.nopromo.prof/ +// RUN: llvm-profdata show --counts --all-functions %t.nopromo.profdata > %t.nopromo.dump +// RUN: diff %t.promo.profdata %t.nopromo.profdata + +int g; +__attribute__((noinline)) void bar(int i) { g += i; } + +__attribute__((noinline)) void foo(int n, int N) { +// PROMO-LABEL: @foo +// PROMO: load{{.*}}@__profc_foo{{.*}} 0){{.*}} +// PROMO-NEXT: add +// PROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 0){{.*}} +// PROMO-NEXT: load{{.*}}@__profc_foo{{.*}} 1){{.*}} +// PROMO-NEXT: add +// PROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 1){{.*}} +// PROMO-NEXT: load{{.*}}@__profc_foo{{.*}} 2){{.*}} +// PROMO-NEXT: add +// PROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 2){{.*}} +// PROMO: load{{.*}}@__profc_foo{{.*}} 3){{.*}} +// PROMO-NEXT: add +// PROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 3){{.*}} +// +// NOPROMO-LABEL: @foo +// NOPROMO: load{{.*}}@__profc_foo{{.*}} 0){{.*}} +// NOPROMO-NEXT: add +// NOPROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 0){{.*}} +// NOPROMO: load{{.*}}@__profc_foo{{.*}} 1){{.*}} +// NOPROMO-NEXT: add +// NOPROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 1){{.*}} +// NOPROMO: load{{.*}}@__profc_foo{{.*}} 2){{.*}} +// NOPROMO-NEXT: add +// NOPROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 2){{.*}} + int i; + for (i = 0; i < N; i++) { + if (i < n + 1) + bar(1); + else if (i < n - 1) + bar(2); + else + bar(3); + } +} + +int main() { + foo(10, 20); + return 0; +} Index: test/profile/Linux/counter_promo_while.c =================================================================== --- test/profile/Linux/counter_promo_while.c +++ test/profile/Linux/counter_promo_while.c @@ -0,0 +1,55 @@ +// RUN: rm -fr %t.promo.prof +// RUN: rm -fr %t.nopromo.prof +// RUN: %clang_pgogen=%t.promo.prof/ -o %t.promo.gen -O2 %s +// RUN: %clang_pgogen=%t.promo.prof/ -o %t.promo.gen.ll -emit-llvm -S -O2 %s +// RUN: cat %t.promo.gen.ll | FileCheck --check-prefix=PROMO %s +// RUN: %run %t.promo.gen +// RUN: llvm-profdata merge -o %t.promo.profdata %t.promo.prof/ +// RUN: llvm-profdata show --counts --all-functions %t.promo.profdata > %t.promo.dump +// RUN: %clang_pgogen=%t.nopromo.prof/ -mllvm -do-counter-promotion=false -o %t.nopromo.gen -O2 %s +// RUN: %clang_pgogen=%t.nopromo.prof/ -mllvm -do-counter-promotion=false -o %t.nopromo.gen.ll -emit-llvm -S -O2 %s +// RUN: cat %t.nopromo.gen.ll | FileCheck --check-prefix=NOPROMO %s +// RUN: %run %t.nopromo.gen +// RUN: llvm-profdata merge -o %t.nopromo.profdata %t.nopromo.prof/ +// RUN: llvm-profdata show --counts --all-functions %t.nopromo.profdata > %t.nopromo.dump +// RUN: diff %t.promo.profdata %t.nopromo.profdata +int g; +__attribute__((noinline)) void bar(int i) { g += i; } +__attribute__((noinline)) void foo(int n, int N) { +// PROMO-LABEL: @foo +// PROMO: load{{.*}}@__profc_foo{{.*}} 0){{.*}} +// PROMO-NEXT: add +// PROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 0){{.*}} +// PROMO-NEXT: load{{.*}}@__profc_foo{{.*}} 1){{.*}} +// PROMO-NEXT: add +// PROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 1){{.*}} +// PROMO-NEXT: load{{.*}}@__profc_foo{{.*}} 2){{.*}} +// PROMO-NEXT: add +// PROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 2){{.*}} +// +// NOPROMO-LABEL: @foo +// NOPROMO: load{{.*}}@__profc_foo{{.*}} 0){{.*}} +// NOPROMO-NEXT: add +// NOPROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 0){{.*}} +// NOPROMO: load{{.*}}@__profc_foo{{.*}} 1){{.*}} +// NOPROMO-NEXT: add +// NOPROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 1){{.*}} +// NOPROMO: load{{.*}}@__profc_foo{{.*}} 2){{.*}} +// NOPROMO-NEXT: add +// NOPROMO-NEXT: store{{.*}}@__profc_foo{{.*}} 2){{.*}} + int i = 0; + while (i < N) { + if (i < n + 1) + bar(1); + else if (i < n - 1) + bar(2); + else + bar(3); + i++; + } +} + +int main() { + foo(10, 20); + return 0; +}