Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -347,6 +347,7 @@ void initializeSlotIndexesPass(PassRegistry&); void initializeSpeculativeExecutionLegacyPassPass(PassRegistry&); void initializeSpillPlacementPass(PassRegistry&); +void initializeSplitIndirectBrCriticalEdgesLegacyPassPass(PassRegistry&); void initializeStackColoringPass(PassRegistry&); void initializeStackMapLivenessPass(PassRegistry&); void initializeStackProtectorPass(PassRegistry&); Index: include/llvm/LinkAllPasses.h =================================================================== --- include/llvm/LinkAllPasses.h +++ include/llvm/LinkAllPasses.h @@ -99,6 +99,7 @@ (void) llvm::createPGOInstrumentationUseLegacyPass(); (void) llvm::createPGOIndirectCallPromotionLegacyPass(); (void) llvm::createPGOMemOPSizeOptLegacyPass(); + (void) llvm::createSplitIndirectBrCriticalEdgesLegacyPass(); (void) llvm::createInstrProfilingLegacyPass(); (void) llvm::createFunctionImportPass(); (void) llvm::createFunctionInliningPass(); Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -77,6 +77,8 @@ bool SamplePGO = false); FunctionPass *createPGOMemOPSizeOptLegacyPass(); +FunctionPass *createSplitIndirectBrCriticalEdgesLegacyPass(); + // Helper function to check if it is legal to promote indirect call \p Inst // to a direct call of function \p F. Stores the reason in \p Reason. bool isLegalToPromote(Instruction *Inst, Function *F, const char **Reason); Index: include/llvm/Transforms/PGOInstrumentation.h =================================================================== --- include/llvm/Transforms/PGOInstrumentation.h +++ include/llvm/Transforms/PGOInstrumentation.h @@ -70,6 +70,17 @@ void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count); +/// The split-indirectbr-critical-edges pass. +class SplitIndirectBrCriticalEdgesPass : + public PassInfoMixin { +public: + SplitIndirectBrCriticalEdgesPass() {} + + static StringRef name() { return "SplitIndirectBrCriticalEdgesPass"; } + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + } // end namespace llvm #endif // LLVM_TRANSFORMS_PGOINSTRUMENTATION_H Index: lib/Passes/PassBuilder.cpp =================================================================== --- lib/Passes/PassBuilder.cpp +++ lib/Passes/PassBuilder.cpp @@ -496,6 +496,10 @@ FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies. FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks. FPM.addPass(InstCombinePass()); // Combine silly sequences. + // Split the indirectbr critical edges before the PGO instrumentation gen + // pass as that could change CFG enough to invalidate the BPI/BFI/MST if + // that's done during the pass. + FPM.addPass(SplitIndirectBrCriticalEdgesPass()); invokePeepholeEPCallbacks(FPM, Level); CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); @@ -523,8 +527,9 @@ MPM.addPass(InstrProfiling(Options)); } - if (!ProfileUseFile.empty()) + if (!ProfileUseFile.empty()) { MPM.addPass(PGOInstrumentationUse(ProfileUseFile)); + } } static InlineParams Index: lib/Passes/PassRegistry.def =================================================================== --- lib/Passes/PassRegistry.def +++ lib/Passes/PassRegistry.def @@ -200,6 +200,8 @@ FUNCTION_PASS("slp-vectorizer", SLPVectorizerPass()) FUNCTION_PASS("speculative-execution", SpeculativeExecutionPass()) FUNCTION_PASS("spec-phis", SpeculateAroundPHIsPass()) +FUNCTION_PASS("split-indirectbr-critical-edges", + SplitIndirectBrCriticalEdgesPass()) FUNCTION_PASS("sroa", SROA()) FUNCTION_PASS("tailcallelim", TailCallElimPass()) FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass()) Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -280,6 +280,10 @@ MPM.add(createEarlyCSEPass()); // Catch trivial redundancies MPM.add(createCFGSimplificationPass()); // Merge & remove BBs MPM.add(createInstructionCombiningPass()); // Combine silly seq's + // Split the indirectbr critical edges before the PGO instrumentation gen + // pass as that could change CFG enough to invalidate the BPI/BFI/MST if + // that's done during the pass. + MPM.add(createSplitIndirectBrCriticalEdgesLegacyPass()); addExtensionsToPM(EP_Peephole, MPM); } if (EnablePGOInstrGen) { Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -418,6 +418,21 @@ } }; +class SplitIndirectBrCriticalEdgesLegacyPass : public FunctionPass { +public: + static char ID; + + SplitIndirectBrCriticalEdgesLegacyPass() : FunctionPass(ID) { + } + + StringRef getPassName() const override { + return "SplitIndirectBrCriticalEdgesPass"; + } + +private: + bool runOnFunction(Function &F) override; +}; + } // end anonymous namespace char PGOInstrumentationGenLegacyPass::ID = 0; @@ -446,6 +461,19 @@ return new PGOInstrumentationUseLegacyPass(Filename.str()); } +char SplitIndirectBrCriticalEdgesLegacyPass::ID = 0; + +INITIALIZE_PASS_BEGIN(SplitIndirectBrCriticalEdgesLegacyPass, + "split-indirectbr-critical-edges", + "Split IndirectBr critical edges.", false, false) +INITIALIZE_PASS_END(SplitIndirectBrCriticalEdgesLegacyPass, + "split-indirectbr-critical-edges", + "Split IndirectBr critical edges.", false, false) + +FunctionPass *llvm::createSplitIndirectBrCriticalEdgesLegacyPass() { + return new SplitIndirectBrCriticalEdgesLegacyPass(); +} + namespace { /// \brief An MST based instrumentation for PGO @@ -1559,6 +1587,19 @@ return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI); } +PreservedAnalyses +SplitIndirectBrCriticalEdgesPass::run(Function &F, + FunctionAnalysisManager &AM) { + return SplitIndirectBrCriticalEdges(F) + ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + +bool SplitIndirectBrCriticalEdgesLegacyPass::runOnFunction(Function &F) { + if (skipFunction(F)) + return false; + return SplitIndirectBrCriticalEdges(F); +} + static std::string getSimpleNodeName(const BasicBlock *Node) { if (!Node->getName().empty()) return Node->getName(); Index: test/Transforms/PGOProfile/split-indirectbr-critical-edges.ll =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/split-indirectbr-critical-edges.ll @@ -0,0 +1,41 @@ + +; RUN: opt < %s -passes=split-indirectbr-critical-edges -S | FileCheck %s +; RUN: opt < %s -passes='function(split-indirectbr-critical-edges),pgo-instr-gen' -S | FileCheck %s + +; Function Attrs: norecurse nounwind readnone uwtable +define i32 @bar(i32 %v) local_unnamed_addr #0 { +entry: + %mul = shl nsw i32 %v, 1 + ret i32 %mul +} + +; Function Attrs: norecurse nounwind readonly uwtable +define i32 @foo(i8* nocapture readonly %p) #1 { +entry: + %targets = alloca [256 x i8*], align 16 + %arrayidx1 = getelementptr inbounds [256 x i8*], [256 x i8*]* %targets, i64 0, i64 93 + store i8* blockaddress(@foo, %if.end), i8** %arrayidx1, align 8 + br label %for.cond2 + +for.cond2: ; preds = %if.end, %for.cond2, %entry +; CHECK: for.cond2: ; preds = %.split1 + %p.addr.0 = phi i8* [ %p, %entry ], [ %incdec.ptr5, %if.end ], [ %incdec.ptr, %for.cond2 ] + %incdec.ptr = getelementptr inbounds i8, i8* %p.addr.0, i64 1 + %0 = load i8, i8* %p.addr.0, align 1 + %cond = icmp eq i8 %0, 93 + br i1 %cond, label %if.end.preheader, label %for.cond2 + +if.end.preheader: ; preds = %for.cond2 + br label %if.end + +if.end: ; preds = %if.end.preheader, %if.end +; CHECK: if.end: ; preds = %.split1 + %p.addr.1 = phi i8* [ %incdec.ptr5, %if.end ], [ %incdec.ptr, %if.end.preheader ] + %incdec.ptr5 = getelementptr inbounds i8, i8* %p.addr.1, i64 1 + %1 = load i8, i8* %p.addr.1, align 1 + %idxprom6 = zext i8 %1 to i64 + %arrayidx7 = getelementptr inbounds [256 x i8*], [256 x i8*]* %targets, i64 0, i64 %idxprom6 + %2 = load i8*, i8** %arrayidx7, align 8 + indirectbr i8* %2, [label %for.cond2, label %if.end] +; CHECK: indirectbr i8* %2, [label %for.cond2, label %if.end] +}