diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1066,10 +1066,12 @@ // Create an early function pass manager to cleanup the output of the // frontend. FunctionPassManager EarlyFPM(DebugLogging); + // Lower llvm.expect to metadata before attempting transforms. + // Compare/branch metadata may alter the behavior of passes like SimplifyCFG. + EarlyFPM.addPass(LowerExpectIntrinsicPass()); EarlyFPM.addPass(SimplifyCFGPass()); EarlyFPM.addPass(SROA()); EarlyFPM.addPass(EarlyCSEPass()); - EarlyFPM.addPass(LowerExpectIntrinsicPass()); if (PTO.Coroutines) EarlyFPM.addPass(CoroEarlyPass()); if (Level == OptimizationLevel::O3) diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp --- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp +++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp @@ -316,10 +316,12 @@ addInitialAliasAnalysisPasses(FPM); + // Lower llvm.expect to metadata before attempting transforms. + // Compare/branch metadata may alter the behavior of passes like SimplifyCFG. + FPM.add(createLowerExpectIntrinsicPass()); FPM.add(createCFGSimplificationPass()); FPM.add(createSROAPass()); FPM.add(createEarlyCSEPass()); - FPM.add(createLowerExpectIntrinsicPass()); } // Do PGO instrumentation generation or use pass as the option specified. diff --git a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll --- a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll @@ -41,11 +41,11 @@ ; GCN-O1-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-NEXT: Function Alias Analysis Results ; GCN-O1-NEXT: Simplify well-known AMD library calls +; GCN-O1-NEXT: Lower 'expect' Intrinsics ; GCN-O1-NEXT: Simplify the CFG ; GCN-O1-NEXT: Dominator Tree Construction ; GCN-O1-NEXT: SROA ; GCN-O1-NEXT: Early CSE -; GCN-O1-NEXT: Lower 'expect' Intrinsics ; GCN-O1-NEXT: Pass Arguments: ; GCN-O1-NEXT: Target Library Information @@ -350,11 +350,11 @@ ; GCN-O2-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O2-NEXT: Function Alias Analysis Results ; GCN-O2-NEXT: Simplify well-known AMD library calls +; GCN-O2-NEXT: Lower 'expect' Intrinsics ; GCN-O2-NEXT: Simplify the CFG ; GCN-O2-NEXT: Dominator Tree Construction ; GCN-O2-NEXT: SROA ; GCN-O2-NEXT: Early CSE -; GCN-O2-NEXT: Lower 'expect' Intrinsics ; GCN-O2-NEXT: Pass Arguments: ; GCN-O2-NEXT: Target Library Information @@ -704,11 +704,11 @@ ; GCN-O3-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O3-NEXT: Function Alias Analysis Results ; GCN-O3-NEXT: Simplify well-known AMD library calls +; GCN-O3-NEXT: Lower 'expect' Intrinsics ; GCN-O3-NEXT: Simplify the CFG ; GCN-O3-NEXT: Dominator Tree Construction ; GCN-O3-NEXT: SROA ; GCN-O3-NEXT: Early CSE -; GCN-O3-NEXT: Lower 'expect' Intrinsics ; GCN-O3-NEXT: Pass Arguments: ; GCN-O3-NEXT: Target Library Information diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -79,6 +79,7 @@ ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Running analysis: PreservedCFGCheckerAnalysis on foo ; CHECK-O-NEXT: Starting llvm::Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -86,7 +87,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished llvm::Function pass manager run. ; CHECK-EP-PIPELINE-EARLY-SIMPLIFICATION-NEXT: Running pass: NoOpModulePass diff --git a/llvm/test/Other/new-pm-pgo.ll b/llvm/test/Other/new-pm-pgo.ll --- a/llvm/test/Other/new-pm-pgo.ll +++ b/llvm/test/Other/new-pm-pgo.ll @@ -20,10 +20,10 @@ ; USE_POST_LINK: Running pass: PGOMemOPSizeOpt ; SAMPLE_USE_O: Running pass: AddDiscriminatorsPass ; SAMPLE_USE_PRE_LINK: Running pass: AddDiscriminatorsPass +; SAMPLE_USE: Running pass: LowerExpectIntrinsicPass ; SAMPLE_USE: Running pass: SimplifyCFGPass ; SAMPLE_USE: Running pass: SROA ; SAMPLE_USE: Running pass: EarlyCSEPass -; SAMPLE_USE: Running pass: LowerExpectIntrinsicPass ; SAMPLE_USE_POST_LINK: Running pass: InstCombinePass ; SAMPLE_USE: Running pass: SampleProfileLoaderPass ; SAMPLE_USE_O: Running pass: PGOIndirectCallPromotion diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll --- a/llvm/test/Other/new-pm-thinlto-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-defaults.ll @@ -64,6 +64,7 @@ ; CHECK-PRELINK-O-NODIS-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting llvm::Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -71,7 +72,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished llvm::Function pass manager run. ; CHECK-POSTLINK-O-NEXT: Running pass: LowerTypeTestsPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -34,6 +34,7 @@ ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -41,7 +42,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run. ; CHECK-O-NEXT: Running pass: LowerTypeTestsPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -36,6 +36,7 @@ ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -43,7 +44,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: InstCombinePass on foo ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll --- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -35,6 +35,7 @@ ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -42,7 +43,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run. ; CHECK-O-NEXT: Running pass: IPSCCPPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll --- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -34,6 +34,7 @@ ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -41,7 +42,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: InstCombinePass on foo ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll --- a/llvm/test/Other/opt-O2-pipeline.ll +++ b/llvm/test/Other/opt-O2-pipeline.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll b/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll --- a/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll +++ b/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll --- a/llvm/test/Other/opt-O3-pipeline.ll +++ b/llvm/test/Other/opt-O3-pipeline.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll --- a/llvm/test/Other/opt-Os-pipeline.ll +++ b/llvm/test/Other/opt-Os-pipeline.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Transforms/PhaseOrdering/expect.ll b/llvm/test/Transforms/PhaseOrdering/expect.ll --- a/llvm/test/Transforms/PhaseOrdering/expect.ll +++ b/llvm/test/Transforms/PhaseOrdering/expect.ll @@ -1,7 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -O2 -S < %s | FileCheck %s -; FIXME: ; Given an "expect" on a compare, we should not combine ; that compare with other instructions in a way that the ; backend can't undo. Expect lowering becomes metadata, @@ -10,10 +9,11 @@ define void @PR49336(i32 %delta, i32 %tag_type, i8* %ip) { ; CHECK-LABEL: @PR49336( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TOBOOL:%.*]] = icmp slt i32 [[DELTA:%.*]], 0 -; CHECK-NEXT: [[CMP1:%.*]] = icmp ne i32 [[TAG_TYPE:%.*]], 0 -; CHECK-NEXT: [[OR_COND:%.*]] = and i1 [[TOBOOL]], [[CMP1]] -; CHECK-NEXT: br i1 [[OR_COND]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[DELTA:%.*]], 0 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END3:%.*]], !prof !0 +; CHECK: if.then: +; CHECK-NEXT: [[CMP1_NOT:%.*]] = icmp eq i32 [[TAG_TYPE:%.*]], 0 +; CHECK-NEXT: br i1 [[CMP1_NOT]], label [[IF_END3]], label [[IF_THEN2:%.*]] ; CHECK: if.then2: ; CHECK-NEXT: store i8 42, i8* [[IP:%.*]], align 1 ; CHECK-NEXT: br label [[IF_END3]]