diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h --- a/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -438,6 +438,10 @@ /// immediately before machine code is emitted. virtual void addPreEmitPass() { } + /// This pass may be implemented by targets that want to run passes + /// immediately after basic block sections are assigned. + virtual void addPostBBSections() {} + /// Targets may add passes immediately before machine code is emitted in this /// callback. This is called even later than `addPreEmitPass`. // FIXME: Rename `addPreEmitPass` to something more sensible given its actual diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -1273,6 +1273,8 @@ addPass(createMachineFunctionSplitterPass()); } + addPostBBSections(); + if (!DisableCFIFixup && TM->Options.EnableCFIFixup) addPass(createCFIFixup()); diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -517,6 +517,7 @@ void addPostRegAlloc() override; void addPreSched2() override; void addPreEmitPass() override; + void addPostBBSections() override; void addPreEmitPass2() override; std::unique_ptr getCSEConfig() const override; @@ -806,11 +807,6 @@ if (EnableBranchTargets) addPass(createAArch64BranchTargetsPass()); - // Relax conditional branch instructions if they're otherwise out of - // range of their destination. - if (BranchRelaxation) - addPass(&BranchRelaxationPassID); - if (TM->getTargetTriple().isOSWindows()) { // Identify valid longjmp targets for Windows Control Flow Guard. addPass(createCFGuardLongjmpPass()); @@ -818,14 +814,21 @@ addPass(createEHContGuardCatchretPass()); } - if (TM->getOptLevel() != CodeGenOpt::None && EnableCompressJumpTables) - addPass(createAArch64CompressJumpTablesPass()); - if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH && TM->getTargetTriple().isOSBinFormatMachO()) addPass(createAArch64CollectLOHPass()); } +void AArch64PassConfig::addPostBBSections() { + // Relax conditional branch instructions if they're otherwise out of + // range of their destination. + if (BranchRelaxation) + addPass(&BranchRelaxationPassID); + + if (TM->getOptLevel() != CodeGenOpt::None && EnableCompressJumpTables) + addPass(createAArch64CompressJumpTablesPass()); +} + void AArch64PassConfig::addPreEmitPass2() { // SVE bundles move prefixes with destructive operations. BLR_RVMARKER pseudo // instructions are lowered to bundles as well. diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -72,11 +72,11 @@ ; CHECK-NEXT: Implement the 'patchable-function' attribute ; CHECK-NEXT: Workaround A53 erratum 835769 pass ; CHECK-NEXT: AArch64 Branch Targets -; CHECK-NEXT: Branch relaxation pass ; CHECK-NEXT: Contiguously Lay Out Funclets ; CHECK-NEXT: StackMap Liveness Analysis ; CHECK-NEXT: Live DEBUG_VALUE analysis ; CHECK-NEXT: Machine Sanitizer Binary Metadata +; CHECK-NEXT: Branch relaxation pass ; CHECK-NEXT: Insert CFI remember/restore state instructions ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -218,14 +218,14 @@ ; CHECK-NEXT: Machine Copy Propagation Pass ; CHECK-NEXT: Workaround A53 erratum 835769 pass ; CHECK-NEXT: AArch64 Branch Targets -; CHECK-NEXT: Branch relaxation pass -; CHECK-NEXT: AArch64 Compress Jump Tables ; CHECK-NEXT: Contiguously Lay Out Funclets ; CHECK-NEXT: StackMap Liveness Analysis ; CHECK-NEXT: Live DEBUG_VALUE analysis ; CHECK-NEXT: Machine Sanitizer Binary Metadata ; CHECK-NEXT: Machine Outliner ; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Branch relaxation pass +; CHECK-NEXT: AArch64 Compress Jump Tables ; CHECK-NEXT: Insert CFI remember/restore state instructions ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter diff --git a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll --- a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll +++ b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll @@ -34,6 +34,14 @@ ; HOTNESS-NEXT: Executing Pass 'Function Pass Manager' ; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' ; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Executing Pass 'Branch relaxation pass' on Function 'empty_func' +; HOTNESS-NEXT: Freeing Pass 'Branch relaxation pass' on Function 'empty_func' +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Executing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; HOTNESS-NEXT: Freeing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' ; HOTNESS-NEXT: Executing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; HOTNESS-NEXT: Freeing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' @@ -59,6 +67,14 @@ ; NO_HOTNESS-NEXT: Executing Pass 'Function Pass Manager' ; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' ; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Executing Pass 'Branch relaxation pass' on Function 'empty_func' +; NO_HOTNESS-NEXT: Freeing Pass 'Branch relaxation pass' on Function 'empty_func' +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Executing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; NO_HOTNESS-NEXT: Freeing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' ; NO_HOTNESS-NEXT: Executing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; NO_HOTNESS-NEXT: Freeing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' diff --git a/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll b/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll --- a/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll +++ b/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll @@ -3,8 +3,8 @@ ; Windows runtime passes the establisher frame as the second argument to the ; termination handler. Check that we copy it into fp. -; CHECK: ?dtor$3@?0?main@4HA": -; CHECK: .seh_proc "?dtor$3@?0?main@4HA" +; CHECK: ?dtor$6@?0?main@4HA": +; CHECK: .seh_proc "?dtor$6@?0?main@4HA" ; CHECK: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill ; CHECK-NEXT: .seh_save_fplr_x 16 ; CHECK-NEXT: .seh_endprologue diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll --- a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll +++ b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll @@ -44,7 +44,7 @@ ; CHECK: [[CATCHRETDEST:.LBB0_[0-9]+]]: // %catchret.dest ; Check the catch funclet. -; CHECK-LABEL: "?catch$2@?0??func@@YAHXZ@4HA": +; CHECK-LABEL: "?catch$4@?0??func@@YAHXZ@4HA": ; Check that the stack space is allocated only for the callee saved registers. ; CHECK: stp x19, x20, [sp, #-48]! @@ -66,7 +66,7 @@ ; CHECK: stur w8, [x29, [[X_OFFSET]]] ; Check that the funclet branches back to the catchret destination -; CHECK: adrp x0, .LBB0_3 +; CHECK: adrp x0, .LBB0_2 ; CHECK-NEXT: add x0, x0, [[CATCHRETDEST]] @@ -91,7 +91,7 @@ ; UNWIND-NEXT: ; str x21, [sp, #16] ; UNWIND-NEXT: ; stp x19, x20, [sp, #-64]! ; UNWIND-NEXT: ; end -; UNWIND: Function: ?catch$2@?0??func@@YAHXZ@4HA +; UNWIND: Function: ?catch$4@?0??func@@YAHXZ@4HA ; UNWIND: Prologue [ ; UNWIND-NEXT: ; stp x29, x30, [sp, #32] ; UNWIND-NEXT: ; str x28, [sp, #24]