diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h --- a/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -438,6 +438,13 @@ /// immediately before machine code is emitted. virtual void addPreEmitPass() { } + /// This pass may be implemented by targets that want to run passes after + /// instructions are finalized. All passes a target implements in this + /// function can assume that no pass after it will modify or move any + /// instructions. Note that the first pass in this function *can* modify and + /// insert instructions as necessary, but no subsequent passes can do so. + virtual void addInstructionPlacement() {} + /// Targets may add passes immediately before machine code is emitted in this /// callback. This is called even later than `addPreEmitPass`. // FIXME: Rename `addPreEmitPass` to something more sensible given its actual diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -1273,6 +1273,8 @@ addPass(createMachineFunctionSplitterPass()); } + addInstructionPlacement(); + if (!DisableCFIFixup && TM->Options.EnableCFIFixup) addPass(createCFIFixup()); diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -517,6 +517,7 @@ void addPostRegAlloc() override; void addPreSched2() override; void addPreEmitPass() override; + void addInstructionPlacement() override; void addPreEmitPass2() override; std::unique_ptr getCSEConfig() const override; @@ -806,11 +807,6 @@ if (EnableBranchTargets) addPass(createAArch64BranchTargetsPass()); - // Relax conditional branch instructions if they're otherwise out of - // range of their destination. - if (BranchRelaxation) - addPass(&BranchRelaxationPassID); - if (TM->getTargetTriple().isOSWindows()) { // Identify valid longjmp targets for Windows Control Flow Guard. addPass(createCFGuardLongjmpPass()); @@ -818,14 +814,21 @@ addPass(createEHContGuardCatchretPass()); } - if (TM->getOptLevel() != CodeGenOpt::None && EnableCompressJumpTables) - addPass(createAArch64CompressJumpTablesPass()); - if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH && TM->getTargetTriple().isOSBinFormatMachO()) addPass(createAArch64CollectLOHPass()); } +void AArch64PassConfig::addInstructionPlacement() { + // Relax conditional branch instructions if they're otherwise out of + // range of their destination. + if (BranchRelaxation) + addPass(&BranchRelaxationPassID); + + if (TM->getOptLevel() != CodeGenOpt::None && EnableCompressJumpTables) + addPass(createAArch64CompressJumpTablesPass()); +} + void AArch64PassConfig::addPreEmitPass2() { // SVE bundles move prefixes with destructive operations. BLR_RVMARKER pseudo // instructions are lowered to bundles as well. diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -72,11 +72,11 @@ ; CHECK-NEXT: Implement the 'patchable-function' attribute ; CHECK-NEXT: Workaround A53 erratum 835769 pass ; CHECK-NEXT: AArch64 Branch Targets -; CHECK-NEXT: Branch relaxation pass ; CHECK-NEXT: Contiguously Lay Out Funclets ; CHECK-NEXT: StackMap Liveness Analysis ; CHECK-NEXT: Live DEBUG_VALUE analysis ; CHECK-NEXT: Machine Sanitizer Binary Metadata +; CHECK-NEXT: Branch relaxation pass ; CHECK-NEXT: Insert CFI remember/restore state instructions ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -218,14 +218,14 @@ ; CHECK-NEXT: Machine Copy Propagation Pass ; CHECK-NEXT: Workaround A53 erratum 835769 pass ; CHECK-NEXT: AArch64 Branch Targets -; CHECK-NEXT: Branch relaxation pass -; CHECK-NEXT: AArch64 Compress Jump Tables ; CHECK-NEXT: Contiguously Lay Out Funclets ; CHECK-NEXT: StackMap Liveness Analysis ; CHECK-NEXT: Live DEBUG_VALUE analysis ; CHECK-NEXT: Machine Sanitizer Binary Metadata ; CHECK-NEXT: Machine Outliner ; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Branch relaxation pass +; CHECK-NEXT: AArch64 Compress Jump Tables ; CHECK-NEXT: Insert CFI remember/restore state instructions ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter diff --git a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll --- a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll +++ b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll @@ -34,6 +34,14 @@ ; HOTNESS-NEXT: Executing Pass 'Function Pass Manager' ; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' ; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Executing Pass 'Branch relaxation pass' on Function 'empty_func' +; HOTNESS-NEXT: Freeing Pass 'Branch relaxation pass' on Function 'empty_func' +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Executing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; HOTNESS-NEXT: Freeing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' ; HOTNESS-NEXT: Executing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; HOTNESS-NEXT: Freeing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' @@ -59,6 +67,14 @@ ; NO_HOTNESS-NEXT: Executing Pass 'Function Pass Manager' ; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' ; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Executing Pass 'Branch relaxation pass' on Function 'empty_func' +; NO_HOTNESS-NEXT: Freeing Pass 'Branch relaxation pass' on Function 'empty_func' +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Executing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; NO_HOTNESS-NEXT: Freeing Pass 'AArch64 Compress Jump Tables' on Function 'empty_func' +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' ; NO_HOTNESS-NEXT: Executing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; NO_HOTNESS-NEXT: Freeing Pass 'Insert CFI remember/restore state instructions' on Function 'empty_func' ; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' diff --git a/llvm/test/CodeGen/AArch64/jti-correct-datatype.mir b/llvm/test/CodeGen/AArch64/jti-correct-datatype.mir --- a/llvm/test/CodeGen/AArch64/jti-correct-datatype.mir +++ b/llvm/test/CodeGen/AArch64/jti-correct-datatype.mir @@ -1,4 +1,4 @@ -# RUN: llc -mtriple=aarch64-linux-gnu -start-after=branch-relaxation --filetype=obj -o %t.o %s +# RUN: llc -mtriple=aarch64-linux-gnu -start-before=aarch64-jump-tables --filetype=obj -o %t.o %s --- | target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll b/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll --- a/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll +++ b/llvm/test/CodeGen/AArch64/seh_funclet_x1.ll @@ -3,8 +3,8 @@ ; Windows runtime passes the establisher frame as the second argument to the ; termination handler. Check that we copy it into fp. -; CHECK: ?dtor$3@?0?main@4HA": -; CHECK: .seh_proc "?dtor$3@?0?main@4HA" +; CHECK: ?dtor$6@?0?main@4HA": +; CHECK: .seh_proc "?dtor$6@?0?main@4HA" ; CHECK: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill ; CHECK-NEXT: .seh_save_fplr_x 16 ; CHECK-NEXT: .seh_endprologue diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll --- a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll +++ b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll @@ -44,7 +44,7 @@ ; CHECK: [[CATCHRETDEST:.LBB0_[0-9]+]]: // %catchret.dest ; Check the catch funclet. -; CHECK-LABEL: "?catch$2@?0??func@@YAHXZ@4HA": +; CHECK-LABEL: "?catch$4@?0??func@@YAHXZ@4HA": ; Check that the stack space is allocated only for the callee saved registers. ; CHECK: stp x19, x20, [sp, #-48]! @@ -66,7 +66,7 @@ ; CHECK: stur w8, [x29, [[X_OFFSET]]] ; Check that the funclet branches back to the catchret destination -; CHECK: adrp x0, .LBB0_3 +; CHECK: adrp x0, .LBB0_2 ; CHECK-NEXT: add x0, x0, [[CATCHRETDEST]] @@ -91,7 +91,7 @@ ; UNWIND-NEXT: ; str x21, [sp, #16] ; UNWIND-NEXT: ; stp x19, x20, [sp, #-64]! ; UNWIND-NEXT: ; end -; UNWIND: Function: ?catch$2@?0??func@@YAHXZ@4HA +; UNWIND: Function: ?catch$4@?0??func@@YAHXZ@4HA ; UNWIND: Prologue [ ; UNWIND-NEXT: ; stp x29, x30, [sp, #32] ; UNWIND-NEXT: ; str x28, [sp, #24] diff --git a/llvm/test/DebugInfo/AArch64/fallthrough-branch.ll b/llvm/test/DebugInfo/AArch64/fallthrough-branch.ll --- a/llvm/test/DebugInfo/AArch64/fallthrough-branch.ll +++ b/llvm/test/DebugInfo/AArch64/fallthrough-branch.ll @@ -10,16 +10,17 @@ %3 = bitcast i1* %2 to i8* call void @llvm.memset.p0i8.i64(i8* align 8 %3, i8 0, i64 1, i1 false) store i1 %0, i1* %2, align 8, !dbg !37 -; CHECK: B %bb.1, debug-location !{{[0-9]+}} +; CHECK: B %[[BB4:bb\.[0-9]+]], debug-location !{{[0-9]+}} br i1 %0, label %4, label %5, !dbg !38 4: ; preds = %1 +; CHECK: [[BB4]] ; Check that at -O0 the branches and their debug locations are not eliminated. -; CHECK: B %bb.3, debug-location !{{[0-9]+}} +; CHECK: B %[[BB6:bb\.[0-9]+]], debug-location !{{[0-9]+}} br label %6, !dbg !39 5: ; preds = %1 -; CHECK: B %bb.3, debug-location !{{[0-9]+}} +; CHECK: B %[[BB6]], debug-location !{{[0-9]+}} br label %6, !dbg !40 6: ; preds = %4, %5