diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -431,6 +431,7 @@ void addPostRegAlloc() override; void addPreSched2() override; void addPreEmitPass() override; + void addPreEmitPass2() override; std::unique_ptr getCSEConfig() const override; }; @@ -692,8 +693,11 @@ if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH && TM->getTargetTriple().isOSBinFormatMachO()) addPass(createAArch64CollectLOHPass()); +} - // SVE bundles move prefixes with destructive operations. +void AArch64PassConfig::addPreEmitPass2() { + // SVE bundles move prefixes with destructive operations. BLR_RVMARKER pseudo + // instructions are lowered to bundles as well. addPass(createUnpackMachineBundles(nullptr)); } diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -64,10 +64,10 @@ ; CHECK-NEXT: Implement the 'patchable-function' attribute ; CHECK-NEXT: AArch64 Branch Targets ; CHECK-NEXT: Branch relaxation pass -; CHECK-NEXT: Unpack machine instruction bundles ; CHECK-NEXT: Contiguously Lay Out Funclets ; CHECK-NEXT: StackMap Liveness Analysis ; CHECK-NEXT: Live DEBUG_VALUE analysis +; CHECK-NEXT: Unpack machine instruction bundles ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter ; CHECK-NEXT: AArch64 Assembly Printer diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -195,12 +195,12 @@ ; CHECK-NEXT: AArch64 Branch Targets ; CHECK-NEXT: Branch relaxation pass ; CHECK-NEXT: AArch64 Compress Jump Tables -; CHECK-NEXT: Unpack machine instruction bundles ; CHECK-NEXT: Contiguously Lay Out Funclets ; CHECK-NEXT: StackMap Liveness Analysis ; CHECK-NEXT: Live DEBUG_VALUE analysis ; CHECK-NEXT: Machine Outliner ; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Unpack machine instruction bundles ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Machine Optimization Remark Emitter ; CHECK-NEXT: AArch64 Assembly Printer diff --git a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll --- a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll +++ b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll @@ -34,6 +34,10 @@ ; HOTNESS-NEXT: Executing Pass 'Function Pass Manager' ; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' ; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Executing Pass 'Unpack machine instruction bundles' +; HOTNESS-NEXT: Freeing Pass 'Unpack machine instruction bundles' +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' ; HOTNESS-NEXT: Executing Pass 'Lazy Machine Block Frequency Analysis' ; HOTNESS-NEXT: Executing Pass 'Machine Optimization Remark Emitter' ; HOTNESS-NEXT: Building MachineBlockFrequencyInfo on the fly @@ -51,6 +55,10 @@ ; NO_HOTNESS-NEXT: Executing Pass 'Function Pass Manager' ; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' ; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Executing Pass 'Unpack machine instruction bundles' +; NO_HOTNESS-NEXT: Freeing Pass 'Unpack machine instruction bundles' +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' ; NO_HOTNESS-NEXT: Executing Pass 'Lazy Machine Block Frequency Analysis' ; NO_HOTNESS-NEXT: Executing Pass 'Machine Optimization Remark Emitter' ; NO_HOTNESS-NEXT: Executing Pass 'AArch64 Assembly Printer' diff --git a/llvm/test/CodeGen/AArch64/rvmarker-pseudo-expansion-and-outlining.mir b/llvm/test/CodeGen/AArch64/rvmarker-pseudo-expansion-and-outlining.mir --- a/llvm/test/CodeGen/AArch64/rvmarker-pseudo-expansion-and-outlining.mir +++ b/llvm/test/CodeGen/AArch64/rvmarker-pseudo-expansion-and-outlining.mir @@ -6,16 +6,18 @@ # CHECK-LABEL: _fn1: # CHECK: bb.0: # CHECK: bl _cb1 +# CHECK-NEXT: mov x29, x29 # CHECK-NEXT: bl _OUTLINED_FUNCTION_0 # # CHECK-LABEL: _fn2: # CHECK: bb.0: # CHECK: bl _cb2 +# CHECK-NEXT: mov x29, x29 # CHECK-NEXT: bl _OUTLINED_FUNCTION_0 # # CHECK-LABEL: _OUTLINED_FUNCTION_0: # CHECK: bb.0: -# CHECK-NEXT: mov x29, x29 +# CHECK-NEXT: orr w12, wzr, #0x3 # CHECK-NEXT: orr w12, wzr, #0x3 # CHECK-NEXT: orr w12, wzr, #0x3 # CHECK-NEXT: orr w12, wzr, #0x3