Index: llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp =================================================================== --- llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -687,9 +687,30 @@ MachineInstrBuilder MIB; DebugLoc DL = I->getDebugLoc(); MachineBasicBlock *MBB = I->getParent(); + MachineOperand RegOp0 = getLdStRegOp(*RtMI); + MachineOperand RegOp1 = getLdStRegOp(*Rt2MI); + // Kill flags may become invalid when moving stores for pairing. + if (RegOp0.isUse()) { + if (!MergeForward) { + // Clear kill flags on store if moving upwards. Example: + // STRWui %w0, ... + // USE %w1 + // STRWui kill %w1 ; need to clear kill flag when moving STRWui upwards + RegOp0.setIsKill(false); + RegOp1.setIsKill(false); + } else { + // Clear kill flags of the first stores register. Example: + // STRWui %w1, ... + // USE kill %w1 ; need to clear kill flag when moving STRWui downwards + // STRW %w0 + unsigned Reg = getLdStRegOp(*I).getReg(); + for (MachineInstr &MI : make_range(std::next(I), Paired)) + MI.clearRegisterKills(Reg, TRI); + } + } MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingPairOpcode(Opc))) - .add(getLdStRegOp(*RtMI)) - .add(getLdStRegOp(*Rt2MI)) + .add(RegOp0) + .add(RegOp1) .add(BaseRegOp) .addImm(OffsetImm) .setMemRefs(I->mergeMemRefsWith(*Paired)); Index: llvm/trunk/test/CodeGen/AArch64/ldst-opt-dbg-limit.mir =================================================================== --- llvm/trunk/test/CodeGen/AArch64/ldst-opt-dbg-limit.mir +++ llvm/trunk/test/CodeGen/AArch64/ldst-opt-dbg-limit.mir @@ -1,133 +0,0 @@ -# RUN: llc -run-pass=aarch64-ldst-opt %s -o - 2>&1 | FileCheck %s ---- | - target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" - target triple = "aarch64--linux-gnu" - - ; Function Attrs: nounwind - define i16 @promote-load-from-store(i32* %dst, i32 %x) #0 { - store i32 %x, i32* %dst - %dst16 = bitcast i32* %dst to i16* - %dst1 = getelementptr inbounds i16, i16* %dst16, i32 1 - %x16 = load i16, i16* %dst1 - ret i16 %x16 - } - - ; Function Attrs: nounwind - define void @store-pair(i32* %dst, i32 %x, i32 %y) #0 { - %dst01 = bitcast i32* %dst to i32* - %dst1 = getelementptr inbounds i32, i32* %dst, i32 1 - store i32 %x, i32* %dst01 - store i32 %x, i32* %dst1 - ret void - } - - attributes #0 = { nounwind } - -... ---- -name: promote-load-from-store -alignment: 2 -exposesReturnsTwice: false -tracksRegLiveness: true -liveins: - - { reg: '%x0' } - - { reg: '%w1' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - maxCallFrameSize: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false -body: | - bb.0 (%ir-block.0): - liveins: %w1, %x0, %lr - - STRWui killed %w1, %x0, 0 :: (store 4 into %ir.dst) - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - %w0 = LDRHHui killed %x0, 1 :: (load 2 from %ir.dst1) - RET %lr, implicit %w0 - -... -# CHECK-LABEL: name: promote-load-from-store -# CHECK: STRWui %w1 -# CHECK: UBFMWri %w1 ---- -name: store-pair -alignment: 2 -exposesReturnsTwice: false -tracksRegLiveness: true -liveins: - - { reg: '%x0' } - - { reg: '%w1' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - maxCallFrameSize: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false -body: | - bb.0 (%ir-block.0): - liveins: %w1, %x0, %lr - - STRWui %w1, %x0, 0 :: (store 4 into %ir.dst01) - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - CFI_INSTRUCTION 0 - STRWui killed %w1, killed %x0, 1 :: (store 4 into %ir.dst1) - RET %lr - -... -# CHECK-LABEL: name: store-pair -# CHECK: STPWi Index: llvm/trunk/test/CodeGen/AArch64/ldst-opt.mir =================================================================== --- llvm/trunk/test/CodeGen/AArch64/ldst-opt.mir +++ llvm/trunk/test/CodeGen/AArch64/ldst-opt.mir @@ -0,0 +1,132 @@ +# RUN: llc -mtriple=aarch64--linux-gnu -run-pass=aarch64-ldst-opt %s -verify-machineinstrs -o - 2>&1 | FileCheck %s +--- | + define void @promote-load-from-store() { ret void } + define void @store-pair() { ret void } + define void @store-pair-clearkill0() { ret void } + define void @store-pair-clearkill1() { ret void } +... +--- +name: promote-load-from-store +tracksRegLiveness: true +body: | + bb.0: + liveins: %w1, %x0, %lr + + STRWui killed %w1, %x0, 0 :: (store 4) + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + %w0 = LDRHHui killed %x0, 1 :: (load 2) + RET %lr, implicit %w0 + +... +# Don't count transient instructions towards search limits. +# CHECK-LABEL: name: promote-load-from-store +# CHECK: STRWui %w1 +# CHECK: UBFMWri %w1 +--- +name: store-pair +tracksRegLiveness: true +body: | + bb.0: + liveins: %w1, %x0, %lr + + STRWui %w1, %x0, 0 :: (store 4) + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + CFI_INSTRUCTION 0 + STRWui killed %w1, killed %x0, 1 :: (store 4) + RET %lr + +... +# CHECK-LABEL: name: store-pair +# CHECK: STPWi +--- +name: store-pair-clearkill0 +tracksRegLiveness: true +body: | + bb.0: + liveins: %w1, %x0, %lr + + STRWui %w1, %x0, 0 :: (store 4) + %w2 = COPY %w1 + %x3 = COPY %x0 + STRWui killed %w1, killed %x0, 1 :: (store 4) + RET %lr +... +# When merging a lower store with an upper one, we must clear kill flags on +# the lower store. +# CHECK-LABEL: store-pair-clearkill0 +# CHECK-NOT: STPWi %w1, killed %w1, %x0, 0 :: (store 4) +# CHECK: STPWi %w1, %w1, %x0, 0 :: (store 4) +# CHECK: %w2 = COPY %w1 +# CHECK: RET %lr +--- +name: store-pair-clearkill1 +tracksRegLiveness: true +body: | + bb.0: + liveins: %x0, %lr + + %w1 = MOVi32imm 13 + %w2 = MOVi32imm 7 + STRWui %w1, %x0, 1 :: (store 4) + %w2 = COPY killed %w1 + STRWui killed %w2, %x0, 0 :: (store 4) + + %w1 = MOVi32imm 42 + %w2 = MOVi32imm 7 + STRWui %w1, %x0, 0 :: (store 4) + %w2 = COPY killed %w1 + STRWui killed %w2, killed %x0, 1 :: (store 4) + + RET %lr +... +# When merging an upper store with a lower one, kill flags along the way need +# to be removed; In this case the kill flag on %w1. +# CHECK-LABEL: store-pair-clearkill1 +# CHECK: %w1 = MOVi32imm +# CHECK: %w2 = MOVi32imm +# CHECK-NOT: %w2 = COPY killed %w1 +# CHECK: %w2 = COPY %w1 +# CHECK: STPWi killed %w2, %w1, %x0, 0 + +# CHECK: %w1 = MOVi32imm +# CHECK: %w2 = MOVi32imm +# CHECK-NOT: %w2 = COPY killed %w1 +# CHECK: %w2 = COPY %w1 +# CHECK: STPWi %w1, killed %w2, killed %x0, 0