Index: lib/Target/AArch64/AArch64RedundantCopyElimination.cpp =================================================================== --- lib/Target/AArch64/AArch64RedundantCopyElimination.cpp +++ lib/Target/AArch64/AArch64RedundantCopyElimination.cpp @@ -12,12 +12,23 @@ // CBZW %W0, // BB#2: // %W0 = COPY %WZR +// +// Also, if the cmp defines a register we can remove a zero copy in some cases. +// BB#0: +// subs w0, w1, w2 +// str w0, [x1] +// b.ne .LBB0_2 +// BB#1: +// mov w0, wzr ; <-- redundant +// str w0, [x2] +// .LBB0_2 +// // Similarly, this pass also handles non-zero copies. // BB#0: // cmp x0, #1 // b.eq .LBB0_1 // .LBB0_1: -// orr x0, xzr, #0x1 +// orr x0, xzr, #0x1 ; <-- redundant // // This pass should be run after register allocation. // @@ -60,9 +71,9 @@ RegImm(MCPhysReg Reg, int32_t Imm) : Reg(Reg), Imm(Imm) {} }; - Optional knownRegValInBlock(MachineInstr &CondBr, - MachineBasicBlock *MBB, - MachineBasicBlock::iterator &FirstUse); + bool knownRegValInBlock(MachineInstr &CondBr, MachineBasicBlock *MBB, + SmallVectorImpl &KnownRegs, + MachineBasicBlock::iterator &FirstUse); bool optimizeCopy(MachineBasicBlock *MBB); bool runOnMachineFunction(MachineFunction &MF) override; MachineFunctionProperties getRequiredProperties() const override { @@ -106,10 +117,9 @@ /// is the target to which a conditional branch \p CondBr jumps and whose /// equality comparison is against a constant. If so, return a known physical /// register and constant value pair. Otherwise, return None. -Optional -AArch64RedundantCopyElimination::knownRegValInBlock( +bool AArch64RedundantCopyElimination::knownRegValInBlock( MachineInstr &CondBr, MachineBasicBlock *MBB, - MachineBasicBlock::iterator &FirstUse) { + SmallVectorImpl &KnownRegs, MachineBasicBlock::iterator &FirstUse) { unsigned Opc = CondBr.getOpcode(); // Check if the current basic block is the target block to which the @@ -119,29 +129,30 @@ ((Opc == AArch64::CBNZW || Opc == AArch64::CBNZX) && MBB != CondBr.getOperand(1).getMBB())) { FirstUse = CondBr; - return RegImm(CondBr.getOperand(0).getReg(), 0); + KnownRegs.push_back(RegImm(CondBr.getOperand(0).getReg(), 0)); + return true; } // Otherwise, must be a conditional branch. if (Opc != AArch64::Bcc) - return None; + return false; // Must be an equality check (i.e., == or !=). AArch64CC::CondCode CC = (AArch64CC::CondCode)CondBr.getOperand(0).getImm(); if (CC != AArch64CC::EQ && CC != AArch64CC::NE) - return None; + return false; MachineBasicBlock *BrTarget = CondBr.getOperand(1).getMBB(); if ((CC == AArch64CC::EQ && BrTarget != MBB) || (CC == AArch64CC::NE && BrTarget == MBB)) - return None; + return false; // Stop if we get to the beginning of PredMBB. MachineBasicBlock *PredMBB = *MBB->pred_begin(); assert(PredMBB == CondBr.getParent() && "Conditional branch not in predecessor block!"); if (CondBr == PredMBB->begin()) - return None; + return false; // Registers clobbered in PredMBB between CondBr instruction and current // instruction being checked in loop. @@ -151,9 +162,6 @@ MachineBasicBlock::reverse_iterator RIt = CondBr.getReverseIterator(); for (MachineInstr &PredI : make_range(std::next(RIt), PredMBB->rend())) { - // Track clobbered registers. - trackRegDefs(PredI, ClobberedRegs, TRI); - bool IsCMN = false; switch (PredI.getOpcode()) { default: @@ -166,33 +174,82 @@ // CMP is an alias for SUBS with a dead destination register. case AArch64::SUBSWri: case AArch64::SUBSXri: { + MCPhysReg DstReg = PredI.getOperand(0).getReg(); MCPhysReg SrcReg = PredI.getOperand(1).getReg(); // Must not be a symbolic immediate. if (!PredI.getOperand(2).isImm()) - return None; + return false; + bool Res = false; // The src register must not be modified between the cmp and conditional // branch. This includes a self-clobbering compare. - if (ClobberedRegs[SrcReg]) - return None; - - // We've found the Cmp that sets NZCV. - int32_t KnownImm = PredI.getOperand(2).getImm(); - int32_t Shift = PredI.getOperand(3).getImm(); - KnownImm <<= Shift; - if (IsCMN) - KnownImm = -KnownImm; + if (!ClobberedRegs[SrcReg] && SrcReg != DstReg) { + // We've found the Cmp that sets NZCV. + int32_t KnownImm = PredI.getOperand(2).getImm(); + int32_t Shift = PredI.getOperand(3).getImm(); + KnownImm <<= Shift; + if (IsCMN) + KnownImm = -KnownImm; + FirstUse = PredI; + KnownRegs.push_back(RegImm(SrcReg, KnownImm)); + Res = true; + } + + // If this instructions defines something other than WZR/XZR, we know it's + // value is zero. The dst register must not be modified between the cmp + // and conditional branch. + if (DstReg != AArch64::WZR && DstReg != AArch64::XZR && + !ClobberedRegs[DstReg]) { + FirstUse = PredI; + KnownRegs.push_back(RegImm(DstReg, 0)); + return true; + } + return Res; + } + + case AArch64::ANDSWri: + case AArch64::ANDSXri: + case AArch64::ADDSWrr: + case AArch64::ADDSXrr: + case AArch64::ANDSWrr: + case AArch64::ANDSXrr: + case AArch64::BICSWrr: + case AArch64::BICSXrr: + case AArch64::SUBSWrr: + case AArch64::SUBSXrr: + case AArch64::ADDSWrs: + case AArch64::ADDSXrs: + case AArch64::ANDSWrs: + case AArch64::ANDSXrs: + case AArch64::BICSWrs: + case AArch64::BICSXrs: + case AArch64::SUBSWrs: + case AArch64::SUBSXrs: { + MCPhysReg DstReg = PredI.getOperand(0).getReg(); + if (DstReg == AArch64::WZR && DstReg == AArch64::XZR) + return false; + + // The dst register must not be modified between the cmp and conditional + // branch. + if (ClobberedRegs[DstReg]) + return false; + + // We've found the instruction that sets NZCV and DstReg == 0. FirstUse = PredI; - return RegImm(SrcReg, KnownImm); + KnownRegs.push_back(RegImm(DstReg, 0)); + return true; } } // Bail if we see an instruction that defines NZCV that we don't handle. if (PredI.definesRegister(AArch64::NZCV)) - return None; + return false; + + // Track clobbered registers. + trackRegDefs(PredI, ClobberedRegs, TRI); } - return None; + return false; } bool AArch64RedundantCopyElimination::optimizeCopy(MachineBasicBlock *MBB) { @@ -226,12 +283,9 @@ do { --Itr; - Optional KnownRegImm = knownRegValInBlock(*Itr, MBB, FirstUse); - if (KnownRegImm == None) + if (!knownRegValInBlock(*Itr, MBB, KnownRegs, FirstUse)) continue; - KnownRegs.push_back(*KnownRegImm); - // Reset the clobber list, which is used by knownRegValInBlock. ClobberedRegs.reset(); Index: test/CodeGen/AArch64/machine-zero-copy-remove.mir =================================================================== --- /dev/null +++ test/CodeGen/AArch64/machine-zero-copy-remove.mir @@ -0,0 +1,556 @@ +# RUN: llc -mtriple=aarch64--linux-gnu -run-pass=aarch64-copyelim %s -verify-machineinstrs -o - | FileCheck %s +--- +# CHECK-LABEL: name: test1 +# CHECK: ANDSWri %w0, 1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test1 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %x1, %x2 + + %w0 = ANDSWri %w0, 1, implicit-def %nzcv + STRWui killed %w0, killed %x1, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x2 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x2, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test2 +# CHECK: ANDSXri %x0, 1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test2 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2 + + %x0 = ANDSXri %x0, 1, implicit-def %nzcv + STRXui killed %x0, killed %x1, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x2 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x2, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test3 +# CHECK: ADDSWri %w0, 1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test3 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %x1, %x2 + + %w0 = ADDSWri %w0, 1, 0, implicit-def %nzcv + STRWui killed %w0, killed %x1, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x2 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x2, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test4 +# CHECK: ADDSXri %x0, 1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test4 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2 + + %x0 = ADDSXri %x0, 1, 0, implicit-def %nzcv + STRXui killed %x0, killed %x1, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x2 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x2, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test5 +# CHECK: SUBSWri %w0, 1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test5 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %x1, %x2 + + %w0 = SUBSWri %w0, 1, 0, implicit-def %nzcv + STRWui killed %w0, killed %x1, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x2 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x2, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test6 +# CHECK: SUBSXri %x0, 1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test6 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2 + + %x0 = SUBSXri %x0, 1, 0, implicit-def %nzcv + STRXui killed %x0, killed %x1, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x2 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x2, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test7 +# CHECK: ADDSWrr %w0, %w1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test7 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = ADDSWrr %w0, %w1, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test8 +# CHECK: ADDSXrr %x0, %x1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test8 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = ADDSXrr %x0, %x1, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test9 +# CHECK: ANDSWrr %w0, %w1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test9 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = ANDSWrr %w0, %w1, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test10 +# CHECK: ANDSXrr %x0, %x1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test10 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = ANDSXrr %x0, %x1, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test11 +# CHECK: BICSWrr %w0, %w1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test11 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = BICSWrr %w0, %w1, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test12 +# CHECK: BICSXrr %x0, %x1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test12 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = BICSXrr %x0, %x1, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test13 +# CHECK: SUBSWrr %w0, %w1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test13 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = SUBSWrr %w0, %w1, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test14 +# CHECK: SUBSXrr %x0, %x1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test14 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = SUBSXrr %x0, %x1, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test15 +# CHECK: ADDSWrs %w0, %w1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test15 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = ADDSWrs %w0, %w1, 0, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test16 +# CHECK: ADDSXrs %x0, %x1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test16 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = ADDSXrs %x0, %x1, 0, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test17 +# CHECK: ANDSWrs %w0, %w1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test17 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = ANDSWrs %w0, %w1, 0, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test18 +# CHECK: ANDSXrs %x0, %x1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test18 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = ANDSXrs %x0, %x1, 0, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test19 +# CHECK: BICSWrs %w0, %w1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test19 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = BICSWrs %w0, %w1, 0, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test20 +# CHECK: BICSXrs %x0, %x1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test20 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = BICSXrs %x0, %x1, 0, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test21 +# CHECK: SUBSWrs %w0, %w1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %wzr +name: test21 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %w1, %x2, %x3 + + %w0 = SUBSWrs %w0, %w1, 0, implicit-def %nzcv + STRWui killed %w0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# CHECK-LABEL: name: test22 +# CHECK: SUBSXrs %x0, %x1, 0, implicit-def %nzcv +# CHECK: bb.1: +# CHECK-NOT: COPY %xzr +name: test22 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %x0, %x1, %x2, %x3 + + %x0 = SUBSXrs %x0, %x1, 0, implicit-def %nzcv + STRXui killed %x0, killed %x2, 0 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x3 + + %x0 = COPY %xzr + STRXui killed %x0, killed %x3, 0 + + bb.2: + RET_ReallyLR +... +# Negative test - MOVi32imm clobbers %w0 +# CHECK-LABEL: name: test23 +# CHECK: ANDSWri %w0, 1, implicit-def %nzcv +# CHECK: bb.1: +# CHECK: %w0 = COPY %wzr +name: test23 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %w0, %x1, %x2 + + %w0 = ANDSWri %w0, 1, implicit-def %nzcv + STRWui killed %w0, %x1, 0 + %w0 = MOVi32imm -1 + STRWui killed %w0, killed %x1, 1 + Bcc 1, %bb.2, implicit killed %nzcv + B %bb.1 + + bb.1: + liveins: %x2 + + %w0 = COPY %wzr + STRWui killed %w0, killed %x2, 0 + + bb.2: + RET_ReallyLR