diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -2748,15 +2748,15 @@ SDValue Add1 = ShiftAmt->getOperand(1); uint64_t Add0Imm; uint64_t Add1Imm; - // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X - // to avoid the ADD/SUB. - if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) + if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) { + // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X + // to avoid the ADD/SUB. NewShiftAmt = Add0; - // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to - // generate a NEG instead of a SUB of a constant. - else if (ShiftAmt->getOpcode() == ISD::SUB && - isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 && - (Add0Imm % Size == 0)) { + } else if (ShiftAmt->getOpcode() == ISD::SUB && + isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 && + (Add0Imm % Size == 0)) { + // If we are shifting by N-X where N == 0 mod Size, then just shift by -X + // to generate a NEG instead of a SUB from a constant. unsigned NegOpc; unsigned ZeroReg; EVT SubVT = ShiftAmt->getValueType(0); @@ -2773,6 +2773,26 @@ MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1); NewShiftAmt = SDValue(Neg, 0); + } else if (ShiftAmt->getOpcode() == ISD::SUB && + isIntImmediate(Add0, Add0Imm) && (Add0Imm % Size == Size - 1)) { + // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X + // to generate a NOT instead of a SUB from a constant. + unsigned NotOpc; + unsigned ZeroReg; + EVT SubVT = ShiftAmt->getValueType(0); + if (SubVT == MVT::i32) { + NotOpc = AArch64::EONWrr; + ZeroReg = AArch64::WZR; + } else { + assert(SubVT == MVT::i64); + NotOpc = AArch64::EONXrr; + ZeroReg = AArch64::XZR; + } + SDValue Zero = + CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT); + MachineSDNode *Not = + CurDAG->getMachineNode(NotOpc, DL, SubVT, Add1, Zero); + NewShiftAmt = SDValue(Not, 0); } else return false; } else { diff --git a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll --- a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll +++ b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll @@ -318,8 +318,7 @@ define i32 @reg32_shl_by_complemented(i32 %val, i32 %shamt) nounwind { ; CHECK-LABEL: reg32_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w1 +; CHECK-NEXT: eon w8, w1, wzr ; CHECK-NEXT: lsl w0, w0, w8 ; CHECK-NEXT: ret %negshamt = sub i32 31, %shamt @@ -330,8 +329,7 @@ ; CHECK-LABEL: load32_shl_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr w8, [x0] -; CHECK-NEXT: mov w9, #31 -; CHECK-NEXT: sub w9, w9, w1 +; CHECK-NEXT: eon w9, w1, wzr ; CHECK-NEXT: lsl w0, w8, w9 ; CHECK-NEXT: ret %val = load i32, i32* %valptr @@ -342,8 +340,7 @@ define void @store32_shl_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { ; CHECK-LABEL: store32_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w2 +; CHECK-NEXT: eon w8, w2, wzr ; CHECK-NEXT: lsl w8, w0, w8 ; CHECK-NEXT: str w8, [x1] ; CHECK-NEXT: ret @@ -356,8 +353,7 @@ ; CHECK-LABEL: modify32_shl_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr w8, [x0] -; CHECK-NEXT: mov w9, #31 -; CHECK-NEXT: sub w9, w9, w1 +; CHECK-NEXT: eon w9, w1, wzr ; CHECK-NEXT: lsl w8, w8, w9 ; CHECK-NEXT: str w8, [x0] ; CHECK-NEXT: ret @@ -371,8 +367,7 @@ define i64 @reg64_shl_by_complemented(i64 %val, i64 %shamt) nounwind { ; CHECK-LABEL: reg64_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x1 +; CHECK-NEXT: eon x8, x1, xzr ; CHECK-NEXT: lsl x0, x0, x8 ; CHECK-NEXT: ret %negshamt = sub i64 63, %shamt @@ -383,8 +378,7 @@ ; CHECK-LABEL: load64_shl_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: mov w9, #63 -; CHECK-NEXT: sub x9, x9, x1 +; CHECK-NEXT: eon x9, x1, xzr ; CHECK-NEXT: lsl x0, x8, x9 ; CHECK-NEXT: ret %val = load i64, i64* %valptr @@ -395,8 +389,7 @@ define void @store64_shl_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { ; CHECK-LABEL: store64_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x2 +; CHECK-NEXT: eon x8, x2, xzr ; CHECK-NEXT: lsl x8, x0, x8 ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: ret @@ -409,8 +402,7 @@ ; CHECK-LABEL: modify64_shl_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: mov w9, #63 -; CHECK-NEXT: sub x9, x9, x1 +; CHECK-NEXT: eon x9, x1, xzr ; CHECK-NEXT: lsl x8, x8, x9 ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: ret @@ -427,8 +419,7 @@ define i32 @reg32_lshr_by_complemented(i32 %val, i32 %shamt) nounwind { ; CHECK-LABEL: reg32_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w1 +; CHECK-NEXT: eon w8, w1, wzr ; CHECK-NEXT: lsr w0, w0, w8 ; CHECK-NEXT: ret %negshamt = sub i32 31, %shamt @@ -439,8 +430,7 @@ ; CHECK-LABEL: load32_lshr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr w8, [x0] -; CHECK-NEXT: mov w9, #31 -; CHECK-NEXT: sub w9, w9, w1 +; CHECK-NEXT: eon w9, w1, wzr ; CHECK-NEXT: lsr w0, w8, w9 ; CHECK-NEXT: ret %val = load i32, i32* %valptr @@ -451,8 +441,7 @@ define void @store32_lshr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { ; CHECK-LABEL: store32_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w2 +; CHECK-NEXT: eon w8, w2, wzr ; CHECK-NEXT: lsr w8, w0, w8 ; CHECK-NEXT: str w8, [x1] ; CHECK-NEXT: ret @@ -465,8 +454,7 @@ ; CHECK-LABEL: modify32_lshr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr w8, [x0] -; CHECK-NEXT: mov w9, #31 -; CHECK-NEXT: sub w9, w9, w1 +; CHECK-NEXT: eon w9, w1, wzr ; CHECK-NEXT: lsr w8, w8, w9 ; CHECK-NEXT: str w8, [x0] ; CHECK-NEXT: ret @@ -480,8 +468,7 @@ define i64 @reg64_lshr_by_complemented(i64 %val, i64 %shamt) nounwind { ; CHECK-LABEL: reg64_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x1 +; CHECK-NEXT: eon x8, x1, xzr ; CHECK-NEXT: lsr x0, x0, x8 ; CHECK-NEXT: ret %negshamt = sub i64 63, %shamt @@ -492,8 +479,7 @@ ; CHECK-LABEL: load64_lshr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: mov w9, #63 -; CHECK-NEXT: sub x9, x9, x1 +; CHECK-NEXT: eon x9, x1, xzr ; CHECK-NEXT: lsr x0, x8, x9 ; CHECK-NEXT: ret %val = load i64, i64* %valptr @@ -504,8 +490,7 @@ define void @store64_lshr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { ; CHECK-LABEL: store64_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x2 +; CHECK-NEXT: eon x8, x2, xzr ; CHECK-NEXT: lsr x8, x0, x8 ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: ret @@ -518,8 +503,7 @@ ; CHECK-LABEL: modify64_lshr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: mov w9, #63 -; CHECK-NEXT: sub x9, x9, x1 +; CHECK-NEXT: eon x9, x1, xzr ; CHECK-NEXT: lsr x8, x8, x9 ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: ret @@ -536,8 +520,7 @@ define i32 @reg32_ashr_by_complemented(i32 %val, i32 %shamt) nounwind { ; CHECK-LABEL: reg32_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w1 +; CHECK-NEXT: eon w8, w1, wzr ; CHECK-NEXT: asr w0, w0, w8 ; CHECK-NEXT: ret %negshamt = sub i32 31, %shamt @@ -548,8 +531,7 @@ ; CHECK-LABEL: load32_ashr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr w8, [x0] -; CHECK-NEXT: mov w9, #31 -; CHECK-NEXT: sub w9, w9, w1 +; CHECK-NEXT: eon w9, w1, wzr ; CHECK-NEXT: asr w0, w8, w9 ; CHECK-NEXT: ret %val = load i32, i32* %valptr @@ -560,8 +542,7 @@ define void @store32_ashr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { ; CHECK-LABEL: store32_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w2 +; CHECK-NEXT: eon w8, w2, wzr ; CHECK-NEXT: asr w8, w0, w8 ; CHECK-NEXT: str w8, [x1] ; CHECK-NEXT: ret @@ -574,8 +555,7 @@ ; CHECK-LABEL: modify32_ashr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr w8, [x0] -; CHECK-NEXT: mov w9, #31 -; CHECK-NEXT: sub w9, w9, w1 +; CHECK-NEXT: eon w9, w1, wzr ; CHECK-NEXT: asr w8, w8, w9 ; CHECK-NEXT: str w8, [x0] ; CHECK-NEXT: ret @@ -589,8 +569,7 @@ define i64 @reg64_ashr_by_complemented(i64 %val, i64 %shamt) nounwind { ; CHECK-LABEL: reg64_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x1 +; CHECK-NEXT: eon x8, x1, xzr ; CHECK-NEXT: asr x0, x0, x8 ; CHECK-NEXT: ret %negshamt = sub i64 63, %shamt @@ -601,8 +580,7 @@ ; CHECK-LABEL: load64_ashr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: mov w9, #63 -; CHECK-NEXT: sub x9, x9, x1 +; CHECK-NEXT: eon x9, x1, xzr ; CHECK-NEXT: asr x0, x8, x9 ; CHECK-NEXT: ret %val = load i64, i64* %valptr @@ -613,8 +591,7 @@ define void @store64_ashr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { ; CHECK-LABEL: store64_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x2 +; CHECK-NEXT: eon x8, x2, xzr ; CHECK-NEXT: asr x8, x0, x8 ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: ret @@ -627,8 +604,7 @@ ; CHECK-LABEL: modify64_ashr_by_complemented: ; CHECK: // %bb.0: ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: mov w9, #63 -; CHECK-NEXT: sub x9, x9, x1 +; CHECK-NEXT: eon x9, x1, xzr ; CHECK-NEXT: asr x8, x8, x9 ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: ret