diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -2827,15 +2827,15 @@ SDValue Add1 = ShiftAmt->getOperand(1); uint64_t Add0Imm; uint64_t Add1Imm; - // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X - // to avoid the ADD/SUB. - if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) + if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) { + // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X + // to avoid the ADD/SUB. NewShiftAmt = Add0; - // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to - // generate a NEG instead of a SUB of a constant. - else if (ShiftAmt->getOpcode() == ISD::SUB && - isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 && - (Add0Imm % Size == 0)) { + } else if (ShiftAmt->getOpcode() == ISD::SUB && + isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 && + (Add0Imm % Size == 0)) { + // If we are shifting by N-X where N == 0 mod Size, then just shift by -X + // to generate a NEG instead of a SUB from a constant. unsigned NegOpc; unsigned ZeroReg; EVT SubVT = ShiftAmt->getValueType(0); @@ -2852,6 +2852,26 @@ MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1); NewShiftAmt = SDValue(Neg, 0); + } else if (ShiftAmt->getOpcode() == ISD::SUB && + isIntImmediate(Add0, Add0Imm) && (Add0Imm % Size == Size - 1)) { + // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X + // to generate a NOT instead of a SUB from a constant. + unsigned NotOpc; + unsigned ZeroReg; + EVT SubVT = ShiftAmt->getValueType(0); + if (SubVT == MVT::i32) { + NotOpc = AArch64::ORNWrr; + ZeroReg = AArch64::WZR; + } else { + assert(SubVT == MVT::i64); + NotOpc = AArch64::ORNXrr; + ZeroReg = AArch64::XZR; + } + SDValue Zero = + CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT); + MachineSDNode *Not = + CurDAG->getMachineNode(NotOpc, DL, SubVT, Zero, Add1); + NewShiftAmt = SDValue(Not, 0); } else return false; } else { diff --git a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll --- a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll +++ b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll @@ -318,8 +318,7 @@ define i32 @reg32_shl_by_complemented(i32 %val, i32 %shamt) nounwind { ; CHECK-LABEL: reg32_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w1 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: lsl w0, w0, w8 ; CHECK-NEXT: ret %negshamt = sub i32 31, %shamt @@ -329,9 +328,8 @@ define i32 @load32_shl_by_complemented(i32* %valptr, i32 %shamt) nounwind { ; CHECK-LABEL: load32_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: ldr w9, [x0] -; CHECK-NEXT: sub w8, w8, w1 ; CHECK-NEXT: lsl w0, w9, w8 ; CHECK-NEXT: ret %val = load i32, i32* %valptr @@ -342,8 +340,7 @@ define void @store32_shl_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { ; CHECK-LABEL: store32_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w2 +; CHECK-NEXT: mvn w8, w2 ; CHECK-NEXT: lsl w8, w0, w8 ; CHECK-NEXT: str w8, [x1] ; CHECK-NEXT: ret @@ -355,9 +352,8 @@ define void @modify32_shl_by_complemented(i32* %valptr, i32 %shamt) nounwind { ; CHECK-LABEL: modify32_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: ldr w9, [x0] -; CHECK-NEXT: sub w8, w8, w1 ; CHECK-NEXT: lsl w8, w9, w8 ; CHECK-NEXT: str w8, [x0] ; CHECK-NEXT: ret @@ -371,8 +367,7 @@ define i64 @reg64_shl_by_complemented(i64 %val, i64 %shamt) nounwind { ; CHECK-LABEL: reg64_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x1 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: lsl x0, x0, x8 ; CHECK-NEXT: ret %negshamt = sub i64 63, %shamt @@ -382,9 +377,8 @@ define i64 @load64_shl_by_complemented(i64* %valptr, i64 %shamt) nounwind { ; CHECK-LABEL: load64_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: ldr x9, [x0] -; CHECK-NEXT: sub x8, x8, x1 ; CHECK-NEXT: lsl x0, x9, x8 ; CHECK-NEXT: ret %val = load i64, i64* %valptr @@ -395,8 +389,7 @@ define void @store64_shl_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { ; CHECK-LABEL: store64_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x2 +; CHECK-NEXT: mvn x8, x2 ; CHECK-NEXT: lsl x8, x0, x8 ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: ret @@ -408,9 +401,8 @@ define void @modify64_shl_by_complemented(i64* %valptr, i64 %shamt) nounwind { ; CHECK-LABEL: modify64_shl_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: ldr x9, [x0] -; CHECK-NEXT: sub x8, x8, x1 ; CHECK-NEXT: lsl x8, x9, x8 ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: ret @@ -427,8 +419,7 @@ define i32 @reg32_lshr_by_complemented(i32 %val, i32 %shamt) nounwind { ; CHECK-LABEL: reg32_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w1 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: lsr w0, w0, w8 ; CHECK-NEXT: ret %negshamt = sub i32 31, %shamt @@ -438,9 +429,8 @@ define i32 @load32_lshr_by_complemented(i32* %valptr, i32 %shamt) nounwind { ; CHECK-LABEL: load32_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: ldr w9, [x0] -; CHECK-NEXT: sub w8, w8, w1 ; CHECK-NEXT: lsr w0, w9, w8 ; CHECK-NEXT: ret %val = load i32, i32* %valptr @@ -451,8 +441,7 @@ define void @store32_lshr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { ; CHECK-LABEL: store32_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w2 +; CHECK-NEXT: mvn w8, w2 ; CHECK-NEXT: lsr w8, w0, w8 ; CHECK-NEXT: str w8, [x1] ; CHECK-NEXT: ret @@ -464,9 +453,8 @@ define void @modify32_lshr_by_complemented(i32* %valptr, i32 %shamt) nounwind { ; CHECK-LABEL: modify32_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: ldr w9, [x0] -; CHECK-NEXT: sub w8, w8, w1 ; CHECK-NEXT: lsr w8, w9, w8 ; CHECK-NEXT: str w8, [x0] ; CHECK-NEXT: ret @@ -480,8 +468,7 @@ define i64 @reg64_lshr_by_complemented(i64 %val, i64 %shamt) nounwind { ; CHECK-LABEL: reg64_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x1 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: lsr x0, x0, x8 ; CHECK-NEXT: ret %negshamt = sub i64 63, %shamt @@ -491,9 +478,8 @@ define i64 @load64_lshr_by_complemented(i64* %valptr, i64 %shamt) nounwind { ; CHECK-LABEL: load64_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: ldr x9, [x0] -; CHECK-NEXT: sub x8, x8, x1 ; CHECK-NEXT: lsr x0, x9, x8 ; CHECK-NEXT: ret %val = load i64, i64* %valptr @@ -504,8 +490,7 @@ define void @store64_lshr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { ; CHECK-LABEL: store64_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x2 +; CHECK-NEXT: mvn x8, x2 ; CHECK-NEXT: lsr x8, x0, x8 ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: ret @@ -517,9 +502,8 @@ define void @modify64_lshr_by_complemented(i64* %valptr, i64 %shamt) nounwind { ; CHECK-LABEL: modify64_lshr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: ldr x9, [x0] -; CHECK-NEXT: sub x8, x8, x1 ; CHECK-NEXT: lsr x8, x9, x8 ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: ret @@ -536,8 +520,7 @@ define i32 @reg32_ashr_by_complemented(i32 %val, i32 %shamt) nounwind { ; CHECK-LABEL: reg32_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w1 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: asr w0, w0, w8 ; CHECK-NEXT: ret %negshamt = sub i32 31, %shamt @@ -547,9 +530,8 @@ define i32 @load32_ashr_by_complemented(i32* %valptr, i32 %shamt) nounwind { ; CHECK-LABEL: load32_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: ldr w9, [x0] -; CHECK-NEXT: sub w8, w8, w1 ; CHECK-NEXT: asr w0, w9, w8 ; CHECK-NEXT: ret %val = load i32, i32* %valptr @@ -560,8 +542,7 @@ define void @store32_ashr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { ; CHECK-LABEL: store32_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 -; CHECK-NEXT: sub w8, w8, w2 +; CHECK-NEXT: mvn w8, w2 ; CHECK-NEXT: asr w8, w0, w8 ; CHECK-NEXT: str w8, [x1] ; CHECK-NEXT: ret @@ -573,9 +554,8 @@ define void @modify32_ashr_by_complemented(i32* %valptr, i32 %shamt) nounwind { ; CHECK-LABEL: modify32_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31 +; CHECK-NEXT: mvn w8, w1 ; CHECK-NEXT: ldr w9, [x0] -; CHECK-NEXT: sub w8, w8, w1 ; CHECK-NEXT: asr w8, w9, w8 ; CHECK-NEXT: str w8, [x0] ; CHECK-NEXT: ret @@ -589,8 +569,7 @@ define i64 @reg64_ashr_by_complemented(i64 %val, i64 %shamt) nounwind { ; CHECK-LABEL: reg64_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x1 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: asr x0, x0, x8 ; CHECK-NEXT: ret %negshamt = sub i64 63, %shamt @@ -600,9 +579,8 @@ define i64 @load64_ashr_by_complemented(i64* %valptr, i64 %shamt) nounwind { ; CHECK-LABEL: load64_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: ldr x9, [x0] -; CHECK-NEXT: sub x8, x8, x1 ; CHECK-NEXT: asr x0, x9, x8 ; CHECK-NEXT: ret %val = load i64, i64* %valptr @@ -613,8 +591,7 @@ define void @store64_ashr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { ; CHECK-LABEL: store64_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 -; CHECK-NEXT: sub x8, x8, x2 +; CHECK-NEXT: mvn x8, x2 ; CHECK-NEXT: asr x8, x0, x8 ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: ret @@ -626,9 +603,8 @@ define void @modify64_ashr_by_complemented(i64* %valptr, i64 %shamt) nounwind { ; CHECK-LABEL: modify64_ashr_by_complemented: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #63 +; CHECK-NEXT: mvn x8, x1 ; CHECK-NEXT: ldr x9, [x0] -; CHECK-NEXT: sub x8, x8, x1 ; CHECK-NEXT: asr x8, x9, x8 ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: ret