diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -2708,15 +2708,15 @@ SDValue Add1 = ShiftAmt->getOperand(1); uint64_t Add0Imm; uint64_t Add1Imm; - // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X - // to avoid the ADD/SUB. - if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) + if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) { + // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X + // to avoid the ADD/SUB. NewShiftAmt = Add0; - // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to - // generate a NEG instead of a SUB of a constant. - else if (ShiftAmt->getOpcode() == ISD::SUB && - isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 && - (Add0Imm % Size == 0)) { + } else if (ShiftAmt->getOpcode() == ISD::SUB && + isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 && + (Add0Imm % Size == 0)) { + // If we are shifting by N-X where N == 0 mod Size, then just shift by -X + // to generate a NEG instead of a SUB from a constant. unsigned NegOpc; unsigned ZeroReg; EVT SubVT = ShiftAmt->getValueType(0); @@ -2733,6 +2733,26 @@ MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1); NewShiftAmt = SDValue(Neg, 0); + } else if (ShiftAmt->getOpcode() == ISD::SUB && + isIntImmediate(Add0, Add0Imm) && (Add0Imm % Size == Size - 1)) { + // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X + // to generate a NOT instead of a SUB from a constant. + unsigned NotOpc; + unsigned ZeroReg; + EVT SubVT = ShiftAmt->getValueType(0); + if (SubVT == MVT::i32) { + NotOpc = AArch64::EONWrr; + ZeroReg = AArch64::WZR; + } else { + assert(SubVT == MVT::i64); + NotOpc = AArch64::EONXrr; + ZeroReg = AArch64::XZR; + } + SDValue Zero = + CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT); + MachineSDNode *Not = + CurDAG->getMachineNode(NotOpc, DL, SubVT, Add1, Zero); + NewShiftAmt = SDValue(Not, 0); } else return false; } else { diff --git a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll --- a/llvm/test/CodeGen/AArch64/shift-amount-mod.ll +++ b/llvm/test/CodeGen/AArch64/shift-amount-mod.ll @@ -308,6 +308,313 @@ ret void } +;==============================================================================; +; the shift amount is complemented (shiftbitwidth - 1 - shiftamt) +;==============================================================================; + +; shift left +;------------------------------------------------------------------------------; + +define i32 @reg32_shl_by_complemented(i32 %val, i32 %shamt) nounwind { +; CHECK-LABEL: reg32_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon w8, w1, wzr +; CHECK-NEXT: lsl w0, w0, w8 +; CHECK-NEXT: ret + %negshamt = sub i32 31, %shamt + %shifted = shl i32 %val, %negshamt + ret i32 %shifted +} +define i32 @load32_shl_by_complemented(i32* %valptr, i32 %shamt) nounwind { +; CHECK-LABEL: load32_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: eon w9, w1, wzr +; CHECK-NEXT: lsl w0, w8, w9 +; CHECK-NEXT: ret + %val = load i32, i32* %valptr + %negshamt = sub i32 31, %shamt + %shifted = shl i32 %val, %negshamt + ret i32 %shifted +} +define void @store32_shl_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { +; CHECK-LABEL: store32_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon w8, w2, wzr +; CHECK-NEXT: lsl w8, w0, w8 +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret + %negshamt = sub i32 31, %shamt + %shifted = shl i32 %val, %negshamt + store i32 %shifted, i32* %dstptr + ret void +} +define void @modify32_shl_by_complemented(i32* %valptr, i32 %shamt) nounwind { +; CHECK-LABEL: modify32_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: eon w9, w1, wzr +; CHECK-NEXT: lsl w8, w8, w9 +; CHECK-NEXT: str w8, [x0] +; CHECK-NEXT: ret + %val = load i32, i32* %valptr + %negshamt = sub i32 31, %shamt + %shifted = shl i32 %val, %negshamt + store i32 %shifted, i32* %valptr + ret void +} + +define i64 @reg64_shl_by_complemented(i64 %val, i64 %shamt) nounwind { +; CHECK-LABEL: reg64_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon x8, x1, xzr +; CHECK-NEXT: lsl x0, x0, x8 +; CHECK-NEXT: ret + %negshamt = sub i64 63, %shamt + %shifted = shl i64 %val, %negshamt + ret i64 %shifted +} +define i64 @load64_shl_by_complemented(i64* %valptr, i64 %shamt) nounwind { +; CHECK-LABEL: load64_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr x8, [x0] +; CHECK-NEXT: eon x9, x1, xzr +; CHECK-NEXT: lsl x0, x8, x9 +; CHECK-NEXT: ret + %val = load i64, i64* %valptr + %negshamt = sub i64 63, %shamt + %shifted = shl i64 %val, %negshamt + ret i64 %shifted +} +define void @store64_shl_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { +; CHECK-LABEL: store64_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon x8, x2, xzr +; CHECK-NEXT: lsl x8, x0, x8 +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret + %negshamt = sub i64 63, %shamt + %shifted = shl i64 %val, %negshamt + store i64 %shifted, i64* %dstptr + ret void +} +define void @modify64_shl_by_complemented(i64* %valptr, i64 %shamt) nounwind { +; CHECK-LABEL: modify64_shl_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr x8, [x0] +; CHECK-NEXT: eon x9, x1, xzr +; CHECK-NEXT: lsl x8, x8, x9 +; CHECK-NEXT: str x8, [x0] +; CHECK-NEXT: ret + %val = load i64, i64* %valptr + %negshamt = sub i64 63, %shamt + %shifted = shl i64 %val, %negshamt + store i64 %shifted, i64* %valptr + ret void +} + +; logical shift right +;------------------------------------------------------------------------------; + +define i32 @reg32_lshr_by_complemented(i32 %val, i32 %shamt) nounwind { +; CHECK-LABEL: reg32_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon w8, w1, wzr +; CHECK-NEXT: lsr w0, w0, w8 +; CHECK-NEXT: ret + %negshamt = sub i32 31, %shamt + %shifted = lshr i32 %val, %negshamt + ret i32 %shifted +} +define i32 @load32_lshr_by_complemented(i32* %valptr, i32 %shamt) nounwind { +; CHECK-LABEL: load32_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: eon w9, w1, wzr +; CHECK-NEXT: lsr w0, w8, w9 +; CHECK-NEXT: ret + %val = load i32, i32* %valptr + %negshamt = sub i32 31, %shamt + %shifted = lshr i32 %val, %negshamt + ret i32 %shifted +} +define void @store32_lshr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { +; CHECK-LABEL: store32_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon w8, w2, wzr +; CHECK-NEXT: lsr w8, w0, w8 +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret + %negshamt = sub i32 31, %shamt + %shifted = lshr i32 %val, %negshamt + store i32 %shifted, i32* %dstptr + ret void +} +define void @modify32_lshr_by_complemented(i32* %valptr, i32 %shamt) nounwind { +; CHECK-LABEL: modify32_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: eon w9, w1, wzr +; CHECK-NEXT: lsr w8, w8, w9 +; CHECK-NEXT: str w8, [x0] +; CHECK-NEXT: ret + %val = load i32, i32* %valptr + %negshamt = sub i32 31, %shamt + %shifted = lshr i32 %val, %negshamt + store i32 %shifted, i32* %valptr + ret void +} + +define i64 @reg64_lshr_by_complemented(i64 %val, i64 %shamt) nounwind { +; CHECK-LABEL: reg64_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon x8, x1, xzr +; CHECK-NEXT: lsr x0, x0, x8 +; CHECK-NEXT: ret + %negshamt = sub i64 63, %shamt + %shifted = lshr i64 %val, %negshamt + ret i64 %shifted +} +define i64 @load64_lshr_by_complemented(i64* %valptr, i64 %shamt) nounwind { +; CHECK-LABEL: load64_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr x8, [x0] +; CHECK-NEXT: eon x9, x1, xzr +; CHECK-NEXT: lsr x0, x8, x9 +; CHECK-NEXT: ret + %val = load i64, i64* %valptr + %negshamt = sub i64 63, %shamt + %shifted = lshr i64 %val, %negshamt + ret i64 %shifted +} +define void @store64_lshr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { +; CHECK-LABEL: store64_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon x8, x2, xzr +; CHECK-NEXT: lsr x8, x0, x8 +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret + %negshamt = sub i64 63, %shamt + %shifted = lshr i64 %val, %negshamt + store i64 %shifted, i64* %dstptr + ret void +} +define void @modify64_lshr_by_complemented(i64* %valptr, i64 %shamt) nounwind { +; CHECK-LABEL: modify64_lshr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr x8, [x0] +; CHECK-NEXT: eon x9, x1, xzr +; CHECK-NEXT: lsr x8, x8, x9 +; CHECK-NEXT: str x8, [x0] +; CHECK-NEXT: ret + %val = load i64, i64* %valptr + %negshamt = sub i64 63, %shamt + %shifted = lshr i64 %val, %negshamt + store i64 %shifted, i64* %valptr + ret void +} + +; arithmetic shift right +;------------------------------------------------------------------------------; + +define i32 @reg32_ashr_by_complemented(i32 %val, i32 %shamt) nounwind { +; CHECK-LABEL: reg32_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon w8, w1, wzr +; CHECK-NEXT: asr w0, w0, w8 +; CHECK-NEXT: ret + %negshamt = sub i32 31, %shamt + %shifted = ashr i32 %val, %negshamt + ret i32 %shifted +} +define i32 @load32_ashr_by_complemented(i32* %valptr, i32 %shamt) nounwind { +; CHECK-LABEL: load32_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: eon w9, w1, wzr +; CHECK-NEXT: asr w0, w8, w9 +; CHECK-NEXT: ret + %val = load i32, i32* %valptr + %negshamt = sub i32 31, %shamt + %shifted = ashr i32 %val, %negshamt + ret i32 %shifted +} +define void @store32_ashr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind { +; CHECK-LABEL: store32_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon w8, w2, wzr +; CHECK-NEXT: asr w8, w0, w8 +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret + %negshamt = sub i32 31, %shamt + %shifted = ashr i32 %val, %negshamt + store i32 %shifted, i32* %dstptr + ret void +} +define void @modify32_ashr_by_complemented(i32* %valptr, i32 %shamt) nounwind { +; CHECK-LABEL: modify32_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: eon w9, w1, wzr +; CHECK-NEXT: asr w8, w8, w9 +; CHECK-NEXT: str w8, [x0] +; CHECK-NEXT: ret + %val = load i32, i32* %valptr + %negshamt = sub i32 31, %shamt + %shifted = ashr i32 %val, %negshamt + store i32 %shifted, i32* %valptr + ret void +} + +define i64 @reg64_ashr_by_complemented(i64 %val, i64 %shamt) nounwind { +; CHECK-LABEL: reg64_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon x8, x1, xzr +; CHECK-NEXT: asr x0, x0, x8 +; CHECK-NEXT: ret + %negshamt = sub i64 63, %shamt + %shifted = ashr i64 %val, %negshamt + ret i64 %shifted +} +define i64 @load64_ashr_by_complemented(i64* %valptr, i64 %shamt) nounwind { +; CHECK-LABEL: load64_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr x8, [x0] +; CHECK-NEXT: eon x9, x1, xzr +; CHECK-NEXT: asr x0, x8, x9 +; CHECK-NEXT: ret + %val = load i64, i64* %valptr + %negshamt = sub i64 63, %shamt + %shifted = ashr i64 %val, %negshamt + ret i64 %shifted +} +define void @store64_ashr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind { +; CHECK-LABEL: store64_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: eon x8, x2, xzr +; CHECK-NEXT: asr x8, x0, x8 +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret + %negshamt = sub i64 63, %shamt + %shifted = ashr i64 %val, %negshamt + store i64 %shifted, i64* %dstptr + ret void +} +define void @modify64_ashr_by_complemented(i64* %valptr, i64 %shamt) nounwind { +; CHECK-LABEL: modify64_ashr_by_complemented: +; CHECK: // %bb.0: +; CHECK-NEXT: ldr x8, [x0] +; CHECK-NEXT: eon x9, x1, xzr +; CHECK-NEXT: asr x8, x8, x9 +; CHECK-NEXT: str x8, [x0] +; CHECK-NEXT: ret + %val = load i64, i64* %valptr + %negshamt = sub i64 63, %shamt + %shifted = ashr i64 %val, %negshamt + store i64 %shifted, i64* %valptr + ret void +} + ;||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||; ; next let's only test simple reg pattern, and only lshr. ;||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||;