[AArch64] Add a special case for shifting by (BitWidth - 1) - X

[foad]

Ping!

[RKSimon]

Commit the new tests against trunk and then rebase to show the codegen diffs

[foad]

Rebase after precommitting tests.

[efriedma]

What happens if the shift amount has more than one use?

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

2873

Please use mvn to NOT a register. (This doesn't affect performance, but it's more readable.)

[foad]

Rebase. Generate mvn instead of eon.

[foad]

In D77316#2040193, @efriedma wrote:

What happens if the shift amount has more than one use?

Not sure, but the same as the existing special case for shifting by BitWidth - X.

[efriedma]

In D77316#3307261, @foad wrote:

In D77316#2040193, @efriedma wrote:

What happens if the shift amount has more than one use?

Not sure, but the same as the existing special case for shifting by BitWidth - X.

So, we emit the shift amount separately. Which isn't a big deal, I guess, but we should have a testcase. Something like:

define void @foo(i64* %valptr, i64* %shamtptr, i64 %shamt) {
  %val = load i64, i64* %valptr
  %negshamt = sub i64 64, %shamt
  %shifted = ashr i64 %val, %negshamt
  store i64 %shifted, i64* %valptr
  store i64 %negshamt, i64* %shamtptr
  ret void
}

[foad]

Rebase on multi-use tests.

[foad]

So, we emit the shift amount separately.

So it looks like we should only do this when the shift amount has a single use? Can I leave that for a separate patch?

[efriedma]

LGTM. I'm fine putting the multi-use handling in a followup. (For that, I guess you might also want to consider the case where a shift amount is used by multiple shifts...)

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
       ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
     ShiftAmt = ShiftAmt->getOperand(0);
 
   if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
     SDValue Add0 = ShiftAmt->getOperand(0);
     SDValue Add1 = ShiftAmt->getOperand(1);
     uint64_t Add0Imm;
     uint64_t Add1Imm;
+    if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0)) {
       // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
       // to avoid the ADD/SUB.
-    if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0))
       NewShiftAmt = Add0;
-    // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
-    // generate a NEG instead of a SUB of a constant.
-    else if (ShiftAmt->getOpcode() == ISD::SUB &&
+    } else if (ShiftAmt->getOpcode() == ISD::SUB &&
                isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 &&
                (Add0Imm % Size == 0)) {
+      // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
+      // to generate a NEG instead of a SUB from a constant.
       unsigned NegOpc;
       unsigned ZeroReg;
       EVT SubVT = ShiftAmt->getValueType(0);
       if (SubVT == MVT::i32) {
         NegOpc = AArch64::SUBWrr;
         ZeroReg = AArch64::WZR;
       } else {
         assert(SubVT == MVT::i64);
         NegOpc = AArch64::SUBXrr;
         ZeroReg = AArch64::XZR;
       }
       SDValue Zero =
           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
       MachineSDNode *Neg =
           CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1);
       NewShiftAmt = SDValue(Neg, 0);
+    } else if (ShiftAmt->getOpcode() == ISD::SUB &&
+               isIntImmediate(Add0, Add0Imm) && (Add0Imm % Size == Size - 1)) {
+      // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
+      // to generate a NOT instead of a SUB from a constant.
+      unsigned NotOpc;
+      unsigned ZeroReg;
+      EVT SubVT = ShiftAmt->getValueType(0);
+      if (SubVT == MVT::i32) {
+        NotOpc = AArch64::ORNWrr;
+        ZeroReg = AArch64::WZR;
+      } else {
+        assert(SubVT == MVT::i64);
+        NotOpc = AArch64::ORNXrr;
+        ZeroReg = AArch64::XZR;
+      }
+      SDValue Zero =
+          CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
+      MachineSDNode *Not =
+          CurDAG->getMachineNode(NotOpc, DL, SubVT, Zero, Add1);
+      NewShiftAmt = SDValue(Not, 0);
     } else
       return false;
   } else {
     // If the shift amount is masked with an AND, check that the mask covers the
     // bits that are implicitly ANDed off by the above opcodes and if so, skip
     // the AND.
     uint64_t MaskImm;
     if (!isOpcWithIntImmediate(ShiftAmt.getNode(), ISD::AND, MaskImm) &&

llvm/test/CodeGen/AArch64/shift-amount-mod.ll

 ;==============================================================================;
 
 ; shift left
 ;------------------------------------------------------------------------------;
 
 define i32 @reg32_shl_by_complemented(i32 %val, i32 %shamt) nounwind {
 ; CHECK-LABEL: reg32_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
-; CHECK-NEXT:    sub w8, w8, w1
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    lsl w0, w0, w8
 ; CHECK-NEXT:    ret
   %negshamt = sub i32 31, %shamt
   %shifted = shl i32 %val, %negshamt
   ret i32 %shifted
 }
 define i32 @load32_shl_by_complemented(i32* %valptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: load32_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
 ; CHECK-NEXT:    lsl w0, w9, w8
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = shl i32 %val, %negshamt
   ret i32 %shifted
 }
 define void @store32_shl_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: store32_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
-; CHECK-NEXT:    sub w8, w8, w2
+; CHECK-NEXT:    mvn w8, w2
 ; CHECK-NEXT:    lsl w8, w0, w8
 ; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    ret
   %negshamt = sub i32 31, %shamt
   %shifted = shl i32 %val, %negshamt
   store i32 %shifted, i32* %dstptr
   ret void
 }
 define void @modify32_shl_by_complemented(i32* %valptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: modify32_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
 ; CHECK-NEXT:    lsl w8, w9, w8
 ; CHECK-NEXT:    str w8, [x0]
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = shl i32 %val, %negshamt
   store i32 %shifted, i32* %valptr
   ret void
 }
 define void @modify32_shl_by_complemented_multi_use(i32* %valptr, i32 %shamt, i32* %shamtptr) nounwind {
 ; CHECK-LABEL: modify32_shl_by_complemented_multi_use:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
-; CHECK-NEXT:    lsl w9, w9, w8
-; CHECK-NEXT:    str w9, [x0]
-; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    mov w10, #31
+; CHECK-NEXT:    lsl w8, w9, w8
+; CHECK-NEXT:    sub w9, w10, w1
+; CHECK-NEXT:    str w8, [x0]
+; CHECK-NEXT:    str w9, [x2]
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = shl i32 %val, %negshamt
   store i32 %shifted, i32* %valptr
   store i32 %negshamt, i32* %shamtptr
   ret void
 }
 
 define i64 @reg64_shl_by_complemented(i64 %val, i64 %shamt) nounwind {
 ; CHECK-LABEL: reg64_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
-; CHECK-NEXT:    sub x8, x8, x1
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    lsl x0, x0, x8
 ; CHECK-NEXT:    ret
   %negshamt = sub i64 63, %shamt
   %shifted = shl i64 %val, %negshamt
   ret i64 %shifted
 }
 define i64 @load64_shl_by_complemented(i64* %valptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: load64_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
 ; CHECK-NEXT:    lsl x0, x9, x8
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = shl i64 %val, %negshamt
   ret i64 %shifted
 }
 define void @store64_shl_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: store64_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
-; CHECK-NEXT:    sub x8, x8, x2
+; CHECK-NEXT:    mvn x8, x2
 ; CHECK-NEXT:    lsl x8, x0, x8
 ; CHECK-NEXT:    str x8, [x1]
 ; CHECK-NEXT:    ret
   %negshamt = sub i64 63, %shamt
   %shifted = shl i64 %val, %negshamt
   store i64 %shifted, i64* %dstptr
   ret void
 }
 define void @modify64_shl_by_complemented(i64* %valptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: modify64_shl_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
 ; CHECK-NEXT:    lsl x8, x9, x8
 ; CHECK-NEXT:    str x8, [x0]
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = shl i64 %val, %negshamt
   store i64 %shifted, i64* %valptr
   ret void
 }
 define void @modify64_shl_by_complemented_multi_use(i64* %valptr, i64 %shamt, i64* %shamtptr) nounwind {
 ; CHECK-LABEL: modify64_shl_by_complemented_multi_use:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
-; CHECK-NEXT:    lsl x9, x9, x8
-; CHECK-NEXT:    str x9, [x0]
-; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    mov w10, #63
+; CHECK-NEXT:    lsl x8, x9, x8
+; CHECK-NEXT:    sub x9, x10, x1
+; CHECK-NEXT:    str x8, [x0]
+; CHECK-NEXT:    str x9, [x2]
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = shl i64 %val, %negshamt
   store i64 %shifted, i64* %valptr
   store i64 %negshamt, i64* %shamtptr
   ret void
 }
 
 ; logical shift right
 ;------------------------------------------------------------------------------;
 
 define i32 @reg32_lshr_by_complemented(i32 %val, i32 %shamt) nounwind {
 ; CHECK-LABEL: reg32_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
-; CHECK-NEXT:    sub w8, w8, w1
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    lsr w0, w0, w8
 ; CHECK-NEXT:    ret
   %negshamt = sub i32 31, %shamt
   %shifted = lshr i32 %val, %negshamt
   ret i32 %shifted
 }
 define i32 @load32_lshr_by_complemented(i32* %valptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: load32_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
 ; CHECK-NEXT:    lsr w0, w9, w8
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = lshr i32 %val, %negshamt
   ret i32 %shifted
 }
 define void @store32_lshr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: store32_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
-; CHECK-NEXT:    sub w8, w8, w2
+; CHECK-NEXT:    mvn w8, w2
 ; CHECK-NEXT:    lsr w8, w0, w8
 ; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    ret
   %negshamt = sub i32 31, %shamt
   %shifted = lshr i32 %val, %negshamt
   store i32 %shifted, i32* %dstptr
   ret void
 }
 define void @modify32_lshr_by_complemented(i32* %valptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: modify32_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
 ; CHECK-NEXT:    lsr w8, w9, w8
 ; CHECK-NEXT:    str w8, [x0]
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = lshr i32 %val, %negshamt
   store i32 %shifted, i32* %valptr
   ret void
 }
 define void @modify32_lshr_by_complemented_multi_use(i32* %valptr, i32 %shamt, i32* %shamtptr) nounwind {
 ; CHECK-LABEL: modify32_lshr_by_complemented_multi_use:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
-; CHECK-NEXT:    lsr w9, w9, w8
-; CHECK-NEXT:    str w9, [x0]
-; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    mov w10, #31
+; CHECK-NEXT:    lsr w8, w9, w8
+; CHECK-NEXT:    sub w9, w10, w1
+; CHECK-NEXT:    str w8, [x0]
+; CHECK-NEXT:    str w9, [x2]
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = lshr i32 %val, %negshamt
   store i32 %shifted, i32* %valptr
   store i32 %negshamt, i32* %shamtptr
   ret void
 }
 
 define i64 @reg64_lshr_by_complemented(i64 %val, i64 %shamt) nounwind {
 ; CHECK-LABEL: reg64_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
-; CHECK-NEXT:    sub x8, x8, x1
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    lsr x0, x0, x8
 ; CHECK-NEXT:    ret
   %negshamt = sub i64 63, %shamt
   %shifted = lshr i64 %val, %negshamt
   ret i64 %shifted
 }
 define i64 @load64_lshr_by_complemented(i64* %valptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: load64_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
 ; CHECK-NEXT:    lsr x0, x9, x8
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = lshr i64 %val, %negshamt
   ret i64 %shifted
 }
 define void @store64_lshr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: store64_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
-; CHECK-NEXT:    sub x8, x8, x2
+; CHECK-NEXT:    mvn x8, x2
 ; CHECK-NEXT:    lsr x8, x0, x8
 ; CHECK-NEXT:    str x8, [x1]
 ; CHECK-NEXT:    ret
   %negshamt = sub i64 63, %shamt
   %shifted = lshr i64 %val, %negshamt
   store i64 %shifted, i64* %dstptr
   ret void
 }
 define void @modify64_lshr_by_complemented(i64* %valptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: modify64_lshr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
 ; CHECK-NEXT:    lsr x8, x9, x8
 ; CHECK-NEXT:    str x8, [x0]
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = lshr i64 %val, %negshamt
   store i64 %shifted, i64* %valptr
   ret void
 }
 define void @modify64_lshr_by_complemented_multi_use(i64* %valptr, i64 %shamt, i64* %shamtptr) nounwind {
 ; CHECK-LABEL: modify64_lshr_by_complemented_multi_use:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
-; CHECK-NEXT:    lsr x9, x9, x8
-; CHECK-NEXT:    str x9, [x0]
-; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    mov w10, #63
+; CHECK-NEXT:    lsr x8, x9, x8
+; CHECK-NEXT:    sub x9, x10, x1
+; CHECK-NEXT:    str x8, [x0]
+; CHECK-NEXT:    str x9, [x2]
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = lshr i64 %val, %negshamt
   store i64 %shifted, i64* %valptr
   store i64 %negshamt, i64* %shamtptr
   ret void
 }
 
 ; arithmetic shift right
 ;------------------------------------------------------------------------------;
 
 define i32 @reg32_ashr_by_complemented(i32 %val, i32 %shamt) nounwind {
 ; CHECK-LABEL: reg32_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
-; CHECK-NEXT:    sub w8, w8, w1
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    asr w0, w0, w8
 ; CHECK-NEXT:    ret
   %negshamt = sub i32 31, %shamt
   %shifted = ashr i32 %val, %negshamt
   ret i32 %shifted
 }
 define i32 @load32_ashr_by_complemented(i32* %valptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: load32_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
 ; CHECK-NEXT:    asr w0, w9, w8
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = ashr i32 %val, %negshamt
   ret i32 %shifted
 }
 define void @store32_ashr_by_complemented(i32 %val, i32* %dstptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: store32_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
-; CHECK-NEXT:    sub w8, w8, w2
+; CHECK-NEXT:    mvn w8, w2
 ; CHECK-NEXT:    asr w8, w0, w8
 ; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    ret
   %negshamt = sub i32 31, %shamt
   %shifted = ashr i32 %val, %negshamt
   store i32 %shifted, i32* %dstptr
   ret void
 }
 define void @modify32_ashr_by_complemented(i32* %valptr, i32 %shamt) nounwind {
 ; CHECK-LABEL: modify32_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
 ; CHECK-NEXT:    asr w8, w9, w8
 ; CHECK-NEXT:    str w8, [x0]
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = ashr i32 %val, %negshamt
   store i32 %shifted, i32* %valptr
   ret void
 }
 define void @modify32_ashr_by_complemented_multi_use(i32* %valptr, i32 %shamt, i32* %shamtptr) nounwind {
 ; CHECK-LABEL: modify32_ashr_by_complemented_multi_use:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31
+; CHECK-NEXT:    mvn w8, w1
 ; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    sub w8, w8, w1
-; CHECK-NEXT:    asr w9, w9, w8
-; CHECK-NEXT:    str w9, [x0]
-; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    mov w10, #31
+; CHECK-NEXT:    asr w8, w9, w8
+; CHECK-NEXT:    sub w9, w10, w1
+; CHECK-NEXT:    str w8, [x0]
+; CHECK-NEXT:    str w9, [x2]
 ; CHECK-NEXT:    ret
   %val = load i32, i32* %valptr
   %negshamt = sub i32 31, %shamt
   %shifted = ashr i32 %val, %negshamt
   store i32 %shifted, i32* %valptr
   store i32 %negshamt, i32* %shamtptr
   ret void
 }
 
 define i64 @reg64_ashr_by_complemented(i64 %val, i64 %shamt) nounwind {
 ; CHECK-LABEL: reg64_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
-; CHECK-NEXT:    sub x8, x8, x1
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    asr x0, x0, x8
 ; CHECK-NEXT:    ret
   %negshamt = sub i64 63, %shamt
   %shifted = ashr i64 %val, %negshamt
   ret i64 %shifted
 }
 define i64 @load64_ashr_by_complemented(i64* %valptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: load64_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
 ; CHECK-NEXT:    asr x0, x9, x8
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = ashr i64 %val, %negshamt
   ret i64 %shifted
 }
 define void @store64_ashr_by_complemented(i64 %val, i64* %dstptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: store64_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
-; CHECK-NEXT:    sub x8, x8, x2
+; CHECK-NEXT:    mvn x8, x2
 ; CHECK-NEXT:    asr x8, x0, x8
 ; CHECK-NEXT:    str x8, [x1]
 ; CHECK-NEXT:    ret
   %negshamt = sub i64 63, %shamt
   %shifted = ashr i64 %val, %negshamt
   store i64 %shifted, i64* %dstptr
   ret void
 }
 define void @modify64_ashr_by_complemented(i64* %valptr, i64 %shamt) nounwind {
 ; CHECK-LABEL: modify64_ashr_by_complemented:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
 ; CHECK-NEXT:    asr x8, x9, x8
 ; CHECK-NEXT:    str x8, [x0]
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = ashr i64 %val, %negshamt
   store i64 %shifted, i64* %valptr
   ret void
 }
 define void @modify64_ashr_by_complemented_multi_use(i64* %valptr, i64 %shamt, i64* %shamtptr) nounwind {
 ; CHECK-LABEL: modify64_ashr_by_complemented_multi_use:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #63
+; CHECK-NEXT:    mvn x8, x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    sub x8, x8, x1
-; CHECK-NEXT:    asr x9, x9, x8
-; CHECK-NEXT:    str x9, [x0]
-; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    mov w10, #63
+; CHECK-NEXT:    asr x8, x9, x8
+; CHECK-NEXT:    sub x9, x10, x1
+; CHECK-NEXT:    str x8, [x0]
+; CHECK-NEXT:    str x9, [x2]
 ; CHECK-NEXT:    ret
   %val = load i64, i64* %valptr
   %negshamt = sub i64 63, %shamt
   %shifted = ashr i64 %val, %negshamt
   store i64 %shifted, i64* %valptr
   store i64 %negshamt, i64* %shamtptr
   ret void
 }