[DAGCombine] Move AND nodes to multiple load leaves

[samparker]

Summary

rL319773 was reverted due to a recursive issue causing timeouts. This happened because I failed to check whether the discovered loads could be narrowed further. In the cases of narrow loads that could not be further narrowed, an AND was being introduced and not combined away - only to be combined into another AND and so on... forever.

[niravd]

So if the Load is the same size as the mask, we used insert an AND but we don't end up eliding it? That seems like a fence post error elsewhere and it may be worth checking that out.

In any case, this change looks good. LGTM. Next time reopen the old revision, and update it. That way we can see the diffs directly on phabricator.

[samparker]

Ok, thanks. In this case the AND does get combined with the load, but a node also needs to be fixed up and so introduces another masking AND. This AND is then later combined with another node, in the process I assume added to the worklist, and then later hits visitAND. visitAND tries to combine with a load, which it does, but it also performs the fix up again, beginning the process again. I'm going to add another test case to satisfy myself that this works.

[samparker]

rL320679 was reverted due to a miscompilation. I had made the false assumption that constant operands would have been narrowed. So now any OR and XOR nodes which use constant operands, which are wider than the mask, have their operand narrowed at the end of the process.

[samparker]

I'm unsure how I've managed to accept the revision...

[niravd]

The previous acceptance still applies. You can explicitly request a review

In D41177#956806, @samparker wrote:

I'm unsure how I've managed to accept the revision...

[niravd]

Do you have a reasonable test case of the failing case you can add?

LGTM otherwise.

[samparker]

Yes, the newly added 'test9' function was taken from the reproducer. I've also added 'test10'.

llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

private:
     bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
                           EVT LoadResultTy, EVT &ExtVT);
 
     /// Helper function to calculate whether the given Load can have its
     /// width reduced to ExtVT.
     bool isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
                            EVT &ExtVT, unsigned ShAmt = 0);
 
+    /// Used by BackwardsPropagateMask to find suitable loads.
+    bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
+                           ConstantSDNode *Mask, SDNode *&UncombinedNode);
+    /// Attempt to propagate a given AND node back to load leaves so that they
+    /// can be combined into narrow loads.
+    bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
+
     /// Helper function for MergeConsecutiveStores which merges the
     /// component store chains.
     SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
                                 unsigned NumStores);
 
     /// This is a helper function for MergeConsecutiveStores. When the
     /// source elements of the consecutive stores are all constants or
     /// all extracted vector elements, try to merge them into one
bool DAGCombiner::isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
   // It's not possible to generate a constant of extended or untyped type.
   EVT PtrType = LoadN->getOperand(1).getValueType();
   if (PtrType == MVT::Untyped || PtrType.isExtended())
     return false;
 
   return true;
 }
 
+bool DAGCombiner::SearchForAndLoads(SDNode *N,
+                                    SmallPtrSetImpl<LoadSDNode*> &Loads,
+                                    ConstantSDNode *Mask,
+                                    SDNode *&NodeToMask) {
+  // Recursively search for the operands, looking for loads which can be
+  // narrowed.
+  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
+    SDValue Op = N->getOperand(i);
+
+    // Constants should already be fixed up...
+    if (isa<ConstantSDNode>(Op))
+      continue;
+
+    if (!Op.hasOneUse() || Op.getValueType().isVector())
+      return false;
+
+    switch(Op.getOpcode()) {
+    case ISD::LOAD: {
+      auto *Load = cast<LoadSDNode>(Op);
+      EVT ExtVT;
+      if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
+          isLegalNarrowLoad(Load, ISD::ZEXTLOAD, ExtVT)) {
+        // Only add this load if we can make it more narrow.
+        if (ExtVT.bitsLT(Load->getMemoryVT()))
+          Loads.insert(Load);
+        continue;
+      }
+      return false;
+    }
+    case ISD::ZERO_EXTEND:
+    case ISD::ANY_EXTEND:
+    case ISD::AssertZext: {
+      unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
+      EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
+      EVT VT = Op.getOpcode() == ISD::AssertZext ?
+        cast<VTSDNode>(Op.getOperand(1))->getVT() :
+        Op.getOperand(0).getValueType();
+
+      // We can accept extending nodes if the mask is wider or an equal
+      // width to the original type.
+      if (ExtVT.bitsGE(VT))
+        continue;
+      break;
+    }
+    case ISD::OR:
+    case ISD::XOR:
+    case ISD::AND:
+      if (!SearchForAndLoads(Op.getNode(), Loads, Mask, NodeToMask))
+        return false;
+      continue;
+    }
+
+    // Allow one node which will masked along with any loads found.
+    if (NodeToMask)
+      return false;
+    NodeToMask = Op.getNode();
+  }
+  return true;
+}
+
+bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
+  auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
+  if (!Mask)
+    return false;
+
+  if (!Mask->getAPIntValue().isMask())
+    return false;
+
+  // No need to do anything if the and directly uses a load.
+  if (isa<LoadSDNode>(N->getOperand(0)))
+    return false;
+
+  SmallPtrSet<LoadSDNode*, 8> Loads;
+  SDNode *FixupNode = nullptr;
+  if (SearchForAndLoads(N, Loads, Mask, FixupNode)) {
+    if (Loads.size() == 0)
+      return false;
+
+    SDValue MaskOp = N->getOperand(1);
+
+    // If it exists, fixup the single node we allow in the tree that needs
+    // masking.
+    if (FixupNode) {
+      SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
+                                FixupNode->getValueType(0),
+                                SDValue(FixupNode, 0), MaskOp);
+      DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
+      DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0),
+                             MaskOp);
+    }
+
+    for (auto *Load : Loads) {
+      SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
+                                SDValue(Load, 0), MaskOp);
+      DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
+      DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp);
+      SDValue NewLoad = ReduceLoadWidth(And.getNode());
+      assert(NewLoad &&
+             "Shouldn't be masking the load if it can't be narrowed");
+      CombineTo(Load, NewLoad, NewLoad.getValue(1));
+    }
+    DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
+    return true;
+  }
+  return false;
+}
+
 SDValue DAGCombiner::visitAND(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N1.getValueType();
 
   // x & x --> x
   if (N0 == N1)
     return N0;
if (SDValue Res = ReduceLoadWidth(N)) {
         ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
 
       AddToWorklist(N);
       CombineTo(LN0, Res, Res.getValue(1));
       return SDValue(N, 0);
     }
   }
 
+  if (Level >= AfterLegalizeTypes) {
+    // Attempt to propagate the AND back up to the leaves which, if they're
+    // loads, can be combined to narrow loads and the AND node can be removed.
+    // Perform after legalization so that extend nodes will already be
+    // combined into the loads.
+    if (BackwardsPropagateMask(N, DAG)) {
+      return SDValue(N, 0);
+    }
+  }
+
   if (SDValue Combined = visitANDLike(N0, N1, N))
     return Combined;
 
   // Simplify: (and (op x...), (op y...))  -> (op (and x, y))
   if (N0.getOpcode() == N1.getOpcode())
     if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
       return Tmp;
 

llvm/trunk/test/CodeGen/ARM/and-load-combine.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=armv7 %s -o - | FileCheck %s --check-prefix=ARM
 ; RUN: llc -mtriple=armv7eb %s -o - | FileCheck %s --check-prefix=ARMEB
 ; RUN: llc -mtriple=armv6m %s -o - | FileCheck %s --check-prefix=THUMB1
 ; RUN: llc -mtriple=thumbv8m.main %s -o - | FileCheck %s --check-prefix=THUMB2
 
 define arm_aapcscc zeroext i1 @cmp_xor8_short_short(i16* nocapture readonly %a,
+                                                    i16* nocapture readonly %b) {
 ; ARM-LABEL: cmp_xor8_short_short:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldrh r0, [r0]
-; ARM-NEXT:    ldrh r1, [r1]
-; ARM-NEXT:    eor r1, r1, r0
+; ARM:         ldrb r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    teq r1, r2
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_xor8_short_short:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldrh r0, [r0]
-; ARMEB-NEXT:    ldrh r1, [r1]
-; ARMEB-NEXT:    eor r1, r1, r0
+; ARMEB:        ldrb r2, [r0, #1]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
+; ARMEB-NEXT:    ldrb r1, [r1, #1]
+; ARMEB-NEXT:    teq r1, r2
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_xor8_short_short:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldrh r0, [r0]
-; THUMB1-NEXT:    ldrh r2, [r1]
+; THUMB1:         ldrb r0, [r0]
+; THUMB1-NEXT:    ldrb r2, [r1]
 ; THUMB1-NEXT:    eors r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB0_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB0_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_xor8_short_short:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldrh r0, [r0]
-; THUMB2-NEXT:    ldrh r1, [r1]
-; THUMB2-NEXT:    eors r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrb r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrb r1, [r1]
+; THUMB2-NEXT:    teq.w r1, r2
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                    i16* nocapture readonly %b) {
 entry:
   %0 = load i16, i16* %a, align 2
   %1 = load i16, i16* %b, align 2
   %xor2 = xor i16 %1, %0
   %2 = and i16 %xor2, 255
   %cmp = icmp eq i16 %2, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_xor8_short_int(i16* nocapture readonly %a,
+                                                  i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_xor8_short_int:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldrh r0, [r0]
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    eor r1, r1, r0
+; ARM:         ldrb r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    teq r1, r2
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_xor8_short_int:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldrh r0, [r0]
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    eor r1, r1, r0
+; ARMEB:         ldrb r2, [r0, #1]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
+; ARMEB-NEXT:    ldrb r1, [r1, #3]
+; ARMEB-NEXT:    teq r1, r2
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_xor8_short_int:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldrh r0, [r0]
-; THUMB1-NEXT:    ldr r2, [r1]
+; THUMB1:         ldrb r0, [r0]
+; THUMB1-NEXT:    ldrb r2, [r1]
 ; THUMB1-NEXT:    eors r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB1_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB1_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_xor8_short_int:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldrh r0, [r0]
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    eors r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrb r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrb r1, [r1]
+; THUMB2-NEXT:    teq.w r1, r2
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                  i32* nocapture readonly %b) {
 entry:
   %0 = load i16, i16* %a, align 2
   %conv = zext i16 %0 to i32
   %1 = load i32, i32* %b, align 4
   %xor = xor i32 %1, %conv
   %and = and i32 %xor, 255
   %cmp = icmp eq i32 %and, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_xor8_int_int(i32* nocapture readonly %a,
+                                                i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_xor8_int_int:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r0, [r0]
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    eor r1, r1, r0
+; ARM:         ldrb r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    teq r1, r2
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_xor8_int_int:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r0, [r0]
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    eor r1, r1, r0
+; ARMEB:         ldrb r2, [r0, #3]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
+; ARMEB-NEXT:    ldrb r1, [r1, #3]
+; ARMEB-NEXT:    teq r1, r2
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_xor8_int_int:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r0, [r0]
-; THUMB1-NEXT:    ldr r2, [r1]
+; THUMB1:         ldrb r0, [r0]
+; THUMB1-NEXT:    ldrb r2, [r1]
 ; THUMB1-NEXT:    eors r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB2_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB2_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_xor8_int_int:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r0, [r0]
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    eors r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrb r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrb r1, [r1]
+; THUMB2-NEXT:    teq.w r1, r2
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                i32* nocapture readonly %b) {
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %xor = xor i32 %1, %0
   %and = and i32 %xor, 255
   %cmp = icmp eq i32 %and, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_xor16(i32* nocapture readonly %a,
+                                         i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_xor16:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r0, [r0]
-; ARM-NEXT:    movw r2, #65535
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    eor r1, r1, r0
+; ARM:         ldrh r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, r2
+; ARM-NEXT:    ldrh r1, [r1]
+; ARM-NEXT:    teq r1, r2
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_xor16:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r0, [r0]
-; ARMEB-NEXT:    movw r2, #65535
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    eor r1, r1, r0
+; ARMEB:         ldrh r2, [r0, #2]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, r2
+; ARMEB-NEXT:    ldrh r1, [r1, #2]
+; ARMEB-NEXT:    teq r1, r2
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_xor16:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r0, [r0]
-; THUMB1-NEXT:    ldr r2, [r1]
+; THUMB1:         ldrh r0, [r0]
+; THUMB1-NEXT:    ldrh r2, [r1]
 ; THUMB1-NEXT:    eors r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #16
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB3_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB3_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_xor16:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r0, [r0]
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    eors r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #16
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrh r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrh r1, [r1]
+; THUMB2-NEXT:    teq.w r1, r2
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                         i32* nocapture readonly %b) {
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %xor = xor i32 %1, %0
   %and = and i32 %xor, 65535
   %cmp = icmp eq i32 %and, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_or8_short_short(i16* nocapture readonly %a,
+                                                   i16* nocapture readonly %b) {
 ; ARM-LABEL: cmp_or8_short_short:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldrh r0, [r0]
-; ARM-NEXT:    ldrh r1, [r1]
-; ARM-NEXT:    orr r1, r1, r0
+; ARM:         ldrb r0, [r0]
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    orrs r0, r1, r0
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_or8_short_short:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldrh r0, [r0]
-; ARMEB-NEXT:    ldrh r1, [r1]
-; ARMEB-NEXT:    orr r1, r1, r0
+; ARMEB:         ldrb r0, [r0, #1]
+; ARMEB-NEXT:    ldrb r1, [r1, #1]
+; ARMEB-NEXT:    orrs r0, r1, r0
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_or8_short_short:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldrh r0, [r0]
-; THUMB1-NEXT:    ldrh r2, [r1]
+; THUMB1:         ldrb r0, [r0]
+; THUMB1-NEXT:    ldrb r2, [r1]
 ; THUMB1-NEXT:    orrs r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB4_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB4_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_or8_short_short:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldrh r0, [r0]
-; THUMB2-NEXT:    ldrh r1, [r1]
+; THUMB2:         ldrb r0, [r0]
+; THUMB2-NEXT:    ldrb r1, [r1]
 ; THUMB2-NEXT:    orrs r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
 ; THUMB2-NEXT:    mov.w r0, #0
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                   i16* nocapture readonly %b) {
 entry:
   %0 = load i16, i16* %a, align 2
   %1 = load i16, i16* %b, align 2
   %or2 = or i16 %1, %0
   %2 = and i16 %or2, 255
   %cmp = icmp eq i16 %2, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_or8_short_int(i16* nocapture readonly %a,
+                                                 i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_or8_short_int:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldrh r0, [r0]
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    orr r1, r1, r0
+; ARM:         ldrb r0, [r0]
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    orrs r0, r1, r0
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_or8_short_int:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldrh r0, [r0]
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    orr r1, r1, r0
+; ARMEB:         ldrb r0, [r0, #1]
+; ARMEB-NEXT:    ldrb r1, [r1, #3]
+; ARMEB-NEXT:    orrs r0, r1, r0
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_or8_short_int:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldrh r0, [r0]
-; THUMB1-NEXT:    ldr r2, [r1]
+; THUMB1:         ldrb r0, [r0]
+; THUMB1-NEXT:    ldrb r2, [r1]
 ; THUMB1-NEXT:    orrs r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB5_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB5_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_or8_short_int:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldrh r0, [r0]
-; THUMB2-NEXT:    ldr r1, [r1]
+; THUMB2:         ldrb r0, [r0]
+; THUMB2-NEXT:    ldrb r1, [r1]
 ; THUMB2-NEXT:    orrs r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
 ; THUMB2-NEXT:    mov.w r0, #0
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                 i32* nocapture readonly %b) {
 entry:
   %0 = load i16, i16* %a, align 2
   %conv = zext i16 %0 to i32
   %1 = load i32, i32* %b, align 4
   %or = or i32 %1, %conv
   %and = and i32 %or, 255
   %cmp = icmp eq i32 %and, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_or8_int_int(i32* nocapture readonly %a,
+                                               i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_or8_int_int:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r0, [r0]
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    orr r1, r1, r0
+; ARM:         ldrb r0, [r0]
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    orrs r0, r1, r0
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_or8_int_int:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r0, [r0]
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    orr r1, r1, r0
+; ARMEB:         ldrb r0, [r0, #3]
+; ARMEB-NEXT:    ldrb r1, [r1, #3]
+; ARMEB-NEXT:    orrs r0, r1, r0
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_or8_int_int:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r0, [r0]
-; THUMB1-NEXT:    ldr r2, [r1]
+; THUMB1:         ldrb r0, [r0]
+; THUMB1-NEXT:    ldrb r2, [r1]
 ; THUMB1-NEXT:    orrs r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB6_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB6_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_or8_int_int:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r0, [r0]
-; THUMB2-NEXT:    ldr r1, [r1]
+; THUMB2:         ldrb r0, [r0]
+; THUMB2-NEXT:    ldrb r1, [r1]
 ; THUMB2-NEXT:    orrs r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
 ; THUMB2-NEXT:    mov.w r0, #0
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                               i32* nocapture readonly %b) {
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %or = or i32 %1, %0
   %and = and i32 %or, 255
   %cmp = icmp eq i32 %and, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_or16(i32* nocapture readonly %a,
+                                        i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_or16:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r0, [r0]
-; ARM-NEXT:    movw r2, #65535
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    orr r1, r1, r0
+; ARM:         ldrh r0, [r0]
+; ARM-NEXT:    ldrh r1, [r1]
+; ARM-NEXT:    orrs r0, r1, r0
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, r2
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_or16:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r0, [r0]
-; ARMEB-NEXT:    movw r2, #65535
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    orr r1, r1, r0
+; ARMEB:         ldrh r0, [r0, #2]
+; ARMEB-NEXT:    ldrh r1, [r1, #2]
+; ARMEB-NEXT:    orrs r0, r1, r0
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, r2
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_or16:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r0, [r0]
-; THUMB1-NEXT:    ldr r2, [r1]
+; THUMB1:         ldrh r0, [r0]
+; THUMB1-NEXT:    ldrh r2, [r1]
 ; THUMB1-NEXT:    orrs r2, r0
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #16
+; THUMB1-NEXT:    cmp r2, #0
 ; THUMB1-NEXT:    beq .LBB7_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB7_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_or16:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r0, [r0]
-; THUMB2-NEXT:    ldr r1, [r1]
+; THUMB2:         ldrh r0, [r0]
+; THUMB2-NEXT:    ldrh r1, [r1]
 ; THUMB2-NEXT:    orrs r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #16
 ; THUMB2-NEXT:    mov.w r0, #0
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                        i32* nocapture readonly %b) {
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %or = or i32 %1, %0
   %and = and i32 %or, 65535
   %cmp = icmp eq i32 %and, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_and8_short_short(i16* nocapture readonly %a,
+                                                    i16* nocapture readonly %b) {
 ; ARM-LABEL: cmp_and8_short_short:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldrh r1, [r1]
-; ARM-NEXT:    ldrh r0, [r0]
-; ARM-NEXT:    and r1, r0, r1
+; ARM:         ldrb r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    tst r2, r1
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_and8_short_short:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldrh r1, [r1]
-; ARMEB-NEXT:    ldrh r0, [r0]
-; ARMEB-NEXT:    and r1, r0, r1
+; ARMEB:         ldrb r2, [r0, #1]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
+; ARMEB-NEXT:    ldrb r1, [r1, #1]
+; ARMEB-NEXT:    tst r2, r1
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_and8_short_short:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldrh r1, [r1]
-; THUMB1-NEXT:    ldrh r2, [r0]
-; THUMB1-NEXT:    ands r2, r1
+; THUMB1:         ldrb r2, [r1]
+; THUMB1-NEXT:    ldrb r3, [r0]
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    tst r3, r2
 ; THUMB1-NEXT:    beq .LBB8_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB8_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_and8_short_short:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldrh r1, [r1]
-; THUMB2-NEXT:    ldrh r0, [r0]
-; THUMB2-NEXT:    ands r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrb r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrb r1, [r1]
+; THUMB2-NEXT:    tst r2, r1
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                    i16* nocapture readonly %b) {
 entry:
   %0 = load i16, i16* %a, align 2
   %1 = load i16, i16* %b, align 2
   %and3 = and i16 %0, 255
   %2 = and i16 %and3, %1
   %cmp = icmp eq i16 %2, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_and8_short_int(i16* nocapture readonly %a,
+                                                  i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_and8_short_int:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldrh r0, [r0]
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    and r1, r1, r0
+; ARM:         ldrb r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    tst r1, r2
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_and8_short_int:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldrh r0, [r0]
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    and r1, r1, r0
+; ARMEB:         ldrb r2, [r0, #1]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
+; ARMEB-NEXT:    ldrb r1, [r1, #3]
+; ARMEB-NEXT:    tst r1, r2
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_and8_short_int:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldrh r0, [r0]
-; THUMB1-NEXT:    ldr r2, [r1]
-; THUMB1-NEXT:    ands r2, r0
+; THUMB1:         ldrb r2, [r0]
+; THUMB1-NEXT:    ldrb r3, [r1]
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    tst r3, r2
 ; THUMB1-NEXT:    beq .LBB9_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB9_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_and8_short_int:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldrh r0, [r0]
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    ands r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrb r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrb r1, [r1]
+; THUMB2-NEXT:    tst r1, r2
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                  i32* nocapture readonly %b) {
 entry:
   %0 = load i16, i16* %a, align 2
   %1 = load i32, i32* %b, align 4
   %2 = and i16 %0, 255
   %and = zext i16 %2 to i32
   %and1 = and i32 %1, %and
   %cmp = icmp eq i32 %and1, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_and8_int_int(i32* nocapture readonly %a,
+                                                i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_and8_int_int:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    ldr r0, [r0]
-; ARM-NEXT:    and r1, r0, r1
+; ARM:         ldrb r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, #255
+; ARM-NEXT:    ldrb r1, [r1]
+; ARM-NEXT:    tst r2, r1
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_and8_int_int:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    ldr r0, [r0]
-; ARMEB-NEXT:    and r1, r0, r1
+; ARMEB:         ldrb r2, [r0, #3]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, #255
+; ARMEB-NEXT:    ldrb r1, [r1, #3]
+; ARMEB-NEXT:    tst r2, r1
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_and8_int_int:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r1, [r1]
-; THUMB1-NEXT:    ldr r2, [r0]
-; THUMB1-NEXT:    ands r2, r1
+; THUMB1:         ldrb r2, [r1]
+; THUMB1-NEXT:    ldrb r3, [r0]
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #24
+; THUMB1-NEXT:    tst r3, r2
 ; THUMB1-NEXT:    beq .LBB10_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB10_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_and8_int_int:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    ldr r0, [r0]
-; THUMB2-NEXT:    ands r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #24
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrb r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrb r1, [r1]
+; THUMB2-NEXT:    tst r2, r1
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                                i32* nocapture readonly %b) {
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %and = and i32 %0, 255
   %and1 = and i32 %and, %1
   %cmp = icmp eq i32 %and1, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc zeroext i1 @cmp_and16(i32* nocapture readonly %a,
+                                         i32* nocapture readonly %b) {
 ; ARM-LABEL: cmp_and16:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    movw r2, #65535
-; ARM-NEXT:    ldr r0, [r0]
-; ARM-NEXT:    and r1, r0, r1
+; ARM:         ldrh r2, [r0]
 ; ARM-NEXT:    mov r0, #0
-; ARM-NEXT:    tst r1, r2
+; ARM-NEXT:    ldrh r1, [r1]
+; ARM-NEXT:    tst r2, r1
 ; ARM-NEXT:    movweq r0, #1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: cmp_and16:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    movw r2, #65535
-; ARMEB-NEXT:    ldr r0, [r0]
-; ARMEB-NEXT:    and r1, r0, r1
+; ARMEB:         ldrh r2, [r0, #2]
 ; ARMEB-NEXT:    mov r0, #0
-; ARMEB-NEXT:    tst r1, r2
+; ARMEB-NEXT:    ldrh r1, [r1, #2]
+; ARMEB-NEXT:    tst r2, r1
 ; ARMEB-NEXT:    movweq r0, #1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: cmp_and16:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r1, [r1]
-; THUMB1-NEXT:    ldr r2, [r0]
-; THUMB1-NEXT:    ands r2, r1
+; THUMB1:         ldrh r2, [r1]
+; THUMB1-NEXT:    ldrh r3, [r0]
 ; THUMB1-NEXT:    movs r0, #1
 ; THUMB1-NEXT:    movs r1, #0
-; THUMB1-NEXT:    lsls r2, r2, #16
+; THUMB1-NEXT:    tst r3, r2
 ; THUMB1-NEXT:    beq .LBB11_2
 ; THUMB1-NEXT:  @ %bb.1: @ %entry
 ; THUMB1-NEXT:    mov r0, r1
 ; THUMB1-NEXT:  .LBB11_2: @ %entry
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: cmp_and16:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    ldr r0, [r0]
-; THUMB2-NEXT:    ands r0, r1
-; THUMB2-NEXT:    lsls r0, r0, #16
-; THUMB2-NEXT:    mov.w r0, #0
+; THUMB2:         ldrh r2, [r0]
+; THUMB2-NEXT:    movs r0, #0
+; THUMB2-NEXT:    ldrh r1, [r1]
+; THUMB2-NEXT:    tst r2, r1
 ; THUMB2-NEXT:    it eq
 ; THUMB2-NEXT:    moveq r0, #1
 ; THUMB2-NEXT:    bx lr
-                                         i32* nocapture readonly %b) {
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %and = and i32 %0, 65535
   %and1 = and i32 %and, %1
   %cmp = icmp eq i32 %and1, 0
   ret i1 %cmp
 }
 
 define arm_aapcscc i32 @add_and16(i32* nocapture readonly %a, i32 %y, i32 %z) {
 ; ARM-LABEL: add_and16:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r0, [r0]
-; ARM-NEXT:    add r1, r1, r2
+; ARM:         add r1, r1, r2
+; ARM-NEXT:    ldrh r0, [r0]
+; ARM-NEXT:    uxth r1, r1
 ; ARM-NEXT:    orr r0, r0, r1
-; ARM-NEXT:    uxth r0, r0
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: add_and16:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r0, [r0]
-; ARMEB-NEXT:    add r1, r1, r2
+; ARMEB:         add r1, r1, r2
+; ARMEB-NEXT:    ldrh r0, [r0, #2]
+; ARMEB-NEXT:    uxth r1, r1
 ; ARMEB-NEXT:    orr r0, r0, r1
-; ARMEB-NEXT:    uxth r0, r0
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: add_and16:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    adds r1, r1, r2
-; THUMB1-NEXT:    ldr r0, [r0]
+; THUMB1:         adds r1, r1, r2
+; THUMB1-NEXT:    uxth r1, r1
+; THUMB1-NEXT:    ldrh r0, [r0]
 ; THUMB1-NEXT:    orrs r0, r1
-; THUMB1-NEXT:    uxth r0, r0
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: add_and16:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r0, [r0]
-; THUMB2-NEXT:    add r1, r2
+; THUMB2:         add r1, r2
+; THUMB2-NEXT:    ldrh r0, [r0]
+; THUMB2-NEXT:    uxth r1, r1
 ; THUMB2-NEXT:    orrs r0, r1
-; THUMB2-NEXT:    uxth r0, r0
 ; THUMB2-NEXT:    bx lr
 entry:
   %x = load i32, i32* %a, align 4
   %add = add i32 %y, %z
   %or = or i32 %x, %add
   %and = and i32 %or, 65535
   ret i32 %and
 }
 
 define arm_aapcscc i32 @test1(i32* %a, i32* %b, i32 %x, i32 %y) {
 ; ARM-LABEL: test1:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    mul r2, r2, r3
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    ldr r0, [r0]
+; ARM:         mul r2, r2, r3
+; ARM-NEXT:    ldrh r1, [r1]
+; ARM-NEXT:    ldrh r0, [r0]
 ; ARM-NEXT:    eor r0, r0, r1
-; ARM-NEXT:    orr r0, r0, r2
-; ARM-NEXT:    uxth r0, r0
+; ARM-NEXT:    uxth r1, r2
+; ARM-NEXT:    orr r0, r0, r1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: test1:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    mul r2, r2, r3
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    ldr r0, [r0]
+; ARMEB:         mul r2, r2, r3
+; ARMEB-NEXT:    ldrh r1, [r1, #2]
+; ARMEB-NEXT:    ldrh r0, [r0, #2]
 ; ARMEB-NEXT:    eor r0, r0, r1
-; ARMEB-NEXT:    orr r0, r0, r2
-; ARMEB-NEXT:    uxth r0, r0
+; ARMEB-NEXT:    uxth r1, r2
+; ARMEB-NEXT:    orr r0, r0, r1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: test1:
-; THUMB1:       @ %bb.0: @ %entry
+; THUMB1:         ldrh r1, [r1]
+; THUMB1-NEXT:    ldrh r4, [r0]
+; THUMB1-NEXT:    eors r4, r1
 ; THUMB1-NEXT:    muls r2, r3, r2
-; THUMB1-NEXT:    ldr r1, [r1]
-; THUMB1-NEXT:    ldr r0, [r0]
-; THUMB1-NEXT:    eors r0, r1
-; THUMB1-NEXT:    orrs r0, r2
-; THUMB1-NEXT:    uxth r0, r0
-; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:    uxth r0, r2
+; THUMB1-NEXT:    orrs r0, r4
+; THUMB1-NEXT:    pop
 ;
 ; THUMB2-LABEL: test1:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    muls r2, r3, r2
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    ldr r0, [r0]
+; THUMB2:         ldrh r1, [r1]
+; THUMB2-NEXT:    ldrh r0, [r0]
 ; THUMB2-NEXT:    eors r0, r1
-; THUMB2-NEXT:    orrs r0, r2
-; THUMB2-NEXT:    uxth r0, r0
+; THUMB2-NEXT:    mul r1, r2, r3
+; THUMB2-NEXT:    uxth r1, r1
+; THUMB2-NEXT:    orrs r0, r1
 ; THUMB2-NEXT:    bx lr
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %mul = mul i32 %x, %y
   %xor = xor i32 %0, %1
   %or = or i32 %xor, %mul
   %and = and i32 %or, 65535
   ret i32 %and
 }
 
 define arm_aapcscc i32 @test2(i32* %a, i32* %b, i32 %x, i32 %y) {
 ; ARM-LABEL: test2:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r1, [r1]
+; ARM:         ldr r1, [r1]
 ; ARM-NEXT:    ldr r0, [r0]
 ; ARM-NEXT:    mul r1, r2, r1
 ; ARM-NEXT:    eor r0, r0, r3
 ; ARM-NEXT:    orr r0, r0, r1
 ; ARM-NEXT:    uxth r0, r0
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: test2:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r1, [r1]
+; ARMEB:         ldr r1, [r1]
 ; ARMEB-NEXT:    ldr r0, [r0]
 ; ARMEB-NEXT:    mul r1, r2, r1
 ; ARMEB-NEXT:    eor r0, r0, r3
 ; ARMEB-NEXT:    orr r0, r0, r1
 ; ARMEB-NEXT:    uxth r0, r0
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: test2:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r1, [r1]
+; THUMB1:         ldr r1, [r1]
 ; THUMB1-NEXT:    muls r1, r2, r1
 ; THUMB1-NEXT:    ldr r0, [r0]
 ; THUMB1-NEXT:    eors r0, r3
 ; THUMB1-NEXT:    orrs r0, r1
 ; THUMB1-NEXT:    uxth r0, r0
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: test2:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r1, [r1]
+; THUMB2:         ldr r1, [r1]
 ; THUMB2-NEXT:    ldr r0, [r0]
 ; THUMB2-NEXT:    muls r1, r2, r1
 ; THUMB2-NEXT:    eors r0, r3
 ; THUMB2-NEXT:    orrs r0, r1
 ; THUMB2-NEXT:    uxth r0, r0
 ; THUMB2-NEXT:    bx lr
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %mul = mul i32 %x, %1
   %xor = xor i32 %0, %y
   %or = or i32 %xor, %mul
   %and = and i32 %or, 65535
   ret i32 %and
 }
 
 define arm_aapcscc i32 @test3(i32* %a, i32* %b, i32 %x, i16* %y) {
 ; ARM-LABEL: test3:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r0, [r0]
+; ARM:         ldr r0, [r0]
 ; ARM-NEXT:    mul r1, r2, r0
 ; ARM-NEXT:    ldrh r2, [r3]
 ; ARM-NEXT:    eor r0, r0, r2
 ; ARM-NEXT:    orr r0, r0, r1
 ; ARM-NEXT:    uxth r0, r0
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: test3:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r0, [r0]
+; ARMEB:         ldr r0, [r0]
 ; ARMEB-NEXT:    mul r1, r2, r0
 ; ARMEB-NEXT:    ldrh r2, [r3]
 ; ARMEB-NEXT:    eor r0, r0, r2
 ; ARMEB-NEXT:    orr r0, r0, r1
 ; ARMEB-NEXT:    uxth r0, r0
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: test3:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r0, [r0]
+; THUMB1:         ldr r0, [r0]
 ; THUMB1-NEXT:    muls r2, r0, r2
 ; THUMB1-NEXT:    ldrh r1, [r3]
 ; THUMB1-NEXT:    eors r1, r0
 ; THUMB1-NEXT:    orrs r1, r2
 ; THUMB1-NEXT:    uxth r0, r1
 ; THUMB1-NEXT:    bx lr
 ;
 ; THUMB2-LABEL: test3:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r0, [r0]
+; THUMB2:         ldr r0, [r0]
 ; THUMB2-NEXT:    mul r1, r2, r0
 ; THUMB2-NEXT:    ldrh r2, [r3]
 ; THUMB2-NEXT:    eors r0, r2
 ; THUMB2-NEXT:    orrs r0, r1
 ; THUMB2-NEXT:    uxth r0, r0
 ; THUMB2-NEXT:    bx lr
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i16, i16* %y, align 4
   %2 = zext i16 %1 to i32
   %mul = mul i32 %x, %0
   %xor = xor i32 %0, %2
   %or = or i32 %xor, %mul
   %and = and i32 %or, 65535
   ret i32 %and
 }
 
 define arm_aapcscc i32 @test4(i32* %a, i32* %b, i32 %x, i32 %y) {
 ; ARM-LABEL: test4:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    mul r2, r2, r3
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    ldr r0, [r0]
+; ARM:         mul r2, r2, r3
+; ARM-NEXT:    ldrh r1, [r1]
+; ARM-NEXT:    ldrh r0, [r0]
 ; ARM-NEXT:    eor r0, r0, r1
-; ARM-NEXT:    orr r0, r0, r2
-; ARM-NEXT:    uxth r0, r0
+; ARM-NEXT:    uxth r1, r2
+; ARM-NEXT:    orr r0, r0, r1
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: test4:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    mul r2, r2, r3
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    ldr r0, [r0]
+; ARMEB:         mul r2, r2, r3
+; ARMEB-NEXT:    ldrh r1, [r1, #2]
+; ARMEB-NEXT:    ldrh r0, [r0, #2]
 ; ARMEB-NEXT:    eor r0, r0, r1
-; ARMEB-NEXT:    orr r0, r0, r2
-; ARMEB-NEXT:    uxth r0, r0
+; ARMEB-NEXT:    uxth r1, r2
+; ARMEB-NEXT:    orr r0, r0, r1
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: test4:
-; THUMB1:       @ %bb.0: @ %entry
+; THUMB1:         ldrh r1, [r1]
+; THUMB1-NEXT:    ldrh r4, [r0]
+; THUMB1-NEXT:    eors r4, r1
 ; THUMB1-NEXT:    muls r2, r3, r2
-; THUMB1-NEXT:    ldr r1, [r1]
-; THUMB1-NEXT:    ldr r0, [r0]
-; THUMB1-NEXT:    eors r0, r1
-; THUMB1-NEXT:    orrs r0, r2
-; THUMB1-NEXT:    uxth r0, r0
-; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:    uxth r0, r2
+; THUMB1-NEXT:    orrs r0, r4
+; THUMB1-NEXT:    pop
 ;
 ; THUMB2-LABEL: test4:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    muls r2, r3, r2
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    ldr r0, [r0]
+; THUMB2:         ldrh r1, [r1]
+; THUMB2-NEXT:    ldrh r0, [r0]
 ; THUMB2-NEXT:    eors r0, r1
-; THUMB2-NEXT:    orrs r0, r2
-; THUMB2-NEXT:    uxth r0, r0
+; THUMB2-NEXT:    mul r1, r2, r3
+; THUMB2-NEXT:    uxth r1, r1
+; THUMB2-NEXT:    orrs r0, r1
 ; THUMB2-NEXT:    bx lr
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %mul = mul i32 %x, %y
   %xor = xor i32 %0, %1
   %or = or i32 %xor, %mul
   %and = and i32 %or, 65535
   ret i32 %and
 }
 
 define arm_aapcscc i32 @test5(i32* %a, i32* %b, i32 %x, i16 zeroext %y) {
 ; ARM-LABEL: test5:
-; ARM:       @ %bb.0: @ %entry
-; ARM-NEXT:    ldr r1, [r1]
-; ARM-NEXT:    ldr r0, [r0]
+; ARM:         ldr r1, [r1]
+; ARM-NEXT:    ldrh r0, [r0]
 ; ARM-NEXT:    mul r1, r2, r1
 ; ARM-NEXT:    eor r0, r0, r3
+; ARM-NEXT:    uxth r1, r1
 ; ARM-NEXT:    orr r0, r0, r1
-; ARM-NEXT:    uxth r0, r0
 ; ARM-NEXT:    bx lr
 ;
 ; ARMEB-LABEL: test5:
-; ARMEB:       @ %bb.0: @ %entry
-; ARMEB-NEXT:    ldr r1, [r1]
-; ARMEB-NEXT:    ldr r0, [r0]
+; ARMEB:         ldr r1, [r1]
+; ARMEB-NEXT:    ldrh r0, [r0, #2]
 ; ARMEB-NEXT:    mul r1, r2, r1
 ; ARMEB-NEXT:    eor r0, r0, r3
+; ARMEB-NEXT:    uxth r1, r1
 ; ARMEB-NEXT:    orr r0, r0, r1
-; ARMEB-NEXT:    uxth r0, r0
 ; ARMEB-NEXT:    bx lr
 ;
 ; THUMB1-LABEL: test5:
-; THUMB1:       @ %bb.0: @ %entry
-; THUMB1-NEXT:    ldr r1, [r1]
-; THUMB1-NEXT:    muls r1, r2, r1
-; THUMB1-NEXT:    ldr r0, [r0]
-; THUMB1-NEXT:    eors r0, r3
-; THUMB1-NEXT:    orrs r0, r1
+; THUMB1:         ldrh r4, [r0]
+; THUMB1-NEXT:    eors r4, r3
+; THUMB1-NEXT:    ldr r0, [r1]
+; THUMB1-NEXT:    muls r0, r2, r0
 ; THUMB1-NEXT:    uxth r0, r0
-; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:    orrs r0, r4
+; THUMB1-NEXT:    pop
 ;
 ; THUMB2-LABEL: test5:
-; THUMB2:       @ %bb.0: @ %entry
-; THUMB2-NEXT:    ldr r1, [r1]
-; THUMB2-NEXT:    ldr r0, [r0]
+; THUMB2:         ldr r1, [r1]
+; THUMB2-NEXT:    ldrh r0, [r0]
 ; THUMB2-NEXT:    muls r1, r2, r1
 ; THUMB2-NEXT:    eors r0, r3
+; THUMB2-NEXT:    uxth r1, r1
 ; THUMB2-NEXT:    orrs r0, r1
-; THUMB2-NEXT:    uxth r0, r0
 ; THUMB2-NEXT:    bx lr
 entry:
   %0 = load i32, i32* %a, align 4
   %1 = load i32, i32* %b, align 4
   %mul = mul i32 %x, %1
   %ext = zext i16 %y to i32
   %xor = xor i32 %0, %ext
   %or = or i32 %xor, %mul
   %and = and i32 %or, 65535
   ret i32 %and
 }
+
+define arm_aapcscc i1 @test6(i8* %x, i8 %y, i8 %z) {
+; ARM-LABEL: test6:
+; ARM:      ldrb r0, [r0]
+; ARM:      uxtb r2, r2
+; ARM:      and r0, r0, r1
+; ARM:      uxtb r1, r0
+
+; ARMEB-LABEL: test6:
+; ARMEB:    ldrb r0, [r0]
+; ARMEB:    uxtb r2, r2
+; ARMEB:    and r0, r0, r1
+; ARMEB:    uxtb r1, r0
+
+; THUMB1-LABEL: test6:
+; THUMB1:    ldrb r0, [r0]
+; THUMB1:    ands r0, r1
+; THUMB1:    uxtb r3, r0
+; THUMB1:    uxtb r2, r2
+
+; THUMB2-LABEL: test6:
+; THUMB2:    ldrb r0, [r0]
+; THUMB2:    uxtb r2, r2
+; THUMB2:    ands r0, r1
+; THUMB2:    uxtb r1, r0
+entry:
+  %0 = load i8, i8* %x, align 4
+  %1 = and i8 %0, %y
+  %2 = icmp eq i8 %1, %z
+  ret i1 %2
+}
+
+define arm_aapcscc i1 @test7(i16* %x, i16 %y, i8 %z) {
+; ARM-LABEL: test7:
+; ARM:      ldrb r0, [r0]
+; ARM:      uxtb r2, r2
+; ARM:      and r1, r0, r1
+
+; ARMEB-LABEL: test7:
+; ARMEB:    ldrb r0, [r0, #1]
+; ARMEB:    uxtb r2, r2
+; ARMEB:    and r1, r0, r1
+
+; THUMB1-LABEL: test7:
+; THUMB1:    ldrb r3, [r0]
+; THUMB1:    ands r3, r1
+; THUMB1:    uxtb r2, r2
+
+; THUMB2-LABEL: test7:
+; THUMB2:    ldrb r0, [r0]
+; THUMB2:    uxtb r2, r2
+; THUMB2:    ands r1, r0
+entry:
+  %0 = load i16, i16* %x, align 4
+  %1 = and i16 %0, %y
+  %2 = trunc i16 %1 to i8
+  %3 = icmp eq i8 %2, %z
+  ret i1 %3
+}
+