[X86] Teach X86DAGToDAGISel::foldLoadStoreIntoMemOperand to handle loads in operand 1 of commutable operations.

[craig.topper]

Summary

Previously we only handled loads in operand 0, but nothing guarantees the load will be operand 0 for commutable operations.

[chandlerc]

LGTM, nice!

llvm/trunk/lib/Target/X86/X86ISelDAGToDAG.cpp

static bool hasNoCarryFlagUses(SDNode *N) {
   }
   return true;
 }
 
 /// Check whether or not the chain ending in StoreNode is suitable for doing
 /// the {load; op; store} to modify transformation.
 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
                                         SDValue StoredVal, SelectionDAG *CurDAG,
+                                        unsigned LoadOpNo,
                                         LoadSDNode *&LoadNode,
                                         SDValue &InputChain) {
   // Is the stored value result 0 of the operation?
   if (StoredVal.getResNo() != 0) return false;
 
   // Are there other uses of the operation other than the store?
   if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false;
 
   // Is the store non-extending and non-indexed?
   if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
     return false;
 
-  SDValue Load = StoredVal->getOperand(0);
+  SDValue Load = StoredVal->getOperand(LoadOpNo);
   // Is the stored value a non-extending and non-indexed load?
   if (!ISD::isNormalLoad(Load.getNode())) return false;
 
   // Return LoadNode by reference.
   LoadNode = cast<LoadSDNode>(Load);
 
   // Is store the only read of the loaded value?
   if (!Load.hasOneUse())
bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
 
   // Before we try to select anything, make sure this is memory operand size
   // and opcode we can handle. Note that this must match the code below that
   // actually lowers the opcodes.
   EVT MemVT = StoreNode->getMemoryVT();
   if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
       MemVT != MVT::i8)
     return false;
+
+  bool IsCommutable = false;
   switch (Opc) {
   default:
     return false;
   case X86ISD::INC:
   case X86ISD::DEC:
-  case X86ISD::ADD:
-  case X86ISD::ADC:
   case X86ISD::SUB:
   case X86ISD::SBB:
+    break;
+  case X86ISD::ADD:
+  case X86ISD::ADC:
   case X86ISD::AND:
   case X86ISD::OR:
   case X86ISD::XOR:
+    IsCommutable = true;
     break;
   }
 
+  unsigned LoadOpNo = 0;
   LoadSDNode *LoadNode = nullptr;
   SDValue InputChain;
-  if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
-                                   InputChain))
+  if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadOpNo,
+                                   LoadNode, InputChain)) {
+    if (!IsCommutable)
+      return false;
+
+    // This operation is commutable, try the other operand.
+    LoadOpNo = 1;
+    if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadOpNo,
+                                     LoadNode, InputChain))
       return false;
+  }
 
   SDValue Base, Scale, Index, Disp, Segment;
   if (!selectAddr(LoadNode, LoadNode->getBasePtr(), Base, Scale, Index, Disp,
                   Segment))
     return false;
 
   auto SelectOpcode = [&](unsigned Opc64, unsigned Opc32, unsigned Opc16,
                           unsigned Opc8) {
auto SelectImmOpcode = [SelectOpcode](unsigned Opc) {
         return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
                             X86::XOR8mi);
       default:
         llvm_unreachable("Invalid opcode!");
       }
     };
 
     unsigned NewOpc = SelectRegOpcode(Opc);
-    SDValue Operand = StoredVal->getOperand(1);
+    SDValue Operand = StoredVal->getOperand(1-LoadOpNo);
 
     // See if the operand is a constant that we can fold into an immediate
     // operand.
     if (auto *OperandC = dyn_cast<ConstantSDNode>(Operand)) {
       auto OperandV = OperandC->getAPIntValue();
 
       // Check if we can shrink the operand enough to fit in an immediate (or
       // fit into a smaller immediate) by negating it and switching the

llvm/trunk/test/CodeGen/X86/addcarry.ll

entry:
   %1 = add i128 %0, %b
   store i128 %1, i128* %a
   ret void
 }
 
 define void @add128_rmw2(i128 %a, i128* %b) nounwind {
 ; CHECK-LABEL: add128_rmw2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addq (%rdx), %rdi
+; CHECK-NEXT:    addq %rdi, (%rdx)
 ; CHECK-NEXT:    adcq %rsi, 8(%rdx)
-; CHECK-NEXT:    movq %rdi, (%rdx)
 ; CHECK-NEXT:    retq
 entry:
   %0 = load i128, i128* %b
   %1 = add i128 %a, %0
   store i128 %1, i128* %b
   ret void
 }
 
entry:
   %1 = add i256 %0, %b
   store i256 %1, i256* %a
   ret void
 }
 
 define void @add256_rmw2(i256 %a, i256* %b) nounwind {
 ; CHECK-LABEL: add256_rmw2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addq (%r8), %rdi
-; CHECK-NEXT:    adcq 8(%r8), %rsi
-; CHECK-NEXT:    adcq 16(%r8), %rdx
+; CHECK-NEXT:    addq %rdi, (%r8)
+; CHECK-NEXT:    adcq %rsi, 8(%r8)
+; CHECK-NEXT:    adcq %rdx, 16(%r8)
 ; CHECK-NEXT:    adcq %rcx, 24(%r8)
-; CHECK-NEXT:    movq %rdi, (%r8)
-; CHECK-NEXT:    movq %rsi, 8(%r8)
-; CHECK-NEXT:    movq %rdx, 16(%r8)
 ; CHECK-NEXT:    retq
 entry:
   %0 = load i256, i256* %b
   %1 = add i256 %a, %0
   store i256 %1, i256* %b
   ret void
 }