[ARM] Add ARMISD::VLD1DUP to match vld1_dup more consistently

[efriedma]

Summary

Currently, there are substantial problems forming vld1_dup even if the
VDUP survives legalization. The lack of an actual node
leads to terrible results: not only can we not form post-increment vld1_dup
instructions, but we form scalar pre-increment and post-increment
loads which force the loaded value into a GPR. This patch fixes that
by combining the vdup+load into an ARMISD node before DAGCombine
messes it up.

Also includes a crash fix for vld2_dup (see testcase @vld2dupi8_postinc_variable).

Not sure who to ask to review this...

[efriedma]

Minor change I forgot to include in the diff.

[rengolin]

Hi Eli,

Nice patch! I only have a few nits inline, but LGTM. Thanks!

--renato

lib/Target/ARM/ARMISelLowering.cpp

10697

Maybe adding a one-line comment as to which pattern you want to combine here, as it isn't obvious.

test/CodeGen/ARM/vlddup.ll

168

nit: clean up the "sroa" names to make it slightly more readable.

[efriedma]

Reverted in 289923.

lib/Target/ARM/ARMISelDAGToDAG.cpp

private:
   /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
   /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
   /// load/store of D registers and Q registers.
   void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
                        unsigned NumVecs, const uint16_t *DOpcodes,
                        const uint16_t *QOpcodes);
 
   /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
-  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
+  /// should be 1, 2, 3 or 4.  The opcode array specifies the instructions used
   /// for loading D registers.  (Q registers are not supported.)
   void SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
-                    const uint16_t *Opcodes);
+                    const uint16_t *DOpcodes,
+                    const uint16_t *QOpcodes = nullptr);
 
   /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
   /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
   /// generated to force the table registers to be consecutive.
   void SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
 
   /// Try to select SBFX/UBFX instructions for ARM.
   bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
static bool isVLDfixed(unsigned Opc)
   case ARM::VLD1d32wb_fixed : return true;
   case ARM::VLD1d64wb_fixed : return true;
   case ARM::VLD1d64TPseudoWB_fixed : return true;
   case ARM::VLD1d64QPseudoWB_fixed : return true;
   case ARM::VLD1q8wb_fixed : return true;
   case ARM::VLD1q16wb_fixed : return true;
   case ARM::VLD1q32wb_fixed : return true;
   case ARM::VLD1q64wb_fixed : return true;
+  case ARM::VLD1DUPd8wb_fixed : return true;
+  case ARM::VLD1DUPd16wb_fixed : return true;
+  case ARM::VLD1DUPd32wb_fixed : return true;
+  case ARM::VLD1DUPq8wb_fixed : return true;
+  case ARM::VLD1DUPq16wb_fixed : return true;
+  case ARM::VLD1DUPq32wb_fixed : return true;
   case ARM::VLD2d8wb_fixed : return true;
   case ARM::VLD2d16wb_fixed : return true;
   case ARM::VLD2d32wb_fixed : return true;
   case ARM::VLD2q8PseudoWB_fixed : return true;
   case ARM::VLD2q16PseudoWB_fixed : return true;
   case ARM::VLD2q32PseudoWB_fixed : return true;
   case ARM::VLD2DUPd8wb_fixed : return true;
   case ARM::VLD2DUPd16wb_fixed : return true;
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
   case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
   case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
   case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
   case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
   case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
   case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
   case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
   case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
+  case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
+  case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
+  case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
+  case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
+  case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
+  case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
 
   case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
   case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
   case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
   case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
   case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
   case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
   case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
ReplaceUses(SDValue(N, Vec),
                 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
   ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
   if (isUpdating)
     ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
   CurDAG->RemoveDeadNode(N);
 }
 
 void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
-                                   const uint16_t *Opcodes) {
-  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
+                                   const uint16_t *DOpcodes,
+                                   const uint16_t *QOpcodes) {
+  assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
   SDLoc dl(N);
 
   SDValue MemAddr, Align;
   if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
     return;
 
   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
   MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
if (Alignment < 8 && Alignment < NumBytes)
       Alignment = 0;
     // Alignment must be a power of two; make sure of that.
     Alignment = (Alignment & -Alignment);
     if (Alignment == 1)
       Alignment = 0;
   }
   Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
 
-  unsigned OpcodeIndex;
+  unsigned Opc;
   switch (VT.getSimpleVT().SimpleTy) {
   default: llvm_unreachable("unhandled vld-dup type");
-  case MVT::v8i8:  OpcodeIndex = 0; break;
-  case MVT::v4i16: OpcodeIndex = 1; break;
+  case MVT::v8i8:  Opc = DOpcodes[0]; break;
+  case MVT::v16i8:  Opc = QOpcodes[0]; break;
+  case MVT::v4i16: Opc = DOpcodes[1]; break;
+  case MVT::v8i16: Opc = QOpcodes[1]; break;
   case MVT::v2f32:
-  case MVT::v2i32: OpcodeIndex = 2; break;
+  case MVT::v2i32: Opc = DOpcodes[2]; break;
+  case MVT::v4f32:
+  case MVT::v4i32: Opc = QOpcodes[2]; break;
   }
 
   SDValue Pred = getAL(CurDAG, dl);
   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
-  SDValue SuperReg;
-  unsigned Opc = Opcodes[OpcodeIndex];
   SmallVector<SDValue, 6> Ops;
   Ops.push_back(MemAddr);
   Ops.push_back(Align);
   if (isUpdating) {
     // fixed-stride update instructions don't have an explicit writeback
     // operand. It's implicit in the opcode itself.
     SDValue Inc = N->getOperand(2);
+    if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
+      Opc = getVLDSTRegisterUpdateOpcode(Opc);
     if (!isa<ConstantSDNode>(Inc.getNode()))
       Ops.push_back(Inc);
     // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
     else if (NumVecs > 2)
       Ops.push_back(Reg0);
   }
   Ops.push_back(Pred);
   Ops.push_back(Reg0);
   Ops.push_back(Chain);
 
   unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
   std::vector<EVT> ResTys;
   ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
   if (isUpdating)
     ResTys.push_back(MVT::i32);
   ResTys.push_back(MVT::Other);
   SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
   cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
-  SuperReg = SDValue(VLdDup, 0);
 
   // Extract the subregisters.
+  if (NumVecs == 1) {
+    ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
+  } else {
+    SDValue SuperReg = SDValue(VLdDup, 0);
     static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
     unsigned SubIdx = ARM::dsub_0;
     for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
       ReplaceUses(SDValue(N, Vec),
                   CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
+  }
   ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
   if (isUpdating)
     ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
   CurDAG->RemoveDeadNode(N);
 }
 
 void ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
                                  unsigned Opc) {
case ARMISD::BUILD_VECTOR: {
     }
     assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
     ReplaceNode(N,
                 createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
                                     N->getOperand(2), N->getOperand(3)));
     return;
   }
 
+  case ARMISD::VLD1DUP: {
+    static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
+                                         ARM::VLD1DUPd32 };
+    static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
+                                         ARM::VLD1DUPq32 };
+    SelectVLDDup(N, false, 1, DOpcodes, QOpcodes);
+    return;
+  }
+
   case ARMISD::VLD2DUP: {
     static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
                                         ARM::VLD2DUPd32 };
     SelectVLDDup(N, false, 2, Opcodes);
     return;
   }
 
   case ARMISD::VLD3DUP: {
     static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
                                         ARM::VLD3DUPd16Pseudo,
                                         ARM::VLD3DUPd32Pseudo };
     SelectVLDDup(N, false, 3, Opcodes);
     return;
   }
 
   case ARMISD::VLD4DUP: {
     static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
                                         ARM::VLD4DUPd16Pseudo,
                                         ARM::VLD4DUPd32Pseudo };
     SelectVLDDup(N, false, 4, Opcodes);
     return;
   }
 
+  case ARMISD::VLD1DUP_UPD: {
+    static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
+                                         ARM::VLD1DUPd16wb_fixed,
+                                         ARM::VLD1DUPd32wb_fixed };
+    static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
+                                         ARM::VLD1DUPq16wb_fixed,
+                                         ARM::VLD1DUPq32wb_fixed };
+    SelectVLDDup(N, true, 1, DOpcodes, QOpcodes);
+    return;
+  }
+
   case ARMISD::VLD2DUP_UPD: {
     static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
                                         ARM::VLD2DUPd16wb_fixed,
                                         ARM::VLD2DUPd32wb_fixed };
     SelectVLDDup(N, true, 2, Opcodes);
     return;
   }
 

lib/Target/ARM/ARMISelLowering.h

enum NodeType : unsigned {
       // Vector bitwise select
       VBSL,
 
       // Pseudo-instruction representing a memory copy using ldm/stm
       // instructions.
       MEMCPY,
 
       // Vector load N-element structure to all lanes:
-      VLD2DUP = ISD::FIRST_TARGET_MEMORY_OPCODE,
+      VLD1DUP = ISD::FIRST_TARGET_MEMORY_OPCODE,
+      VLD2DUP,
       VLD3DUP,
       VLD4DUP,
 
       // NEON loads with post-increment base updates:
       VLD1_UPD,
       VLD2_UPD,
       VLD3_UPD,
       VLD4_UPD,
       VLD2LN_UPD,
       VLD3LN_UPD,
       VLD4LN_UPD,
+      VLD1DUP_UPD,
       VLD2DUP_UPD,
       VLD3DUP_UPD,
       VLD4DUP_UPD,
 
       // NEON stores with post-increment base updates:
       VST1_UPD,
       VST2_UPD,
       VST3_UPD,

lib/Target/ARM/ARMISelLowering.cpp

const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::UMLAL:         return "ARMISD::UMLAL";
   case ARMISD::SMLAL:         return "ARMISD::SMLAL";
   case ARMISD::BUILD_VECTOR:  return "ARMISD::BUILD_VECTOR";
   case ARMISD::BFI:           return "ARMISD::BFI";
   case ARMISD::VORRIMM:       return "ARMISD::VORRIMM";
   case ARMISD::VBICIMM:       return "ARMISD::VBICIMM";
   case ARMISD::VBSL:          return "ARMISD::VBSL";
   case ARMISD::MEMCPY:        return "ARMISD::MEMCPY";
+  case ARMISD::VLD1DUP:       return "ARMISD::VLD1DUP";
   case ARMISD::VLD2DUP:       return "ARMISD::VLD2DUP";
   case ARMISD::VLD3DUP:       return "ARMISD::VLD3DUP";
   case ARMISD::VLD4DUP:       return "ARMISD::VLD4DUP";
   case ARMISD::VLD1_UPD:      return "ARMISD::VLD1_UPD";
   case ARMISD::VLD2_UPD:      return "ARMISD::VLD2_UPD";
   case ARMISD::VLD3_UPD:      return "ARMISD::VLD3_UPD";
   case ARMISD::VLD4_UPD:      return "ARMISD::VLD4_UPD";
   case ARMISD::VLD2LN_UPD:    return "ARMISD::VLD2LN_UPD";
   case ARMISD::VLD3LN_UPD:    return "ARMISD::VLD3LN_UPD";
   case ARMISD::VLD4LN_UPD:    return "ARMISD::VLD4LN_UPD";
+  case ARMISD::VLD1DUP_UPD:   return "ARMISD::VLD1DUP_UPD";
   case ARMISD::VLD2DUP_UPD:   return "ARMISD::VLD2DUP_UPD";
   case ARMISD::VLD3DUP_UPD:   return "ARMISD::VLD3DUP_UPD";
   case ARMISD::VLD4DUP_UPD:   return "ARMISD::VLD4DUP_UPD";
   case ARMISD::VST1_UPD:      return "ARMISD::VST1_UPD";
   case ARMISD::VST2_UPD:      return "ARMISD::VST2_UPD";
   case ARMISD::VST3_UPD:      return "ARMISD::VST3_UPD";
   case ARMISD::VST4_UPD:      return "ARMISD::VST4_UPD";
   case ARMISD::VST2LN_UPD:    return "ARMISD::VST2LN_UPD";
if (isIntrinsic) {
         NumVecs = 3; isLoadOp = false; isLaneOp = true; break;
       case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD;
         NumVecs = 4; isLoadOp = false; isLaneOp = true; break;
       }
     } else {
       isLaneOp = true;
       switch (N->getOpcode()) {
       default: llvm_unreachable("unexpected opcode for Neon base update");
+      case ARMISD::VLD1DUP: NewOpc = ARMISD::VLD1DUP_UPD; NumVecs = 1; break;
       case ARMISD::VLD2DUP: NewOpc = ARMISD::VLD2DUP_UPD; NumVecs = 2; break;
       case ARMISD::VLD3DUP: NewOpc = ARMISD::VLD3DUP_UPD; NumVecs = 3; break;
       case ARMISD::VLD4DUP: NewOpc = ARMISD::VLD4DUP_UPD; NumVecs = 4; break;
       case ISD::LOAD:       NewOpc = ARMISD::VLD1_UPD;
         NumVecs = 1; isLaneOp = false; break;
       case ISD::STORE:      NewOpc = ARMISD::VST1_UPD;
         NumVecs = 1; isLaneOp = false; isLoadOp = false; break;
       }
for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
 
     // If this is a non-standard-aligned STORE, the penultimate operand is the
     // stored value.  Bitcast it to the aligned type.
     if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
       SDValue &StVal = Ops[Ops.size()-2];
       StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
     }
 
-    SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys,
-                                           Ops, AlignedVecTy,
+    EVT LoadVT = isLaneOp ? AlignedVecTy.getVectorElementType() : AlignedVecTy;
+    SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT,
                                            MemN->getMemOperand());
 
     // Update the uses.
     SmallVector<SDValue, 5> NewResults;
     for (unsigned i = 0; i < NumResultVecs; ++i)
       NewResults.push_back(SDValue(UpdN.getNode(), i));
 
     // If this is an non-standard-aligned LOAD, the first result is the loaded
if (ARM_AM::decodeNEONModImm(Imm, EltBits) == 0)
     EltSize = 8;
   EVT VT = N->getValueType(0);
   if (EltSize > VT.getScalarSizeInBits())
     return SDValue();
 
   return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
 }
 
+/// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
+static SDValue PerformVDUPCombine(SDNode *N,
+                                  TargetLowering::DAGCombinerInfo &DCI) {
+  SelectionDAG &DAG = DCI.DAG;
+  SDValue Op = N->getOperand(0);
+
+  if (Op.getOpcode() == ISD::LOAD && Op.hasOneUse()) {
+    LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
+    SDValue Ops[] = { LD->getOperand(0), LD->getOperand(1),
+                      DAG.getConstant(LD->getAlignment(), SDLoc(N), MVT::i32) };
+    SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other);
+    SDValue VLDDup = DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys,
+                                             Ops, LD->getMemoryVT(),
+                                             LD->getMemOperand());
+    DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1));
+    return VLDDup;
+  }
+
+  return SDValue();
+}
+
 static SDValue PerformLOADCombine(SDNode *N,
                                   TargetLowering::DAGCombinerInfo &DCI) {
   EVT VT = N->getValueType(0);
 
   // If this is a legal vector load, try to combine it into a VLD1_UPD.
   if (ISD::isNormalLoad(N) && VT.isVector() &&
       DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return CombineBaseUpdate(N, DCI);
SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ARMISD::BFI:     return PerformBFICombine(N, DCI);
   case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
   case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
   case ISD::STORE:      return PerformSTORECombine(N, DCI);
   case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
   case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
   case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
   case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI);
+  case ARMISD::VDUP: return PerformVDUPCombine(N, DCI);
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT:
     return PerformVCVTCombine(N, DCI.DAG, Subtarget);
   case ISD::FDIV:
     return PerformVDIVCombine(N, DCI.DAG, Subtarget);
   case ISD::INTRINSIC_WO_CHAIN: return PerformIntrinsicCombine(N, DCI.DAG);
   case ISD::SHL:
   case ISD::SRA:
   case ISD::SRL:        return PerformShiftCombine(N, DCI.DAG, Subtarget);
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
   case ISD::ANY_EXTEND: return PerformExtendCombine(N, DCI.DAG, Subtarget);
   case ARMISD::CMOV: return PerformCMOVCombine(N, DCI.DAG);
   case ARMISD::BRCOND: return PerformBRCONDCombine(N, DCI.DAG);
   case ISD::LOAD:       return PerformLOADCombine(N, DCI);
+  case ARMISD::VLD1DUP:
   case ARMISD::VLD2DUP:
   case ARMISD::VLD3DUP:
   case ARMISD::VLD4DUP:
     return PerformVLDCombine(N, DCI);
   case ARMISD::BUILD_VECTOR:
     return PerformARMBUILD_VECTORCombine(N, DCI);
   case ISD::INTRINSIC_VOID:
   case ISD::INTRINSIC_W_CHAIN:

test/CodeGen/ARM/vlddup.ll

 ; RUN: llc -mtriple=arm-eabi -float-abi=soft -mattr=+neon %s -o - | FileCheck %s
 
 define <8 x i8> @vld1dupi8(i8* %A) nounwind {
 ;CHECK-LABEL: vld1dupi8:
 ;Check the (default) alignment value.
 ;CHECK: vld1.8 {d16[]}, [r0]
   %tmp1 = load i8, i8* %A, align 8
   %tmp2 = insertelement <8 x i8> undef, i8 %tmp1, i32 0
   %tmp3 = shufflevector <8 x i8> %tmp2, <8 x i8> undef, <8 x i32> zeroinitializer
         ret <8 x i8> %tmp3
 }
 
+define <8 x i8> @vld1dupi8_preinc(i8** noalias nocapture %a, i32 %b) nounwind {
+entry:
+;CHECK-LABEL: vld1dupi8_preinc:
+;CHECK: vld1.8 {d16[]}, [r1]
+  %0 = load i8*, i8** %a, align 4
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 %b
+  %1 = load i8, i8* %add.ptr, align 1
+  %2 = insertelement <8 x i8> undef, i8 %1, i32 0
+  %lane = shufflevector <8 x i8> %2, <8 x i8> undef, <8 x i32> zeroinitializer
+  store i8* %add.ptr, i8** %a, align 4
+  ret <8 x i8> %lane
+}
+
+define <8 x i8> @vld1dupi8_postinc_fixed(i8** noalias nocapture %a) nounwind {
+entry:
+;CHECK-LABEL: vld1dupi8_postinc_fixed:
+;CHECK: vld1.8 {d16[]}, [r1]!
+  %0 = load i8*, i8** %a, align 4
+  %1 = load i8, i8* %0, align 1
+  %2 = insertelement <8 x i8> undef, i8 %1, i32 0
+  %lane = shufflevector <8 x i8> %2, <8 x i8> undef, <8 x i32> zeroinitializer
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 1
+  store i8* %add.ptr, i8** %a, align 4
+  ret <8 x i8> %lane
+}
+
+define <8 x i8> @vld1dupi8_postinc_register(i8** noalias nocapture %a, i32 %n) nounwind {
+entry:
+;CHECK-LABEL: vld1dupi8_postinc_register:
+;CHECK: vld1.8 {d16[]}, [r2], r1
+  %0 = load i8*, i8** %a, align 4
+  %1 = load i8, i8* %0, align 1
+  %2 = insertelement <8 x i8> undef, i8 %1, i32 0
+  %lane = shufflevector <8 x i8> %2, <8 x i8> undef, <8 x i32> zeroinitializer
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 %n
+  store i8* %add.ptr, i8** %a, align 4
+  ret <8 x i8> %lane
+}
+
+define <16 x i8> @vld1dupqi8_preinc(i8** noalias nocapture %a, i32 %b) nounwind {
+entry:
+;CHECK-LABEL: vld1dupqi8_preinc:
+;CHECK: vld1.8 {d16[], d17[]}, [r1]
+  %0 = load i8*, i8** %a, align 4
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 %b
+  %1 = load i8, i8* %add.ptr, align 1
+  %2 = insertelement <16 x i8> undef, i8 %1, i32 0
+  %lane = shufflevector <16 x i8> %2, <16 x i8> undef, <16 x i32> zeroinitializer
+  store i8* %add.ptr, i8** %a, align 4
+  ret <16 x i8> %lane
+}
+
+define <16 x i8> @vld1dupqi8_postinc_fixed(i8** noalias nocapture %a) nounwind {
+entry:
+;CHECK-LABEL: vld1dupqi8_postinc_fixed:
+;CHECK: vld1.8 {d16[], d17[]}, [r1]!
+  %0 = load i8*, i8** %a, align 4
+  %1 = load i8, i8* %0, align 1
+  %2 = insertelement <16 x i8> undef, i8 %1, i32 0
+  %lane = shufflevector <16 x i8> %2, <16 x i8> undef, <16 x i32> zeroinitializer
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 1
+  store i8* %add.ptr, i8** %a, align 4
+  ret <16 x i8> %lane
+}
+
+define <16 x i8> @vld1dupqi8_postinc_register(i8** noalias nocapture %a, i32 %n) nounwind {
+entry:
+;CHECK-LABEL: vld1dupqi8_postinc_register:
+;CHECK: vld1.8 {d16[], d17[]}, [r2], r1
+  %0 = load i8*, i8** %a, align 4
+  %1 = load i8, i8* %0, align 1
+  %2 = insertelement <16 x i8> undef, i8 %1, i32 0
+  %lane = shufflevector <16 x i8> %2, <16 x i8> undef, <16 x i32> zeroinitializer
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 %n
+  store i8* %add.ptr, i8** %a, align 4
+  ret <16 x i8> %lane
+}
+
 define <4 x i16> @vld1dupi16(i16* %A) nounwind {
 ;CHECK-LABEL: vld1dupi16:
 ;Check the alignment value.  Max for this instruction is 16 bits:
 ;CHECK: vld1.16 {d16[]}, [r0:16]
   %tmp1 = load i16, i16* %A, align 8
   %tmp2 = insertelement <4 x i16> undef, i16 %tmp1, i32 0
   %tmp3 = shufflevector <4 x i16> %tmp2, <4 x i16> undef, <4 x i32> zeroinitializer
         ret <4 x i16> %tmp3
;CHECK: vld2.8 {d16[], d17[]}, [r0]
   %tmp1 = extractvalue %struct.__neon_int8x8x2_t %tmp0, 0
   %tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> zeroinitializer
   %tmp3 = extractvalue %struct.__neon_int8x8x2_t %tmp0, 1
   %tmp4 = shufflevector <8 x i8> %tmp3, <8 x i8> undef, <8 x i32> zeroinitializer
         %tmp5 = add <8 x i8> %tmp2, %tmp4
         ret <8 x i8> %tmp5
 }
 
+define void @vld2dupi8_preinc(%struct.__neon_int8x8x2_t* noalias nocapture sret %agg.result, i8** noalias nocapture %a, i32 %b) nounwind {
+;CHECK-LABEL: vld2dupi8_preinc:
+;CHECK: vld2.8 {d16[], d17[]}, [r2]
+entry:
+  %0 = load i8*, i8** %a, align 4
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 %b
+  %vld_dup = tail call %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2lane.v8i8.p0i8(i8* %add.ptr, <8 x i8> undef, <8 x i8> undef, i32 0, i32 1)
+  %1 = extractvalue %struct.__neon_int8x8x2_t %vld_dup, 0
+  %lane = shufflevector <8 x i8> %1, <8 x i8> undef, <8 x i32> zeroinitializer
+  %2 = extractvalue %struct.__neon_int8x8x2_t %vld_dup, 1
+  %lane1 = shufflevector <8 x i8> %2, <8 x i8> undef, <8 x i32> zeroinitializer
+  store i8* %add.ptr, i8** %a, align 4
+  %r.sroa.0.0..sroa_idx8 = getelementptr inbounds %struct.__neon_int8x8x2_t, %struct.__neon_int8x8x2_t* %agg.result, i32 0, i32 0
+  store <8 x i8> %lane, <8 x i8>* %r.sroa.0.0..sroa_idx8, align 8
+  %r.sroa.4.0..sroa_idx11 = getelementptr inbounds %struct.__neon_int8x8x2_t, %struct.__neon_int8x8x2_t* %agg.result, i32 0, i32 1
+  store <8 x i8> %lane1, <8 x i8>* %r.sroa.4.0..sroa_idx11, align 8
+  ret void
+}
+
+define void @vld2dupi8_postinc_fixed(%struct.__neon_int8x8x2_t* noalias nocapture sret %agg.result, i8** noalias nocapture %a) nounwind {
+entry:
+;CHECK-LABEL: vld2dupi8_postinc_fixed:
+;CHECK: vld2.8 {d16[], d17[]}, [r2]!
+  %0 = load i8*, i8** %a, align 4
+  %vld_dup = tail call %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2lane.v8i8.p0i8(i8* %0, <8 x i8> undef, <8 x i8> undef, i32 0, i32 1)
+  %1 = extractvalue %struct.__neon_int8x8x2_t %vld_dup, 0
+  %lane = shufflevector <8 x i8> %1, <8 x i8> undef, <8 x i32> zeroinitializer
+  %2 = extractvalue %struct.__neon_int8x8x2_t %vld_dup, 1
+  %lane1 = shufflevector <8 x i8> %2, <8 x i8> undef, <8 x i32> zeroinitializer
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 2
+  store i8* %add.ptr, i8** %a, align 4
+  %r.sroa.0.0..sroa_idx7 = getelementptr inbounds %struct.__neon_int8x8x2_t, %struct.__neon_int8x8x2_t* %agg.result, i32 0, i32 0
+  store <8 x i8> %lane, <8 x i8>* %r.sroa.0.0..sroa_idx7, align 8
+  %r.sroa.4.0..sroa_idx10 = getelementptr inbounds %struct.__neon_int8x8x2_t, %struct.__neon_int8x8x2_t* %agg.result, i32 0, i32 1
+  store <8 x i8> %lane1, <8 x i8>* %r.sroa.4.0..sroa_idx10, align 8
+  ret void
+}
+
+define void @vld2dupi8_postinc_variable(%struct.__neon_int8x8x2_t* noalias nocapture sret %agg.result, i8** noalias nocapture %a, i32 %n) nounwind {
+entry:
+;CHECK-LABEL: vld2dupi8_postinc_variable:
+;CHECK: vld2.8 {d16[], d17[]}, [r3], r2
+  %0 = load i8*, i8** %a, align 4
+  %vld_dup = tail call %struct.__neon_int8x8x2_t @llvm.arm.neon.vld2lane.v8i8.p0i8(i8* %0, <8 x i8> undef, <8 x i8> undef, i32 0, i32 1)
+  %1 = extractvalue %struct.__neon_int8x8x2_t %vld_dup, 0
+  %lane = shufflevector <8 x i8> %1, <8 x i8> undef, <8 x i32> zeroinitializer
+  %2 = extractvalue %struct.__neon_int8x8x2_t %vld_dup, 1
+  %lane1 = shufflevector <8 x i8> %2, <8 x i8> undef, <8 x i32> zeroinitializer
+  %add.ptr = getelementptr inbounds i8, i8* %0, i32 %n
+  store i8* %add.ptr, i8** %a, align 4
+  %r.sroa.0.0..sroa_idx7 = getelementptr inbounds %struct.__neon_int8x8x2_t, %struct.__neon_int8x8x2_t* %agg.result, i32 0, i32 0
+  store <8 x i8> %lane, <8 x i8>* %r.sroa.0.0..sroa_idx7, align 8
+  %r.sroa.4.0..sroa_idx10 = getelementptr inbounds %struct.__neon_int8x8x2_t, %struct.__neon_int8x8x2_t* %agg.result, i32 0, i32 1
+  store <8 x i8> %lane1, <8 x i8>* %r.sroa.4.0..sroa_idx10, align 8
+  ret void
+}
+
 define <4 x i16> @vld2dupi16(i8* %A) nounwind {
 ;CHECK-LABEL: vld2dupi16:
 ;Check that a power-of-two alignment smaller than the total size of the memory
 ;being loaded is ignored.
 ;CHECK: vld2.16 {d16[], d17[]}, [r0]
   %tmp0 = tail call %struct.__neon_int4x16x2_t @llvm.arm.neon.vld2lane.v4i16.p0i8(i8* %A, <4 x i16> undef, <4 x i16> undef, i32 0, i32 2)
   %tmp1 = extractvalue %struct.__neon_int4x16x2_t %tmp0, 0
   %tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer

test/CodeGen/ARM/vmul.ll

entry:
   %0 = zext <4 x i8> %wide.load283.i to <4 x i32>
   %1 = sub nsw <4 x i32> %0, %a
   %2 = mul nsw <4 x i32> %1, %1
   %predphi290.v.i = select <4 x i1> undef, <4 x i32> undef, <4 x i32> %2
   store <4 x i32> %predphi290.v.i, <4 x i32>* undef, align 4
   ret void
 }
 
-define void @foo(<4 x float> * %a, <4 x float>* nocapture %dst, float* nocapture readonly %src) nounwind {
-;   Look for doing a normal scalar FP load rather than an to-all-lanes load.
-;   e.g., "ldr s0, [r2]" rathern than "vld1.32  {d18[], d19[]}, [r2:32]"
-;   Then check that the vector multiply has folded the splat to all lanes
-;   and used a vector * scalar instruction.
-; CHECK: vldr  {{s[0-9]+}}, [r2]
+define void @fmul_splat(<4 x float> * %a, <4 x float>* nocapture %dst, float %tmp) nounwind {
+; Look for a scalar float rather than a splat, then a vector*scalar multiply.
+; CHECK: vmov s0, r2
 ; CHECK: vmul.f32  q8, q8, d0[0]
+  %tmp5 = load <4 x float>, <4 x float>* %a, align 4
+  %tmp6 = insertelement <4 x float> undef, float %tmp, i32 0
+  %tmp7 = insertelement <4 x float> %tmp6, float %tmp, i32 1
+  %tmp8 = insertelement <4 x float> %tmp7, float %tmp, i32 2
+  %tmp9 = insertelement <4 x float> %tmp8, float %tmp, i32 3
+  %tmp10 = fmul <4 x float> %tmp9, %tmp5
+  store <4 x float> %tmp10, <4 x float>* %dst, align 4
+  ret void
+}
+
+define void @fmul_splat_load(<4 x float> * %a, <4 x float>* nocapture %dst, float* nocapture readonly %src) nounwind {
+; Look for doing a normal scalar FP load rather than an to-all-lanes load,
+; then a vector*scalar multiply.
+; FIXME: Temporarily broken due to splat representation changes.
+; CHECK: vld1.32 {d18[], d19[]}, [r2:32]
+; CHECK: vmul.f32  q8, q9, q8
   %tmp = load float, float* %src, align 4
   %tmp5 = load <4 x float>, <4 x float>* %a, align 4
   %tmp6 = insertelement <4 x float> undef, float %tmp, i32 0
   %tmp7 = insertelement <4 x float> %tmp6, float %tmp, i32 1
   %tmp8 = insertelement <4 x float> %tmp7, float %tmp, i32 2
   %tmp9 = insertelement <4 x float> %tmp8, float %tmp, i32 3
   %tmp10 = fmul <4 x float> %tmp9, %tmp5
   store <4 x float> %tmp10, <4 x float>* %dst, align 4
   ret void
 }