[VE] Split v512.32 load store into interleaved v256.32 ops

[simoll]

Summary

Without passthru for now. Support for packed passthru requires evl-into-mask folding.

[simoll]

Add missing test for store

[simoll]

NFC. Formatting

[kaz7]

llvm/lib/Target/VE/VVPISelLowering.cpp

151–153

This function is implemented for only PackedLoad and PackedStore, isn't this?

If so, I think splitPackedLoadStore is better name for this function. I also think adding an assert to check PackedVector type or something related to packed-load and packed-store if it is possible.

[simoll]

• splitLoadStore -> splitPackedLoadStore
• Add assertion for packed mode
• Rebased

[kaz7]

LGTM.

[hctim]

llvm/lib/Target/VE/VVPISelLowering.cpp

161

Unused variable in opt build, might cause some bots to trap with -Wall.

llvm/lib/Target/VE/VECustomDAG.h

public:
   SDValue getConstantMask(Packing Packing, bool AllTrue) const;
   SDValue getMaskBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
   SDValue getBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
 
   // Wrap AVL in a LEGALAVL node (unless it is one already).
   SDValue annotateLegalAVL(SDValue AVL) const;
   VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
                                    PackElem Part) const;
+
+  // Splitting support
+  SDValue getSplitPtrOffset(SDValue Ptr, SDValue ByteStride,
+                            PackElem Part) const;
+  SDValue getSplitPtrStride(SDValue PackStride) const;
 };
 
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_VE_VECUSTOMDAG_H

llvm/lib/Target/VE/VECustomDAG.cpp

if (isVVPBinaryOp(Opc))
     return 3;
 
   // VM Opcodes.
   switch (Opc) {
   case VEISD::VEC_BROADCAST:
     return 1;
   case VEISD::VVP_SELECT:
     return 3;
+  case VEISD::VVP_LOAD:
+    return 4;
+  case VEISD::VVP_STORE:
+    return 5;
   }
 
   return None;
 }
 
 Optional<int> getMaskPos(unsigned Opc) {
   // This is only available for VP SDNodes
   auto PosOpt = ISD::getVPMaskIdx(Opc);
VETargetMasks VECustomDAG::getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
   if (!RawMask)
     NewMask = getConstantMask(Packing::Normal, true);
   else
     NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);
 
   return VETargetMasks(NewMask, NewAVL);
 }
 
+SDValue VECustomDAG::getSplitPtrOffset(SDValue Ptr, SDValue ByteStride,
+                                       PackElem Part) const {
+  // High starts at base ptr but has more significant bits in the 64bit vector
+  // element.
+  if (Part == PackElem::Hi)
+    return Ptr;
+  return getNode(ISD::ADD, MVT::i64, {Ptr, ByteStride});
+}
+
+SDValue VECustomDAG::getSplitPtrStride(SDValue PackStride) const {
+  if (auto ConstBytes = dyn_cast<ConstantSDNode>(PackStride))
+    return getConstant(2 * ConstBytes->getSExtValue(), MVT::i64);
+  return getNode(ISD::SHL, MVT::i64, {PackStride, getConstant(1, MVT::i32)});
+}
+
 } // namespace llvm

llvm/lib/Target/VE/VEISelLowering.h

Register prepareSymbol(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                          bool IsCall) const;
   /// } Custom Inserter
 
   /// VVP Lowering {
   SDValue lowerToVVP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerVVP_LOAD_STORE(SDValue Op, VECustomDAG&) const;
 
   SDValue legalizeInternalVectorOp(SDValue Op, SelectionDAG &DAG) const;
+  SDValue legalizeInternalLoadStoreOp(SDValue Op, VECustomDAG &CDAG) const;
   SDValue splitVectorOp(SDValue Op, VECustomDAG &CDAG) const;
+  SDValue splitLoadStore(SDValue Op, VECustomDAG &CDAG) const;
   SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const;
   SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const;
   /// } VVPLowering
 
   /// Custom DAGCombine {
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
   SDValue combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const;

llvm/lib/Target/VE/VVPISelLowering.cpp

SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
   // Store specific.
   SDValue Data = getStoredValue(Op);
   // Load specific.
   SDValue PassThru = getNodePassthru(Op);
 
   auto DataVT = *getIdiomaticVectorType(Op.getNode());
   auto Packing = getTypePacking(DataVT);
 
-  assert(Packing == Packing::Normal && "TODO Packed load store isel");
-
   // TODO: Infer lower AVL from mask.
   if (!AVL)
     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
 
   // Default to the all-true mask.
   if (!Mask)
     Mask = CDAG.getConstantMask(Packing, true);
 
SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
   }
 
   // VVP_STORE
   assert(VVPOpc == VEISD::VVP_STORE);
   return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
                       {Chain, Data, BasePtr, StrideV, Mask, AVL});
 }
 
+SDValue VETargetLowering::splitLoadStore(SDValue Op, VECustomDAG &CDAG) const {
+  auto VVPOC = *getVVPOpcode(Op.getOpcode());
+  assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
+
+  MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
+  MVT SplitDataVT = splitVectorType(DataVT);
+
+  SDValue PassThru = getNodePassthru(Op);
+  assert(!PassThru && "Should have been folded in lowering to VVP layer");
+
+  // Analyze the operation
+  SDValue PackedMask = getNodeMask(Op);
+  SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;
+  SDValue PackPtr = getMemoryPtr(Op);
+  SDValue PackData = getStoredValue(Op);
+  SDValue PackStride = getLoadStoreStride(Op, CDAG);
+
+  unsigned ChainResIdx = PackData ? 0 : 1;
+
+  SDValue PartOps[2];
+
+  SDValue UpperPartAVL; // we will use this for packing things back together
+  for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
+    // VP ops already have an explicit mask and AVL. When expanding from non-VP
+    // attach those additional inputs here.
+    auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
+
+    // Keep track of the (higher) lvl.
+    if (Part == PackElem::Hi)
+      UpperPartAVL = SplitTM.AVL;
+
+    // Attach non-predicating value operands
+    SmallVector<SDValue, 4> OpVec;
+
+    // Chain
+    OpVec.push_back(getNodeChain(Op));
+
+    // Data
+    if (PackData) {
+      SDValue PartData = CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
+      OpVec.push_back(PartData);
+    }
+
+    // Ptr & Stride
+    // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes)
+    // Stride info
+    // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);
+    OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part));
+    OpVec.push_back(CDAG.getSplitPtrStride(PackStride));
+
+    // Add predicating args and generate part node
+    OpVec.push_back(SplitTM.Mask);
+    OpVec.push_back(SplitTM.AVL);
+
+    if (PackData) {
+      // Store
+      PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);
+    } else {
+      // Load
+      PartOps[(int)Part] = CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);
+    }
+  }
+
+  // Merge the chains
+  SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx);
+  SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx);
+  SDValue FusedChains =
+      CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain});
+
+  // Chain only [store]
+  if (PackData)
+    return FusedChains;
+
+  // Re-pack into full packed vector result
+  MVT PackedVT =
+      getLegalVectorType(Packing::Dense, DataVT.getVectorElementType());
+  SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo],
+                                    PartOps[(int)PackElem::Hi], UpperPartAVL);
+
+  return CDAG.getMergeValues({PackedVals, FusedChains});
+}
+
+SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op,
+                                                      VECustomDAG &CDAG) const {
+  LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);
+  MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
+
+  // TODO: Recognize packable load,store.
+  if (isPackedVectorType(DataVT))
+    return splitLoadStore(Op, CDAG);
+
+  return legalizePackedAVL(Op, CDAG);
+}
+
 SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
                                                    SelectionDAG &DAG) const {
+  LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);
   VECustomDAG CDAG(DAG, Op);
 
+  // Dispatch to specialized legalization functions.
+  switch (Op->getOpcode()) {
+  case VEISD::VVP_LOAD:
+  case VEISD::VVP_STORE:
+    return legalizeInternalLoadStoreOp(Op, CDAG);
+  }
+
   EVT IdiomVT = Op.getValueType();
   if (isPackedVectorType(IdiomVT) &&
       !supportsPackedMode(Op.getOpcode(), IdiomVT))
     return splitVectorOp(Op, CDAG);
 
   // TODO: Implement odd/even splitting.
   return legalizePackedAVL(Op, CDAG);
 }
SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
 
   // Operation already has a legal AVL.
   auto AVL = getNodeAVL(Op);
   if (isLegalAVL(AVL))
     return Op;
 
   // Half and round up EVL for 32bit element types.
   SDValue LegalAVL = AVL;
-  if (isPackedVectorType(Op.getValueType())) {
+  MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
+  if (isPackedVectorType(IdiomVT)) {
     assert(maySafelyIgnoreMask(Op) &&
            "TODO Shift predication from EVL into Mask");
 
     if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
       LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
     } else {
       auto ConstOne = CDAG.getConstant(1, MVT::i32);
       auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});

llvm/test/CodeGen/VE/Packed/vec_load.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=ve-unknown-unknown -mattr=+vpu | FileCheck %s
+
+declare <512 x float> @llvm.masked.load.v512f32.p0v512f32(<512 x float>* %0, i32 immarg %1, <512 x i1> %2, <512 x float> %3) #0
+
+; Function Attrs: nounwind
+define fastcc <512 x float> @vec_mload_v512f32(<512 x float>* %P, <512 x i1> %M) {
+; CHECK-LABEL: vec_mload_v512f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vldu %v0, 8, %s0
+; CHECK-NEXT:    lea %s0, 4(, %s0)
+; CHECK-NEXT:    vldu %v1, 8, %s0
+; CHECK-NEXT:    vshf %v0, %v1, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %r = call <512 x float> @llvm.masked.load.v512f32.p0v512f32(<512 x float>* %P, i32 16, <512 x i1> %M, <512 x float> undef)
+  ret <512 x float> %r
+}
+
+; TODO: Packed select legalization
+; Function Attrs: nounwind
+; define fastcc <512 x float> @vec_mload_pt_v512f32(<512 x float>* %P, <512 x float> %PT, <512 x i1> %M) {
+;   %r = call <512 x float> @llvm.masked.load.v512f32.p0v512f32(<512 x float>* %P, i32 16, <512 x i1> %M, <512 x float> %PT)
+;   ret <512 x float> %r
+; }
+
+declare <512 x i32> @llvm.masked.load.v512i32.p0v512i32(<512 x i32>* %0, i32 immarg %1, <512 x i1> %2, <512 x i32> %3) #0
+
+; Function Attrs: nounwind
+define fastcc <512 x i32> @vec_mload_v512i32(<512 x i32>* %P, <512 x i1> %M) {
+; CHECK-LABEL: vec_mload_v512i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vldl.zx %v0, 8, %s0
+; CHECK-NEXT:    lea %s0, 4(, %s0)
+; CHECK-NEXT:    vldl.zx %v1, 8, %s0
+; CHECK-NEXT:    vshf %v0, %v1, %v0, 13
+; CHECK-NEXT:    b.l.t (, %s10)
+  %r = call <512 x i32> @llvm.masked.load.v512i32.p0v512i32(<512 x i32>* %P, i32 16, <512 x i1> %M, <512 x i32> undef)
+  ret <512 x i32> %r
+}
+
+; TODO: Packed select legalization
+; ; Function Attrs: nounwind
+; define fastcc <512 x i32> @vec_mload_pt_v512i32(<512 x i32>* %P, <512 x i32> %PT, <512 x i1> %M) {
+;   %r = call <512 x i32> @llvm.masked.load.v512i32.p0v512i32(<512 x i32>* %P, i32 16, <512 x i1> %M, <512 x i32> %PT)
+;   ret <512 x i32> %r
+; }
+
+attributes #0 = { argmemonly nounwind readonly willreturn }

llvm/test/CodeGen/VE/Packed/vec_store.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=ve-unknown-unknown -mattr=+vpu | FileCheck %s
+
+declare void @llvm.masked.store.v512f32.p0v512f32(<512 x float>, <512 x float>*, i32 immarg, <512 x i1>)
+
+define fastcc void @vec_mstore_v512f32(<512 x float>* %P, <512 x float> %V, <512 x i1> %M) {
+; CHECK-LABEL: vec_mstore_v512f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vstu %v0, 8, %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v0, 4
+; CHECK-NEXT:    lea %s0, 4(, %s0)
+; CHECK-NEXT:    vstu %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  call void @llvm.masked.store.v512f32.p0v512f32(<512 x float> %V, <512 x float>* %P, i32 16, <512 x i1> %M)
+  ret void
+}
+
+
+declare void @llvm.masked.store.v512i32.p0v512i32(<512 x i32>, <512 x i32>*, i32 immarg, <512 x i1>)
+
+define fastcc void @vec_mstore_v512i32(<512 x i32>* %P, <512 x i32> %V, <512 x i1> %M) {
+; CHECK-LABEL: vec_mstore_v512i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 4(, %s0)
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vstl %v0, 8, %s1
+; CHECK-NEXT:    vshf %v0, %v0, %v0, 0
+; CHECK-NEXT:    vstl %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  call void @llvm.masked.store.v512i32.p0v512i32(<512 x i32> %V, <512 x i32>* %P, i32 16, <512 x i1> %M)
+  ret void
+}