Implementation of PPC lowering for vp_load/vp_store with no mask

[hussainjk]

Summary

On PowerPC, lower vp_load and vp_store nodes with %evl but no %mask operands to lxvl and stxvl instructions. Depends on D109377.

[bmahjour]

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

1339–1384

in what patch is this function first introduced? I don't see it in main or any of the other recent patches you posted.

[hussainjk]

Fixing some arc errors when making the patch.

[hussainjk]

Putting back accidentally-deleted test file.

[hussainjk]

Expanded LIT tests with a lot more cases, including vector splitting.

[nemanjai]

llvm/lib/Target/PowerPC/PPCInstrVSX.td

2437

Why are these here? These nodes are target independent so it definitely seems like the wrong thing to do to define them in a PPC-specific file.

llvm/test/CodeGen/PowerPC/ldst-with-length-vector.ll

3

This needs to have a big endian line as well and also a -O0 for each to ensure that it works with FastISEL.

[nemanjai]

llvm/test/CodeGen/PowerPC/ldst-with-length-vector.ll

8

I must be missing something here. I thought the %evl parameter is the explicit vector length but it is clearly getting ignored and we are shifting a constant. So I don't really follow what is happening here.

[nemanjai]

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

15242

The semantics here are very different from what I assumed. I assumed one can load 5 byte elements of a v16i8 vector by using the v16i8 type and a length of 5. But we have to use a different type (i.e. v5i8)?

[bmahjour]

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

15249

should this be NewOps[3] instead?

15276

5 seems out or range. Shouldn't it be NewOps[4] = ShiftedLength; ?

[hussainjk]

Fixed tests incorrectly using zeroinitializer instead of undef for %evl, causing it to be ignored.

[RolandF]

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

1339–1384

This is a target specialization of the default function from https://reviews.llvm.org/D78203.

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   // To handle counter-based loop conditions.
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom);
 
   setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom);
   setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom);
   setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom);
   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 
+  setOperationAction(ISD::VP_LOAD, MVT::i32, Promote);
+  setOperationAction(ISD::VP_STORE, MVT::i32, Promote);
+
   // Comparisons that require checking two conditions.
   if (Subtarget.hasSPE()) {
     setCondCodeAction(ISD::SETO, MVT::f32, Expand);
     setCondCodeAction(ISD::SETO, MVT::f64, Expand);
     setCondCodeAction(ISD::SETUO, MVT::f32, Expand);
     setCondCodeAction(ISD::SETUO, MVT::f64, Expand);
   }
   setCondCodeAction(ISD::SETULT, MVT::f32, Expand);

   setTargetDAGCombine(ISD::MUL);
   setTargetDAGCombine(ISD::FMA);
   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::BUILD_VECTOR);
   if (Subtarget.hasFPCVT())
     setTargetDAGCombine(ISD::UINT_TO_FP);
   setTargetDAGCombine(ISD::LOAD);
   setTargetDAGCombine(ISD::STORE);
+  setTargetDAGCombine(ISD::VP_LOAD);
+  setTargetDAGCombine(ISD::VP_STORE);
   setTargetDAGCombine(ISD::BR_CC);
   if (Subtarget.useCRBits())
     setTargetDAGCombine(ISD::BRCOND);
   setTargetDAGCombine(ISD::BSWAP);
   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
   setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
   setTargetDAGCombine(ISD::INTRINSIC_VOID);
 
if (LD->isUnindexed() && VT.isVector() &&
 
       // The output of the permutation is our loaded result, the TokenFactor is
       // our new chain.
       DCI.CombineTo(N, Perm, TF);
       return SDValue(N, 0);
     }
     }
     break;
+    case ISD::VP_LOAD: {
+      if (!DCI.isAfterLegalizeDAG())
+        break;
+      auto *LD = cast<VPLoadSDNode>(N);
+      SDValue Length = LD->getVectorLength();
+      SDNode *LengthNode = Length.getNode();
+      // We don't want to shift on subsequent combines.
+      // As a workaround, we check if the length node is a constant shift of at
+      // least 56. The shift+mul might get combined, so we also check for a
+      // corresponding constant multiplication.
+      if (LengthNode->getOpcode() == ISD::SHL)
+        if (isa<ConstantSDNode>(Length.getOperand(1).getNode()))
+          if (Length.getConstantOperandVal(1) >= 56)
+            return SDValue();
+      if (LengthNode->getOpcode() == ISD::MUL)
+        if (isa<ConstantSDNode>(Length.getOperand(1).getNode()))
+          if (Length.getConstantOperandVal(1) >> 56)
+            return SDValue();
+      SDLoc DL(N);
+      unsigned EltBytes = N->getValueType(0).getScalarSizeInBits() / 8;
+      SDValue ExtLength = DAG.getZExtOrTrunc(Length, DL, MVT::i64);
+      SDValue ShiftedLength =
+          DAG.getNode(ISD::SHL, DL, MVT::i64, ExtLength,
+                      DAG.getConstant(56 + countTrailingZeros(EltBytes), DL,
+                                      getPointerTy(DAG.getDataLayout())));
+      SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
+      NewOps[4] = ShiftedLength;
+      return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+      break;
+    }
+    case ISD::VP_STORE: {
+      if (!DCI.isAfterLegalizeDAG())
+        break;
+      auto *ST = cast<VPStoreSDNode>(N);
+      SDValue Length = ST->getVectorLength();
+      SDNode *LengthNode = Length.getNode();
+      if (LengthNode->getOpcode() == ISD::SHL)
+        if (isa<ConstantSDNode>(Length.getOperand(1).getNode()))
+          if (Length.getConstantOperandVal(1) >= 56)
+            return SDValue();
+      if (LengthNode->getOpcode() == ISD::MUL)
+        if (isa<ConstantSDNode>(Length.getOperand(1).getNode()))
+          if (Length.getConstantOperandVal(1) >> 56)
+            return SDValue();
+      SDLoc DL(N);
+      unsigned EltBytes =
+          N->getOperand(1).getValueType().getScalarSizeInBits() / 8;
+      SDValue ExtLength = DAG.getZExtOrTrunc(Length, DL, MVT::i64);
+      SDValue ShiftedLength =
+          DAG.getNode(ISD::SHL, DL, MVT::i64, ExtLength,
+                      DAG.getConstant(56 + countTrailingZeros(EltBytes), DL,
+                                      getPointerTy(DAG.getDataLayout())));
+      SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
+      NewOps[5] = ShiftedLength;
+      return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+      break;
+    }
     case ISD::INTRINSIC_WO_CHAIN: {
       bool isLittleEndian = Subtarget.isLittleEndian();
       unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
       Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
                                            : Intrinsic::ppc_altivec_lvsl);
       if (IID == Intr && N->getOperand(1)->getOpcode() == ISD::ADD) {
         SDValue Add = N->getOperand(1);
 

llvm/lib/Target/PowerPC/PPCInstrVSX.td

def DblwdCmp {
   dag MRGSGT = (v2i64 (XXPERMDI (v2i64 (XXSPLTW SGTWOR, 0)),
                                 (v2i64 (XXSPLTW SGTWOR, 2)), 0));
   dag MRGUGT = (v2i64 (XXPERMDI (v2i64 (XXSPLTW UGTWOR, 0)),
                                 (v2i64 (XXSPLTW UGTWOR, 2)), 0));
   dag MRGEQ = (v2i64 (XXPERMDI (v2i64 (XXSPLTW EQWSHAND, 0)),
                                (v2i64 (XXSPLTW EQWSHAND, 2)), 0));
 }
 
+def SDTVPLoad: SDTypeProfile<1, 4, [
+  SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisSameNumEltsAs<0, 3>, SDTCisInt<3>, SDTCisInt<4>
+]>;
+def SDTVPStore: SDTypeProfile<0, 5, [
+  SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>,  SDTCisSameNumEltsAs<0, 3>, SDTCisInt<3>, SDTCisInt<4>
+]>;
+def vp_load    : SDNode<"ISD::VP_LOAD",  SDTVPLoad,
+  [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def vp_store    : SDNode<"ISD::VP_STORE",  SDTVPStore,
+  [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+def load_vl : PatFrags<(ops node:$src1, node:$src2), [
+    (vp_load node:$src1, undef, undef, node:$src2),
+    (vp_load node:$src1, undef, immAllZerosV, node:$src2)
+  ], [{
+    return !cast<VPLoadSDNode>(N)->isExpandingLoad() &&
+      cast<VPLoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD &&
+      cast<VPLoadSDNode>(N)->isUnindexed();
+}]>;
+def store_vl : PatFrags<(ops node:$src1, node:$src2, node:$src3), [
+    (vp_store node:$src1, node:$src2, undef, undef, node:$src3),
+    (vp_store node:$src1, node:$src2, undef, immAllZerosV, node:$src3)
+  ], [{
+    return !cast<VPStoreSDNode>(N)->isTruncatingStore() &&
+        cast<VPStoreSDNode>(N)->isUnindexed();
+}]>;
+
 //---------------------------- Anonymous Patterns ----------------------------//
 // Predicate combinations are kept in roughly chronological order in terms of
 // instruction availability in the architecture. For example, VSX came in with
 // ISA 2.06 (Power7). There have since been additions in ISA 2.07 (Power8) and
 // ISA 3.0 (Power9). However, the granularity of features on later subtargets
 // is finer for various reasons. For example, we have Power8Vector,
 // Power8Altivec, DirectMove that all came in with ISA 2.07. The situation is
 // similar with ISA 3.0 with Power9Vector, Power9Altivec, IsISA3_0. Then there
def : Pat<(int_ppc_vsx_stxvw4x v4i32:$rS, DQForm:$dst),
           (STXV $rS, memrix16:$dst)>;
 def : Pat<(int_ppc_vsx_stxvd2x v2f64:$rS, DQForm:$dst),
           (STXV $rS, memrix16:$dst)>;
 def : Pat<(int_ppc_vsx_stxvw4x v4i32:$rS, XForm:$dst),
           (STXVX $rS, XForm:$dst)>;
 def : Pat<(int_ppc_vsx_stxvd2x v2f64:$rS, XForm:$dst),
           (STXVX $rS, XForm:$dst)>;
 
+// ld/st-with-length patterns
+foreach vt = [ v2i64, v4i32, v2f64, v4f32, ] in {
+  def : Pat<(!cast<ValueType>(""#vt) (load_vl addr:$src, i64:$rB)),
+            (LXVL $src, $rB)>;
+  def : Pat<(store_vl !cast<ValueType>(""#vt):$rS, addr:$dst, i64:$rB),
+            (STXVL $rS, $dst, $rB)>;
+}
+// We have to do v8i16 and v16i8 separately because they are not in VSRC.
+foreach vt = [ v8i16, v16i8, ] in {
+  def : Pat<(!cast<ValueType>(""#vt) (load_vl addr:$src, i64:$rB)),
+            (COPY_TO_REGCLASS (LXVL $src, $rB), VRRC)>;
+  def : Pat<(store_vl !cast<ValueType>(""#vt):$rS, addr:$dst, i64:$rB),
+            (STXVL (COPY_TO_REGCLASS $rS, VSRC), $dst, $rB)>;
+}
+
 // Build vectors from i8 loads
 defm : ScalToVecWPermute<v8i16, ScalarLoads.ZELi8,
                          (VSPLTHs 3, (LXSIBZX ForceXForm:$src)),
                          (SUBREG_TO_REG (i64 1), (LXSIBZX ForceXForm:$src), sub_64)>;
 defm : ScalToVecWPermute<v4i32, ScalarLoads.ZELi8,
                          (XXSPLTWs (LXSIBZX ForceXForm:$src), 1),
                          (SUBREG_TO_REG (i64 1), (LXSIBZX ForceXForm:$src), sub_64)>;
 defm : ScalToVecWPermute<v2i64, ScalarLoads.ZELi8i64,

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h

InstructionCost getInterleavedMemoryOpCost(
       Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
       bool UseMaskForCond = false, bool UseMaskForGaps = false);
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind);
   bool areFunctionArgsABICompatible(const Function *Caller,
                                     const Function *Callee,
                                     SmallPtrSetImpl<Argument *> &Args) const;
+  TTI::VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const;
   /// @}
 };
 
 } // end namespace llvm
 
 #endif

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

case Intrinsic::ppc_vsx_stxvp: {
     Info.WriteMem = true;
     return true;
   }
   default:
     break;
   }
 
   return false;
 }
+
+TTI::VPLegalization
+PPCTTIImpl::getVPLegalizationStrategy(const VPIntrinsic &PI) const {
+  auto Legal = TargetTransformInfo::VPLegalization(
+      /* EVLParamStrategy */ TargetTransformInfo::VPLegalization::Legal,
+      /* OperatorStrategy */ TargetTransformInfo::VPLegalization::Legal);
+  auto Illegal = BaseT::getVPLegalizationStrategy(PI);
+  // Masks are unsupported.
+  if (!isa<UndefValue>(PI.getMaskParam()))
+    return Illegal;
+  switch (PI.getIntrinsicID()) {
+  default:
+    return Illegal;
+  case Intrinsic::vp_load:
+  case Intrinsic::vp_store: {
+    // We currently don't support the target-independent interface for Altivec.
+    // Loads/stores with length instructions use bits 0-7 of the GPR operand and
+    // therefore cannot be used in 32-bit mode.
+    if ((!ST->hasP9Vector() && !ST->hasP10Vector()) || !ST->isPPC64())
+      return Illegal;
+    Type *DataType = PI.getIntrinsicID() == Intrinsic::vp_load
+                         ? PI.getType()
+                         : PI.getMemoryDataParam()->getType();
+    if (isa<FixedVectorType>(DataType)) {
+      //      auto *VecTy = dyn_cast<FixedVectorType>(DataType);
+      unsigned VecWidth = DataType->getPrimitiveSizeInBits();
+      return VecWidth == 128 ? Legal : Illegal;
+    }
+    Type *ScalarTy = DataType->getScalarType();
+    if (ScalarTy->isPointerTy())
+      return Legal;
+
+    if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
+      return Legal;
+
+    if (!ScalarTy->isIntegerTy())
+      return Illegal;
+
+    unsigned IntWidth = ScalarTy->getIntegerBitWidth();
+    if (IntWidth == 8 || IntWidth == 16 || IntWidth == 32 || IntWidth == 64)
+      return Legal;
+    return Illegal;
+  }
+  }
+}

llvm/test/CodeGen/PowerPC/ldst-with-length-vector.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+
+define void @partial_st_4i32(<4 x i32>* %ptr, <4 x i32> %value, i32 %len) {
+; CHECK-LABEL: partial_st_4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    sldi 4, 7, 58
+; CHECK-NEXT:    stxvl 34, 3, 4
+; CHECK-NEXT:    blr
+  call void @llvm.vp.store.v4i32.p0v4i32.i64(<4 x i32> %value, <4 x i32>* %ptr, <4 x i1> undef, i32 %len)
+  ret void
+}
+declare void @llvm.vp.store.v4i32.p0v4i32.i64(<4 x i32>, <4 x i32>*, <4 x i1>, i32)
+
+define <4 x i32> @partial_ld_4i32(<4 x i32>* %ptr, i32 %len) {
+; CHECK-LABEL: partial_ld_4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    sldi 4, 4, 58
+; CHECK-NEXT:    lxvl 34, 3, 4
+; CHECK-NEXT:    blr
+  %res = call <4 x i32> @llvm.vp.load.v4i32.p0v4i32.i64(<4 x i32>* %ptr, <4 x i1> undef, i32 %len)
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.vp.load.v4i32.p0v4i32.i64(<4 x i32>*, <4 x i1>, i32)