[CodeGen] Prepare for introduction of v3 and v5 MVTs

[tpr]

Summary

AMDGPU would like to have MVTs for v3i32, v3f32, v5i32, v5f32. This
commit does not add them, but makes preparatory changes:

• Exclude non-legal non-power-of-2 vector types from ComputeRegisterProp mechanism in TargetLoweringBase::getTypeConversion.

• Cope with SETCC and VSELECT for odd-width i1 vector when the other vectors are legal type.

• Fixed an assumption of power-of-2 vector type in ARM.

• Fixed assumptions of power-of-2 vector type in AMDGPU kernel arg handling.

• Fixed AMDGPU cost analysis to behave the same.

Some of this patch is from Matt Arsenault, also of AMD.

Change-Id: Ib5f23377dbef511be3a936211a0b9f94e46331f8

[arsenm]

The ARM patch at least should be split into a separate patch

lib/Target/AMDGPU/AMDGPUISelLowering.cpp

1034 ↗

(On Diff #189142)

Braces

1036 ↗

(On Diff #189142)

Should also make sure to add some 5x vector argument tests to the kernarg tests since you are adding those

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

343 ↗

(On Diff #189142)

This could use a test in test/Analysis/CostModel/AMDGPU

lib/Target/ARM/ARMISelLowering.cpp

12153 ↗

(On Diff #189142)

Combine the last 2 checks into NumLanes >= 3

12211 ↗

(On Diff #189142)

Ditto

[tpr]

lib/Target/ARM/ARMISelLowering.cpp

12153 ↗

(On Diff #189142)

That won't work -- it needs to be true for ==3 and >4, but not for ==4.

[efriedma]

The general idea makes sense, I think.

lib/CodeGen/TargetLoweringBase.cpp

746

The reason we're checking for isSimple() here is precisely so that we can perform a lookup in the TransformToType array. If we need a couple extra lines of code to handle TypeWidenVector, we should just add them (and maybe refactor the code so it doesn't repeat itself so much).

[tpr]

V2: Moved ARM and AMDGPU changes out to their own commits.

[tpr]

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

343 ↗

(On Diff #189142)

I have separated out the amdgpu changes into another commit, but:

In adding v5 cost model tests, I found that (a) this change here is unnecessary, and (b) I had a bug in one of my later changes (the one that makes v5 legal on amdgpu) that made the test fail. So I have added the tests.

[tpr]

V3: Addressed review comment by widening illegal odd vectors in a
different way.

[efriedma]

LGTM

include/llvm/CodeGen/SelectionDAG.h

public:
 
   /// Split the node's operand with EXTRACT_SUBVECTOR and
   /// return the low/high part.
   std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
   {
     return SplitVector(N->getOperand(OpNo), SDLoc(N));
   }
 
+  /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
+  SDValue WidenVector(const SDValue &N, const SDLoc &DL);
+
   /// Append the extracted elements from Start to Count out of the vector Op
   /// in Args. If Count is 0, all of the elements will be extracted.
   void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
                              unsigned Start = 0, unsigned Count = 0);
 
   /// Compute the default alignment value for the given type.
   unsigned getEVTAlignment(EVT MemoryVT) const;
 

lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   SDValue WidenVecOp_EXTEND(SDNode *N);
   SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue WidenVecOp_STORE(SDNode* N);
   SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_SETCC(SDNode* N);
+  SDValue WidenVecOp_VSELECT(SDNode *N);
 
   SDValue WidenVecOp_Convert(SDNode *N);
   SDValue WidenVecOp_FCOPYSIGN(SDNode *N);
 
   //===--------------------------------------------------------------------===//
   // Vector Widening Utilities Support: LegalizeVectorTypes.cpp
   //===--------------------------------------------------------------------===//
 

lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
   // we'd prefer to widen the result type, the input operands have been split.
   // In this case, we also need to split the result of this node as well.
   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
     SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
     SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
     return Res;
   }
 
+  // If the inputs also widen, handle them directly. Otherwise widen by hand.
+  SDValue InOp2 = N->getOperand(1);
+  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
     InOp1 = GetWidenedVector(InOp1);
-  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
+    InOp2 = GetWidenedVector(InOp2);
+  } else {
+    InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
+    InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
+  }
 
   // Assume that the input and output will be widen appropriately.  If not,
   // we will have to unroll it at some point.
   assert(InOp1.getValueType() == WidenInVT &&
          InOp2.getValueType() == WidenInVT &&
          "Input not widened to expected type!");
   (void)WidenInVT;
   return DAG.getNode(ISD::SETCC, SDLoc(N),
#endif
   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
   case ISD::EXTRACT_SUBVECTOR:  Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
   case ISD::MSTORE:             Res = WidenVecOp_MSTORE(N, OpNo); break;
   case ISD::MGATHER:            Res = WidenVecOp_MGATHER(N, OpNo); break;
   case ISD::MSCATTER:           Res = WidenVecOp_MSCATTER(N, OpNo); break;
   case ISD::SETCC:              Res = WidenVecOp_SETCC(N); break;
+  case ISD::VSELECT:            Res = WidenVecOp_VSELECT(N); break;
   case ISD::FCOPYSIGN:          Res = WidenVecOp_FCOPYSIGN(N); break;
 
   case ISD::ANY_EXTEND:
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
     Res = WidenVecOp_EXTEND(N);
     break;
 
EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
                                VT.getVectorNumElements());
   SDValue CC = DAG.getNode(
       ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
       DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
 
   return PromoteTargetBoolean(CC, VT);
 }
 
+SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
+  // This only gets called in the case that the left and right inputs and
+  // result are of a legal odd vector type, and the condition is illegal i1 of
+  // the same odd width that needs widening.
+  EVT VT = N->getValueType(0);
+  assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
+
+  SDValue Cond = GetWidenedVector(N->getOperand(0));
+  SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
+  SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
+  SDLoc DL(N);
+
+  SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
+                               LeftIn, RightIn);
+  return DAG.getNode(
+      ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
+      DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
+}
 
 //===----------------------------------------------------------------------===//
 // Vector Widening Utilities
 //===----------------------------------------------------------------------===//
 
 // Utility function to find the type to chop up a widen vector for load/store
 //  TLI:       Target lowering used to determine legal types.
 //  Width:     Width left need to load/store.

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
   Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
                getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
   Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
                getConstant(LoVT.getVectorNumElements(), DL,
                            TLI->getVectorIdxTy(getDataLayout())));
   return std::make_pair(Lo, Hi);
 }
 
+/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
+SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
+  EVT VT = N.getValueType();
+  EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
+                                NextPowerOf2(VT.getVectorNumElements()));
+  return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
+                 getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
+}
+
 void SelectionDAG::ExtractVectorElements(SDValue Op,
                                          SmallVectorImpl<SDValue> &Args,
                                          unsigned Start, unsigned Count) {
   EVT VT = Op.getValueType();
   if (Count == 0)
     Count = VT.getVectorNumElements();
 
   EVT EltVT = VT.getVectorElementType();

lib/CodeGen/TargetLoweringBase.cpp

 void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {
   // If the command-line option was specified, ignore this request.
   if (!JumpIsExpensiveOverride.getNumOccurrences())
     JumpIsExpensive = isExpensive;
 }
 
 TargetLoweringBase::LegalizeKind
 TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {
-  // If this is a simple type, use the ComputeRegisterProp mechanism.
-  if (VT.isSimple()) {
+  // If this is a simple type, excluding a non-legal non-power-of-two vector,
+  // use the ComputeRegisterProp mechanism.
+  if (VT.isSimple() &&
+      (!VT.isVector() || VT.isPow2VectorType() || isTypeLegal(VT))) {
     MVT SVT = VT.getSimpleVT();
     assert((unsigned)SVT.SimpleTy < array_lengthof(TransformToType));
     MVT NVT = TransformToType[SVT.SimpleTy];
     LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
 
     assert((LA == TypeLegal || LA == TypeSoftenFloat ||
             ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger) &&
            "Promote may not follow Expand or Promote");