[SVE] Update API ConstantVector::getSplat() to use ElementCount.

[huihuiz]

Summary

Support ConstantInt::get() and Constant::getAllOnesValue() for scalable
vector type, this requires ConstantVector::getSplat() to take in 'ElementCount',
instead of 'unsigned' number of element count.

This change is needed for D73753.

[efriedma]

This is sort of a large number of functional changes, but it's probably okay to skip test coverage for all the simple getNumElements->getElementCount changes; it's obvious it can't break anything.

llvm/include/llvm/Analysis/Utils/Local.h

67

getVectorElementCount? (Same in other places.)

llvm/lib/IR/Constants.cpp

2129

I'd like to see test coverage for the GEP changes here, since the logic is relatively intricate.

[huihuiz]

Thanks Eli for the review!

Add test cases to check for size query changes in ConstantExpr::getGetElementPtr().

[efriedma]

clang/lib/CodeGen/CGBuiltin.cpp

4497

Please write out the type ElementCOunt.

llvm/lib/CodeGen/CodeGenPrepare.cpp

6542

Please write out the type ElementCount.

This is unfortunately turning the explicit assertion if the type is scalable into a later verifier failure in the case where it isn't a splat. Please either fix it properly, or change it so the non-splat codepath still asserts.

[huihuiz]

Addressed review comments.

llvm/lib/CodeGen/CodeGenPrepare.cpp

6542

Good catch! Thanks Eli!

Going with assert for non-splat codepath for scalable vector.

We should implement like:

UndefValue *UndefVal = UndefValue::get(getTransitionType());
Type *I32Ty = Type::getInt32Ty(getTransitionType()->getContext());
return ConstantExpr::getInsertElement(UndefVal, Val, ConstantInt::get(I32Ty, ExtractIdx));

But current target lowering will reject scalable vector earlier while checking isTypeLegal(EVT VT).

I am adding a test to check this. So we get assert once target lowering is ready. Then I can bring in this implementation and check for its correctness.

define void @simpleOneInstructionPromotion(<vscale x 2 x i32>* %addr1, i32* %dest) {
  %in1 = load <vscale x 2 x i32>, <vscale x 2 x i32>* %addr1, align 8
  %extract = extractelement <vscale x 2 x i32> %in1, i32 1
  %out = or i32 %extract, 1
  store i32 %out, i32* %dest, align 4
  ret void
}

[efriedma]

LGTM

clang/lib/CodeGen/CGBuiltin.cpp

static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
   case NeonTypeFlags::Int32:
     return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad));
   case NeonTypeFlags::Int64:
     return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad));
   default:
     llvm_unreachable("Type can't be converted to floating-point!");
   }
 }
 
 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
-  unsigned nElts = V->getType()->getVectorNumElements();
-  Value* SV = llvm::ConstantVector::getSplat(nElts, C);
+  ElementCount EC = V->getType()->getVectorElementCount();
+  Value *SV = llvm::ConstantVector::getSplat(EC, C);
   return Builder.CreateShuffleVector(V, V, SV, "lane");
 }
 
 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
                                      const char *name,
                                      unsigned shift, bool rightshift) {
   unsigned j = 0;
   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
     Ops[1] = LaneSource;
     Ops[2] = Addend;
 
     // Now adjust things to handle the lane access.
     llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
       llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
       VTy;
     llvm::Constant *cst = cast<Constant>(Ops[3]);
-    Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
+    Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
     Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
     Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
 
     Ops.pop_back();
     Int = Intrinsic::fma;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
   }
   case NEON::BI__builtin_neon_vfma_laneq_v: {
case NEON::BI__builtin_neon_vfma_laneq_v: {
     }
     Function *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
 
     llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
                                             VTy->getNumElements() * 2);
     Ops[2] = Builder.CreateBitCast(Ops[2], STy);
-    Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
+    Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
                                                cast<ConstantInt>(Ops[3]));
     Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
 
     return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
   }
   case NEON::BI__builtin_neon_vfmaq_laneq_v: {
     Function *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);

llvm/include/llvm/Analysis/Utils/Local.h

if (Constant *OpC = dyn_cast<Constant>(Op)) {
         if (Size)
           Result = Builder->CreateAdd(Result, ConstantInt::get(IntIdxTy, Size),
                                       GEP->getName()+".offs");
         continue;
       }
 
       // Splat the constant if needed.
       if (IntIdxTy->isVectorTy() && !OpC->getType()->isVectorTy())
-        OpC = ConstantVector::getSplat(IntIdxTy->getVectorNumElements(), OpC);
+        OpC = ConstantVector::getSplat(IntIdxTy->getVectorElementCount(), OpC);
 
       Constant *Scale = ConstantInt::get(IntIdxTy, Size);
       Constant *OC = ConstantExpr::getIntegerCast(OpC, IntIdxTy, true /*SExt*/);
       Scale =
           ConstantExpr::getMul(OC, Scale, false /*NUW*/, isInBounds /*NSW*/);
       // Emit an add instruction.
       Result = Builder->CreateAdd(Result, Scale, GEP->getName()+".offs");
       continue;
     }

llvm/include/llvm/IR/Constants.h

public:
   // ConstantVector accessors
   static Constant *get(ArrayRef<Constant*> V);
 
 private:
   static Constant *getImpl(ArrayRef<Constant *> V);
 
 public:
   /// Return a ConstantVector with the specified constant in each element.
-  static Constant *getSplat(unsigned NumElts, Constant *Elt);
+  static Constant *getSplat(ElementCount EC, Constant *Elt);
 
   /// Specialize the getType() method to always return a VectorType,
   /// which reduces the amount of casting needed in parts of the compiler.
   inline VectorType *getType() const {
     return cast<VectorType>(Value::getType());
   }
 
   /// If all elements of the vector constant have the same value, return that

llvm/lib/Analysis/InstructionSimplify.cpp

static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V,
 
   V = V->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds);
   // As that strip may trace through `addrspacecast`, need to sext or trunc
   // the offset calculated.
   IntIdxTy = DL.getIndexType(V->getType())->getScalarType();
   Offset = Offset.sextOrTrunc(IntIdxTy->getIntegerBitWidth());
 
   Constant *OffsetIntPtr = ConstantInt::get(IntIdxTy, Offset);
-  if (V->getType()->isVectorTy())
-    return ConstantVector::getSplat(V->getType()->getVectorNumElements(),
-                                    OffsetIntPtr);
+  if (VectorType *VecTy = dyn_cast<VectorType>(V->getType()))
+    return ConstantVector::getSplat(VecTy->getElementCount(), OffsetIntPtr);
   return OffsetIntPtr;
 }
 
 /// Compute the constant difference between two pointer values.
 /// If the difference is not a constant, returns zero.
 static Constant *computePointerDifference(const DataLayout &DL, Value *LHS,
                                           Value *RHS) {
   Constant *LHSOffset = stripAndComputeConstantOffsets(DL, LHS);

llvm/lib/CodeGen/CodeGenPrepare.cpp

for (const auto &Inst : InstsToBePromoted) {
       TargetTransformInfo::OperandValueKind Arg1OVK =
           !IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue
                           : TargetTransformInfo::OK_AnyValue;
       ScalarCost += TTI.getArithmeticInstrCost(
           Inst->getOpcode(), Inst->getType(), Arg0OVK, Arg1OVK);
       VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType,
                                                Arg0OVK, Arg1OVK);
     }
     LLVM_DEBUG(
         dbgs() << "Estimated cost of computation to be promoted:\nScalar: "
                << ScalarCost << "\nVector: " << VectorCost << '\n');
     return ScalarCost > VectorCost;
   }
 
   /// Generate a constant vector with \p Val with the same
   /// number of elements as the transition.
   /// \p UseSplat defines whether or not \p Val should be replicated
if (!UseSplat) {
       // use a splat constant.
       Value *ValExtractIdx = Transition->getOperand(getTransitionIdx());
       if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx))
         ExtractIdx = CstVal->getSExtValue();
       else
         UseSplat = true;
     }
 
-    unsigned End = getTransitionType()->getVectorNumElements();
+    ElementCount EC = getTransitionType()->getVectorElementCount();
     if (UseSplat)
-      return ConstantVector::getSplat(End, Val);
+      return ConstantVector::getSplat(EC, Val);
 
+    if (!EC.Scalable) {
       SmallVector<Constant *, 4> ConstVec;
       UndefValue *UndefVal = UndefValue::get(Val->getType());
-    for (unsigned Idx = 0; Idx != End; ++Idx) {
+      for (unsigned Idx = 0; Idx != EC.Min; ++Idx) {
         if (Idx == ExtractIdx)
           ConstVec.push_back(Val);
         else
           ConstVec.push_back(UndefVal);
       }
       return ConstantVector::get(ConstVec);
+    } else
+      llvm_unreachable(
+          "Generate scalable vector for non-splat is unimplemented");
   }
 
   /// Check if promoting to a vector type an operand at \p OperandIdx
   /// in \p Use can trigger undefined behavior.
   static bool canCauseUndefinedBehavior(const Instruction *Use,
                                         unsigned OperandIdx) {
     // This is not safe to introduce undef when the operand is on
     // the right hand side of a division-like instruction.

llvm/lib/IR/ConstantFold.cpp

Type *GEPTy = GetElementPtrInst::getGEPReturnType(
       PointeeTy, C, makeArrayRef((Value *const *)Idxs.data(), Idxs.size()));
 
   if (isa<UndefValue>(C))
     return UndefValue::get(GEPTy);
 
   Constant *Idx0 = cast<Constant>(Idxs[0]);
   if (Idxs.size() == 1 && (Idx0->isNullValue() || isa<UndefValue>(Idx0)))
     return GEPTy->isVectorTy() && !C->getType()->isVectorTy()
-               ? ConstantVector::getSplat(
-                     cast<VectorType>(GEPTy)->getNumElements(), C)
+               ? ConstantVector::getSplat(GEPTy->getVectorElementCount(), C)
                : C;
 
   if (C->isNullValue()) {
     bool isNull = true;
     for (unsigned i = 0, e = Idxs.size(); i != e; ++i)
       if (!isa<UndefValue>(Idxs[i]) &&
           !cast<Constant>(Idxs[i])->isNullValue()) {
         isNull = false;

llvm/lib/IR/Constants.cpp

Constant *Constant::getIntegerValue(Type *Ty, const APInt &V) {
   Constant *C = ConstantInt::get(Ty->getContext(), V);
 
   // Convert an integer to a pointer, if necessary.
   if (PointerType *PTy = dyn_cast<PointerType>(ScalarTy))
     C = ConstantExpr::getIntToPtr(C, PTy);
 
   // Broadcast a scalar to a vector, if necessary.
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    C = ConstantVector::getSplat(VTy->getNumElements(), C);
+    C = ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 Constant *Constant::getAllOnesValue(Type *Ty) {
   if (IntegerType *ITy = dyn_cast<IntegerType>(Ty))
     return ConstantInt::get(Ty->getContext(),
                             APInt::getAllOnesValue(ITy->getBitWidth()));
 
   if (Ty->isFloatingPointTy()) {
     APFloat FL = APFloat::getAllOnesValue(Ty->getPrimitiveSizeInBits(),
                                           !Ty->isPPC_FP128Ty());
     return ConstantFP::get(Ty->getContext(), FL);
   }
 
   VectorType *VTy = cast<VectorType>(Ty);
-  return ConstantVector::getSplat(VTy->getNumElements(),
+  return ConstantVector::getSplat(VTy->getElementCount(),
                                   getAllOnesValue(VTy->getElementType()));
 }
 
 Constant *Constant::getAggregateElement(unsigned Elt) const {
   if (const ConstantAggregate *CC = dyn_cast<ConstantAggregate>(this))
     return Elt < CC->getNumOperands() ? CC->getOperand(Elt) : nullptr;
 
   if (const ConstantAggregateZero *CAZ = dyn_cast<ConstantAggregateZero>(this))
if (!pImpl->TheFalseVal)
     pImpl->TheFalseVal = ConstantInt::get(Type::getInt1Ty(Context), 0);
   return pImpl->TheFalseVal;
 }
 
 Constant *ConstantInt::getTrue(Type *Ty) {
   assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1.");
   ConstantInt *TrueC = ConstantInt::getTrue(Ty->getContext());
   if (auto *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), TrueC);
+    return ConstantVector::getSplat(VTy->getElementCount(), TrueC);
   return TrueC;
 }
 
 Constant *ConstantInt::getFalse(Type *Ty) {
   assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1.");
   ConstantInt *FalseC = ConstantInt::getFalse(Ty->getContext());
   if (auto *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), FalseC);
+    return ConstantVector::getSplat(VTy->getElementCount(), FalseC);
   return FalseC;
 }
 
 // Get a ConstantInt from an APInt.
 ConstantInt *ConstantInt::get(LLVMContext &Context, const APInt &V) {
   // get an existing value or the insertion position
   LLVMContextImpl *pImpl = Context.pImpl;
   std::unique_ptr<ConstantInt> &Slot = pImpl->IntConstants[V];
   if (!Slot) {
     // Get the corresponding integer type for the bit width of the value.
     IntegerType *ITy = IntegerType::get(Context, V.getBitWidth());
     Slot.reset(new ConstantInt(ITy, V));
   }
   assert(Slot->getType() == IntegerType::get(Context, V.getBitWidth()));
   return Slot.get();
 }
 
 Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) {
   Constant *C = get(cast<IntegerType>(Ty->getScalarType()), V, isSigned);
 
   // For vectors, broadcast the value.
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, bool isSigned) {
   return get(Ty->getContext(), APInt(Ty->getBitWidth(), V, isSigned));
 }
 
 ConstantInt *ConstantInt::getSigned(IntegerType *Ty, int64_t V) {
   return get(Ty, V, true);
 }
 
 Constant *ConstantInt::getSigned(Type *Ty, int64_t V) {
   return get(Ty, V, true);
 }
 
 Constant *ConstantInt::get(Type *Ty, const APInt& V) {
   ConstantInt *C = get(Ty->getContext(), V);
   assert(C->getType() == Ty->getScalarType() &&
          "ConstantInt type doesn't match the type implied by its value!");
 
   // For vectors, broadcast the value.
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 ConstantInt *ConstantInt::get(IntegerType* Ty, StringRef Str, uint8_t radix) {
   return get(Ty->getContext(), APInt(Ty->getBitWidth(), Str, radix));
 }
 
Constant *ConstantFP::get(Type *Ty, double V) {
   APFloat FV(V);
   bool ignored;
   FV.convert(*TypeToFloatSemantics(Ty->getScalarType()),
              APFloat::rmNearestTiesToEven, &ignored);
   Constant *C = get(Context, FV);
 
   // For vectors, broadcast the value.
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 Constant *ConstantFP::get(Type *Ty, const APFloat &V) {
   ConstantFP *C = get(Ty->getContext(), V);
   assert(C->getType() == Ty->getScalarType() &&
          "ConstantFP type doesn't match the type implied by its value!");
 
   // For vectors, broadcast the value.
   if (auto *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 Constant *ConstantFP::get(Type *Ty, StringRef Str) {
   LLVMContext &Context = Ty->getContext();
 
   APFloat FV(*TypeToFloatSemantics(Ty->getScalarType()), Str);
   Constant *C = get(Context, FV);
 
   // For vectors, broadcast the value.
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 Constant *ConstantFP::getNaN(Type *Ty, bool Negative, uint64_t Payload) {
   const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
   APFloat NaN = APFloat::getNaN(Semantics, Negative, Payload);
   Constant *C = get(Ty->getContext(), NaN);
 
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 Constant *ConstantFP::getQNaN(Type *Ty, bool Negative, APInt *Payload) {
   const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
   APFloat NaN = APFloat::getQNaN(Semantics, Negative, Payload);
   Constant *C = get(Ty->getContext(), NaN);
 
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 Constant *ConstantFP::getSNaN(Type *Ty, bool Negative, APInt *Payload) {
   const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
   APFloat NaN = APFloat::getSNaN(Semantics, Negative, Payload);
   Constant *C = get(Ty->getContext(), NaN);
 
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 Constant *ConstantFP::getNegativeZero(Type *Ty) {
   const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
   APFloat NegZero = APFloat::getZero(Semantics, /*Negative=*/true);
   Constant *C = get(Ty->getContext(), NegZero);
 
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 
 Constant *ConstantFP::getZeroValueForNegation(Type *Ty) {
   if (Ty->isFPOrFPVectorTy())
     return getNegativeZero(Ty);
ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) {
   return Slot.get();
 }
 
 Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) {
   const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
   Constant *C = get(Ty->getContext(), APFloat::getInf(Semantics, Negative));
 
   if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    return ConstantVector::getSplat(VTy->getNumElements(), C);
+    return ConstantVector::getSplat(VTy->getElementCount(), C);
 
   return C;
 }
 
 ConstantFP::ConstantFP(Type *Ty, const APFloat &V)
     : ConstantData(Ty, ConstantFPVal), Val(V) {
   assert(&V.getSemantics() == TypeToFloatSemantics(Ty) &&
          "FP type Mismatch");
Constant *ConstantVector::getImpl(ArrayRef<Constant*> V) {
   if (ConstantDataSequential::isElementTypeCompatible(C->getType()))
     return getSequenceIfElementsMatch<ConstantDataVector>(C, V);
 
   // Otherwise, the element type isn't compatible with ConstantDataVector, or
   // the operand list contains a ConstantExpr or something else strange.
   return nullptr;
 }
 
-Constant *ConstantVector::getSplat(unsigned NumElts, Constant *V) {
+Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) {
+  if (!EC.Scalable) {
     // If this splat is compatible with ConstantDataVector, use it instead of
     // ConstantVector.
     if ((isa<ConstantFP>(V) || isa<ConstantInt>(V)) &&
         ConstantDataSequential::isElementTypeCompatible(V->getType()))
-    return ConstantDataVector::getSplat(NumElts, V);
+      return ConstantDataVector::getSplat(EC.Min, V);
 
-  SmallVector<Constant*, 32> Elts(NumElts, V);
+    SmallVector<Constant *, 32> Elts(EC.Min, V);
     return get(Elts);
   }
 
+  Type *VTy = VectorType::get(V->getType(), EC);
+
+  if (V->isNullValue())
+    return ConstantAggregateZero::get(VTy);
+  else if (isa<UndefValue>(V))
+    return UndefValue::get(VTy);
+
+  Type *I32Ty = Type::getInt32Ty(VTy->getContext());
+
+  // Move scalar into vector.
+  Constant *UndefV = UndefValue::get(VTy);
+  V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0));
+  // Build shuffle mask to perform the splat.
+  Type *MaskTy = VectorType::get(I32Ty, EC);
+  Constant *Zeros = ConstantAggregateZero::get(MaskTy);
+  // Splat.
+  return ConstantExpr::getShuffleVector(V, UndefV, Zeros);
+}
+
 ConstantTokenNone *ConstantTokenNone::get(LLVMContext &Context) {
   LLVMContextImpl *pImpl = Context.pImpl;
   if (!pImpl->TheNoneToken)
     pImpl->TheNoneToken.reset(new ConstantTokenNone(Context));
   return pImpl->TheNoneToken.get();
 }
 
 /// Remove the constant from the constant table.
if (Constant *FC =
     return FC;          // Fold a few common cases.
 
   // Get the result type of the getelementptr!
   Type *DestTy = GetElementPtrInst::getIndexedType(Ty, Idxs);
   assert(DestTy && "GEP indices invalid!");
   unsigned AS = C->getType()->getPointerAddressSpace();
   Type *ReqTy = DestTy->getPointerTo(AS);
 
-  unsigned NumVecElts = 0;
-  if (C->getType()->isVectorTy())
-    NumVecElts = C->getType()->getVectorNumElements();
+  ElementCount EltCount = {0, false};
+  if (VectorType *VecTy = dyn_cast<VectorType>(C->getType()))
+    EltCount = VecTy->getElementCount();
   else for (auto Idx : Idxs)
-    if (Idx->getType()->isVectorTy())
-      NumVecElts = Idx->getType()->getVectorNumElements();
+      if (VectorType *VecTy = dyn_cast<VectorType>(Idx->getType()))
+        EltCount = VecTy->getElementCount();
 
-  if (NumVecElts)
-    ReqTy = VectorType::get(ReqTy, NumVecElts);
+  if (EltCount.Min != 0)
+    ReqTy = VectorType::get(ReqTy, EltCount);
 
   if (OnlyIfReducedTy == ReqTy)
     return nullptr;
 
   // Look up the constant in the table first to ensure uniqueness
   std::vector<Constant*> ArgVec;
   ArgVec.reserve(1 + Idxs.size());
   ArgVec.push_back(C);
   for (unsigned i = 0, e = Idxs.size(); i != e; ++i) {
     assert((!Idxs[i]->getType()->isVectorTy() ||
-            Idxs[i]->getType()->getVectorNumElements() == NumVecElts) &&
+            Idxs[i]->getType()->getVectorElementCount() == EltCount) &&
            "getelementptr index type missmatch");
 
     Constant *Idx = cast<Constant>(Idxs[i]);
-    if (NumVecElts && !Idxs[i]->getType()->isVectorTy())
-      Idx = ConstantVector::getSplat(NumVecElts, Idx);
+    if (EltCount.Min != 0 && !Idxs[i]->getType()->isVectorTy())
+      Idx = ConstantVector::getSplat(EltCount, Idx);
     ArgVec.push_back(Idx);
   }
 
   unsigned SubClassOptionalData = InBounds ? GEPOperator::IsInBounds : 0;
   if (InRangeIndex && *InRangeIndex < 63)
     SubClassOptionalData |= (*InRangeIndex + 1) << 1;
   const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0,
                                 SubClassOptionalData, None, Ty);
if (CFP->getType()->isFloatTy()) {
       return getFP(V->getContext(), Elts);
     }
     if (CFP->getType()->isDoubleTy()) {
       SmallVector<uint64_t, 16> Elts(
           NumElts, CFP->getValueAPF().bitcastToAPInt().getLimitedValue());
       return getFP(V->getContext(), Elts);
     }
   }
-  return ConstantVector::getSplat(NumElts, V);
+  return ConstantVector::getSplat({NumElts, false}, V);
 }
 
 
 uint64_t ConstantDataSequential::getElementAsInteger(unsigned Elt) const {
   assert(isa<IntegerType>(getElementType()) &&
          "Accessor can only be used when element is an integer");
   const char *EltPtr = getElementPointer(Elt);
 

llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp

static Instruction *foldVectorCmp(CmpInst &Cmp,
 
   // Length-changing splats are ok, so adjust the constants as needed:
   // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
   Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true);
   Constant *ScalarM = M->getSplatValue(/* AllowUndefs */ true);
   if (ScalarC && ScalarM) {
     // We allow undefs in matching, but this transform removes those for safety.
     // Demanded elements analysis should be able to recover some/all of that.
-    C = ConstantVector::getSplat(V1Ty->getVectorNumElements(), ScalarC);
-    M = ConstantVector::getSplat(M->getType()->getVectorNumElements(), ScalarM);
+    C = ConstantVector::getSplat(V1Ty->getVectorElementCount(), ScalarC);
+    M = ConstantVector::getSplat(M->getType()->getVectorElementCount(),
+                                 ScalarM);
     Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C)
                          : Builder.CreateICmp(Pred, V1, C);
     return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M);
   }
 
   return nullptr;
 }
 

llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp

if (BinaryOperator *Op0BO = dyn_cast<BinaryOperator>(Op0)) {
           // (X + (Y << C))
           Value *X = Builder.CreateBinOp(Op0BO->getOpcode(), YS, V1,
                                          Op0BO->getOperand(1)->getName());
           unsigned Op1Val = Op1C->getLimitedValue(TypeBits);
 
           APInt Bits = APInt::getHighBitsSet(TypeBits, TypeBits - Op1Val);
           Constant *Mask = ConstantInt::get(I.getContext(), Bits);
           if (VectorType *VT = dyn_cast<VectorType>(X->getType()))
-            Mask = ConstantVector::getSplat(VT->getNumElements(), Mask);
+            Mask = ConstantVector::getSplat(VT->getElementCount(), Mask);
           return BinaryOperator::CreateAnd(X, Mask);
         }
 
         // Turn (Y + ((X >> C) & CC)) << C  ->  ((X & (CC << C)) + (Y << C))
         Value *Op0BOOp1 = Op0BO->getOperand(1);
         if (isLeftShift && Op0BOOp1->hasOneUse() &&
             match(Op0BOOp1,
                   m_And(m_OneUse(m_Shr(m_Value(V1), m_Specific(Op1))),
if (BinaryOperator *Op0BO = dyn_cast<BinaryOperator>(Op0)) {
           // (X + (Y << C))
           Value *X = Builder.CreateBinOp(Op0BO->getOpcode(), V1, YS,
                                          Op0BO->getOperand(0)->getName());
           unsigned Op1Val = Op1C->getLimitedValue(TypeBits);
 
           APInt Bits = APInt::getHighBitsSet(TypeBits, TypeBits - Op1Val);
           Constant *Mask = ConstantInt::get(I.getContext(), Bits);
           if (VectorType *VT = dyn_cast<VectorType>(X->getType()))
-            Mask = ConstantVector::getSplat(VT->getNumElements(), Mask);
+            Mask = ConstantVector::getSplat(VT->getElementCount(), Mask);
           return BinaryOperator::CreateAnd(X, Mask);
         }
 
         // Turn (((X >> C)&CC) + Y) << C  ->  (X + (Y << C)) & (CC << C)
         if (isLeftShift && Op0BO->getOperand(0)->hasOneUse() &&
             match(Op0BO->getOperand(0),
                   m_And(m_OneUse(m_Shr(m_Value(V1), m_Value(V2))),
                         m_ConstantInt(CC))) && V2 == Op1) {

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
   Value *ConstVF = getSignedIntOrFpConstant(Step->getType(), VF);
   Value *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, Step, ConstVF));
 
   // Create a vector splat to use in the induction update.
   //
   // FIXME: If the step is non-constant, we create the vector splat with
   //        IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
   //        handle a constant vector splat.
-  Value *SplatVF = isa<Constant>(Mul)
-                       ? ConstantVector::getSplat(VF, cast<Constant>(Mul))
-                       : Builder.CreateVectorSplat(VF, Mul);
+  Value *SplatVF =
+      isa<Constant>(Mul)
+          ? ConstantVector::getSplat({VF, false}, cast<Constant>(Mul))
   Builder.restoreIP(CurrIP);
 
   // We may need to add the step a number of times, depending on the unroll
   // factor. The last of those goes into the PHI.
   PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind",
                                     &*LoopVectorBody->getFirstInsertionPt());
   VecInd->setDebugLoc(EntryVal->getDebugLoc());
   Instruction *LastInduction = VecInd;
if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||
     Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
         RK, VecTy->getScalarType());
     if (VF == 1) {
       Identity = Iden;
       // This vector is the Identity vector where the first element is the
       // incoming scalar reduction.
       VectorStart = ReductionStartValue;
     } else {
-      Identity = ConstantVector::getSplat(VF, Iden);
+      Identity = ConstantVector::getSplat({VF, false}, Iden);
 
       // This vector is the Identity vector where the first element is the
       // incoming scalar reduction.
       VectorStart =
         Builder.CreateInsertElement(Identity, ReductionStartValue, Zero);
     }
   }
 

llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=aarch64 -codegenprepare -S < %s | FileCheck %s
+
+; This test intends to check vector promotion for scalable vector. Current target lowering
+; rejects scalable vector before reaching getConstantVector() in CodeGenPrepare. This test
+; will assert once target lowering is ready, then we can bring in implementation for non-splat
+; codepath for scalable vector.
+
+define void @simpleOneInstructionPromotion(<vscale x 2 x i32>* %addr1, i32* %dest) {
+; CHECK-LABEL: @simpleOneInstructionPromotion(
+; CHECK-NEXT:    [[IN1:%.*]] = load <vscale x 2 x i32>, <vscale x 2 x i32>* [[ADDR1:%.*]], align 8
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <vscale x 2 x i32> [[IN1]], i32 1
+; CHECK-NEXT:    [[OUT:%.*]] = or i32 [[EXTRACT]], 1
+; CHECK-NEXT:    store i32 [[OUT]], i32* [[DEST:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  %in1 = load <vscale x 2 x i32>, <vscale x 2 x i32>* %addr1, align 8
+  %extract = extractelement <vscale x 2 x i32> %in1, i32 1
+  %out = or i32 %extract, 1
+  store i32 %out, i32* %dest, align 4
+  ret void
+}
+

llvm/test/Transforms/InstSimplify/gep.ll

 define <8 x i64*> @undef_vec2() {
 ; CHECK-LABEL: @undef_vec2(
 ; CHECK-NEXT:    ret <8 x i64*> undef
 ;
   %el = getelementptr i64, <8 x i64*> undef, <8 x i64> undef
   ret <8 x i64*> %el
 }
 
+; Check ConstantExpr::getGetElementPtr() using ElementCount for size queries - begin.
+
+; Constant ptr
+
+define i32* @ptr_idx_scalar() {
+; CHECK-LABEL: @ptr_idx_scalar(
+; CHECK-NEXT:    ret i32* inttoptr (i64 4 to i32*)
+;
+  %gep = getelementptr <4 x i32>, <4 x i32>* null, i64 0, i64 1
+  ret i32* %gep
+}
+
+define <2 x i32*> @ptr_idx_vector() {
+; CHECK-LABEL: @ptr_idx_vector(
+; CHECK-NEXT:    ret <2 x i32*> getelementptr (i32, i32* null, <2 x i64> <i64 1, i64 1>)
+;
+  %gep = getelementptr i32, i32* null, <2 x i64> <i64 1, i64 1>
+  ret <2 x i32*> %gep
+}
+
+define <4 x i32*> @ptr_idx_mix_scalar_vector(){
+; CHECK-LABEL: @ptr_idx_mix_scalar_vector(
+; CHECK-NEXT:    ret <4 x i32*> getelementptr ([42 x [3 x i32]], [42 x [3 x i32]]* null, <4 x i64> zeroinitializer, <4 x i64> <i64 0, i64 1, i64 2, i64 3>, <4 x i64> zeroinitializer)
+;
+  %gep = getelementptr [42 x [3 x i32]], [42 x [3 x i32]]* null, i64 0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>, i64 0
+  ret <4 x i32*> %gep
+}
+
+; Constant vector
+
+define <4 x i32*> @vector_idx_scalar() {
+; CHECK-LABEL: @vector_idx_scalar(
+; CHECK-NEXT:    ret <4 x i32*> getelementptr (i32, <4 x i32*> zeroinitializer, <4 x i64> <i64 1, i64 1, i64 1, i64 1>)
+;
+  %gep = getelementptr i32, <4 x i32*> zeroinitializer, i64 1
+  ret <4 x i32*> %gep
+}
+
+define <4 x i32*> @vector_idx_vector() {
+; CHECK-LABEL: @vector_idx_vector(
+; CHECK-NEXT:    ret <4 x i32*> getelementptr (i32, <4 x i32*> zeroinitializer, <4 x i64> <i64 1, i64 1, i64 1, i64 1>)
+;
+  %gep = getelementptr i32, <4 x i32*> zeroinitializer, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
+  ret <4 x i32*> %gep
+}
+
+%struct = type { double, float }
+define <4 x float*> @vector_idx_mix_scalar_vector() {
+; CHECK-LABEL: @vector_idx_mix_scalar_vector(
+; CHECK-NEXT:    ret <4 x float*> getelementptr (%struct, <4 x %struct*> zeroinitializer, <4 x i64> zeroinitializer, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+;
+  %gep = getelementptr %struct, <4 x %struct*> zeroinitializer, i32 0, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x float*> %gep
+}
+
+; Constant scalable
+
+define <vscale x 4 x i32*> @scalable_idx_scalar() {
+; CHECK-LABEL: @scalable_idx_scalar(
+; CHECK-NEXT:    ret <vscale x 4 x i32*> getelementptr (i32, <vscale x 4 x i32*> zeroinitializer, <vscale x 4 x i64> shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> undef, i64 1, i32 0), <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer))
+;
+  %gep = getelementptr i32, <vscale x 4 x i32*> zeroinitializer, i64 1
+  ret <vscale x 4 x i32*> %gep
+}
+
+; Check ConstantExpr::getGetElementPtr() using ElementCount for size queries - end.

llvm/unittests/FuzzMutate/OperationsTest.cpp

Constant *i64 = ConstantInt::get(Type::getInt64Ty(Ctx),
                                    std::numeric_limits<uint64_t>::max());
   Constant *f16 = ConstantFP::getInfinity(Type::getHalfTy(Ctx));
   Constant *f32 = ConstantFP::get(Type::getFloatTy(Ctx), 0.0);
   Constant *f64 = ConstantFP::get(Type::getDoubleTy(Ctx), 123.45);
   Constant *s =
       ConstantStruct::get(StructType::create(Ctx, "OpaqueStruct"));
   Constant *a =
       ConstantArray::get(ArrayType::get(i32->getType(), 2), {i32, i32});
-  Constant *v8i8 = ConstantVector::getSplat(8, i8);
-  Constant *v4f16 = ConstantVector::getSplat(4, f16);
+  Constant *v8i8 = ConstantVector::getSplat({8, false}, i8);
+  Constant *v4f16 = ConstantVector::getSplat({4, false}, f16);
   Constant *p0i32 =
       ConstantPointerNull::get(PointerType::get(i32->getType(), 0));
 
   auto OnlyI32 = onlyType(i32->getType());
   EXPECT_TRUE(OnlyI32.matches({}, i32));
   EXPECT_FALSE(OnlyI32.matches({}, i64));
   EXPECT_FALSE(OnlyI32.matches({}, p0i32));
   EXPECT_FALSE(OnlyI32.matches({}, a));

llvm/unittests/IR/VerifierTest.cpp

TEST(VerifierTest, Freeze) {
   Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
   BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
   ReturnInst *RI = ReturnInst::Create(C, Entry);
 
   IntegerType *ITy = IntegerType::get(C, 32);
   ConstantInt *CI = ConstantInt::get(ITy, 0);
 
   // Valid type : freeze(<2 x i32>)
-  Constant *CV = ConstantVector::getSplat(2, CI);
+  Constant *CV = ConstantVector::getSplat({2, false}, CI);
   FreezeInst *FI_vec = new FreezeInst(CV);
   FI_vec->insertBefore(RI);
 
   EXPECT_FALSE(verifyFunction(*F));
 
   FI_vec->eraseFromParent();
 
   // Valid type : freeze(float)