[SVE] Remove calls to VectorType::getNumElements from CodeGen

[ctetreau]

rebase

[RKSimon]

LGTM with one minor

llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp

522–523

I know you're being cautious about changes but I think this assertion can use isa<FixedVectorType>? Or drop the assert entirely - cast<> implicitly asserts for the correct type.

[ctetreau]

address code review issues

[ctetreau]

@sdesmalen Would you like me to hold off on this one until after the branch?

[sdesmalen]

In D82210#2125933, @ctetreau wrote:

@sdesmalen Would you like me to hold off on this one until after the branch?

Thanks for checking @ctetreau. I actually think most of the changes are fine to go in now, with the exception of:

• the cast to FixedVectorType in llvm/lib/CodeGen/ValueTypes.cpp.
• the changes to InterleavedAccessPass, because this pass is not yet guarded by bailing out early.

The other changes are fine because:

• The ACLE uses custom target-specific intrinsics instead of llvm.masked.load/store/gather/scatter intrinsics.
• The ACLE uses custom target-specific intrinsics instead of the llvm.experimental.reduction intrinsics.
• GlobalISel already falls back to DAGISel if the function uses scalable vectors (D81557 and D82524)
• The InterleavedLoadCombinePass was guarded in D79700 to bail out early for scalable vectors.
• shouldConvertSplatType returns nullptr for AArch64 so the change in CodeGenPrepare::optimizeShuffleVectorInst is never hit.

If you can add a check to bail out early for the InterleavedAccessPass for scalable vectors and move the change to EVT::getExtendedVectorNumElements into a separate patch, then I'm happy with most of this to go in now.

[ctetreau]

rebase

[ctetreau]

address code review issues

[ctetreau]

@sdesmalen I went ahead and made the requested changes. In ValueType.cpp, rather than leaving the code alone, I went ahead and returned getElementCount().Min along with adding a warning if the vector was scalable. This way getNumElement() can be removed in the future without having to touch this file again.

[ctetreau]

account for abandoned D82329

llvm/lib/CodeGen/CodeGenPrepare.cpp

if (Ops[FinalIndex]->getType()->isVectorTy()) {
     }
   }
 
   // If we made any changes or the we have extra operands, we need to generate
   // new instructions.
   if (!RewriteGEP && Ops.size() == 2)
     return false;
 
-  unsigned NumElts = cast<VectorType>(Ptr->getType())->getNumElements();
+  unsigned NumElts = cast<FixedVectorType>(Ptr->getType())->getNumElements();
 
   IRBuilder<> Builder(MemoryInst);
 
   Type *ScalarIndexTy = DL->getIndexType(Ops[0]->getType()->getScalarType());
 
   Value *NewAddr;
 
   // If the final index isn't a vector, emit a scalar GEP containing all ops

 bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
   if (!match(SVI, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
                             m_Undef(), m_ZeroMask())))
     return false;
   Type *NewType = TLI->shouldConvertSplatType(SVI);
   if (!NewType)
     return false;
 
-  VectorType *SVIVecType = cast<VectorType>(SVI->getType());
+  auto *SVIVecType = cast<FixedVectorType>(SVI->getType());
   assert(!NewType->isVectorTy() && "Expected a scalar type!");
   assert(NewType->getScalarSizeInBits() == SVIVecType->getScalarSizeInBits() &&
          "Expected a type of the same size!");
   auto *NewVecType =
       FixedVectorType::get(NewType, SVIVecType->getNumElements());
 
   // Create a bitcast (shuffle (insert (bitcast(..))))
   IRBuilder<> Builder(SVI->getContext());

llvm/lib/CodeGen/ExpandReductions.cpp

for (auto *II : Worklist) {
     case Intrinsic::experimental_vector_reduce_v2_fmul: {
       // FMFs must be attached to the call, otherwise it's an ordered reduction
       // and it can't be handled by generating a shuffle sequence.
       Value *Acc = II->getArgOperand(0);
       Value *Vec = II->getArgOperand(1);
       if (!FMF.allowReassoc())
         Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK);
       else {
-        if (!isPowerOf2_32(cast<VectorType>(Vec->getType())->getNumElements()))
+        if (!isPowerOf2_32(
+                cast<FixedVectorType>(Vec->getType())->getNumElements()))
           continue;
 
         Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
         Rdx = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(ID),
                                   Acc, Rdx, "bin.rdx");
       }
       break;
     }
     case Intrinsic::experimental_vector_reduce_add:
     case Intrinsic::experimental_vector_reduce_mul:
     case Intrinsic::experimental_vector_reduce_and:
     case Intrinsic::experimental_vector_reduce_or:
     case Intrinsic::experimental_vector_reduce_xor:
     case Intrinsic::experimental_vector_reduce_smax:
     case Intrinsic::experimental_vector_reduce_smin:
     case Intrinsic::experimental_vector_reduce_umax:
     case Intrinsic::experimental_vector_reduce_umin:
     case Intrinsic::experimental_vector_reduce_fmax:
     case Intrinsic::experimental_vector_reduce_fmin: {
       Value *Vec = II->getArgOperand(0);
-      if (!isPowerOf2_32(cast<VectorType>(Vec->getType())->getNumElements()))
+      if (!isPowerOf2_32(
+              cast<FixedVectorType>(Vec->getType())->getNumElements()))
         continue;
 
       Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
       break;
     }
     }
     II->replaceAllUsesWith(Rdx);
     II->eraseFromParent();

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

bool IRTranslator::translateGetElementPtr(const User &U,
   LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
   Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
   LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
 
   // Normalize Vector GEP - all scalar operands should be converted to the
   // splat vector.
   unsigned VectorWidth = 0;
   if (auto *VT = dyn_cast<VectorType>(U.getType()))
-    VectorWidth = VT->getNumElements();
+    VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
 
   // We might need to splat the base pointer into a vector if the offsets
   // are vectors.
   if (VectorWidth && !PtrTy.isVector()) {
     BaseReg =
         MIRBuilder.buildSplatVector(LLT::vector(VectorWidth, PtrTy), BaseReg)
             .getReg(0);
     PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
                          DL->getABITypeAlign(U.getType()).value()});
   return true;
 }
 
 bool IRTranslator::translateInsertElement(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
   // If it is a <1 x Ty> vector, use the scalar as it is
   // not a legal vector type in LLT.
-  if (cast<VectorType>(U.getType())->getNumElements() == 1)
+  if (cast<FixedVectorType>(U.getType())->getNumElements() == 1)
     return translateCopy(U, *U.getOperand(1), MIRBuilder);
 
   Register Res = getOrCreateVReg(U);
   Register Val = getOrCreateVReg(*U.getOperand(0));
   Register Elt = getOrCreateVReg(*U.getOperand(1));
   Register Idx = getOrCreateVReg(*U.getOperand(2));
   MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
   return true;
 }
 
 bool IRTranslator::translateExtractElement(const User &U,
                                            MachineIRBuilder &MIRBuilder) {
   // If it is a <1 x Ty> vector, use the scalar as it is
   // not a legal vector type in LLT.
-  if (cast<VectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
+  if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
     return translateCopy(U, *U.getOperand(0), MIRBuilder);
 
   Register Res = getOrCreateVReg(U);
   Register Val = getOrCreateVReg(*U.getOperand(0));
   const auto &TLI = *MF->getSubtarget().getTargetLowering();
   unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
   Register Idx;
   if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {

llvm/lib/CodeGen/InterleavedAccessPass.cpp

for (auto UI = LI->user_begin(), E = LI->user_end(); UI != E; UI++) {
     Shuffles.push_back(SVI);
   }
 
   if (Shuffles.empty())
     return false;
 
   unsigned Factor, Index;
 
-  unsigned NumLoadElements = cast<VectorType>(LI->getType())->getNumElements();
+  unsigned NumLoadElements =
+      cast<FixedVectorType>(LI->getType())->getNumElements();
   // Check if the first shufflevector is DE-interleave shuffle.
   if (!isDeInterleaveMask(Shuffles[0]->getShuffleMask(), Factor, Index,
                           MaxFactor, NumLoadElements))
     return false;
 
   // Holds the corresponding index for each DE-interleave shuffle.
   SmallVector<unsigned, 4> Indices;
   Indices.push_back(Index);
bool InterleavedAccess::lowerInterleavedStore(
 
   ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());
   if (!SVI || !SVI->hasOneUse())
     return false;
 
   // Check if the shufflevector is RE-interleave shuffle.
   unsigned Factor;
   unsigned OpNumElts =
-      cast<VectorType>(SVI->getOperand(0)->getType())->getNumElements();
+      cast<FixedVectorType>(SVI->getOperand(0)->getType())->getNumElements();
   if (!isReInterleaveMask(SVI->getShuffleMask(), Factor, MaxFactor, OpNumElts))
     return false;
 
   LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *SI << "\n");
 
   // Try to create target specific intrinsics to replace the store and shuffle.
   if (!TLI->lowerInterleavedStore(SI, SVI, Factor))
     return false;

llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp

if (!DT.dominates(InsertionPoint, VI.SVI))
       return false;
   }
 
   // All checks are done. Add instructions detectable by InterleavedAccessPass
   // The old instruction will are left dead.
   IRBuilder<> Builder(InsertionPoint);
   Type *ETy = InterleavedLoad.front().SVI->getType()->getElementType();
   unsigned ElementsPerSVI =
-      InterleavedLoad.front().SVI->getType()->getNumElements();
+      cast<FixedVectorType>(InterleavedLoad.front().SVI->getType())
+          ->getNumElements();
   FixedVectorType *ILTy = FixedVectorType::get(ETy, Factor * ElementsPerSVI);
 
   SmallVector<unsigned, 4> Indices;
   for (unsigned i = 0; i < Factor; i++)
     Indices.push_back(i);
   InterleavedCost = TTI.getInterleavedMemoryOpCost(
       Instruction::Load, ILTy, Factor, Indices, InsertionPoint->getAlign(),
       InsertionPoint->getPointerAddressSpace());

llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp

FunctionPass *llvm::createScalarizeMaskedMemIntrinPass() {
   return new ScalarizeMaskedMemIntrin();
 }
 
 static bool isConstantIntVector(Value *Mask) {
   Constant *C = dyn_cast<Constant>(Mask);
   if (!C)
     return false;
 
-  unsigned NumElts = cast<VectorType>(Mask->getType())->getNumElements();
+  unsigned NumElts = cast<FixedVectorType>(Mask->getType())->getNumElements();
   for (unsigned i = 0; i != NumElts; ++i) {
     Constant *CElt = C->getAggregateElement(i);
     if (!CElt || !isa<ConstantInt>(CElt))
       return false;
   }
 
   return true;
 }

 //
 static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
   Value *Src0 = CI->getArgOperand(3);
 
   const Align AlignVal = cast<ConstantInt>(Alignment)->getAlignValue();
-  VectorType *VecType = cast<VectorType>(CI->getType());
+  VectorType *VecType = cast<FixedVectorType>(CI->getType());
 
   Type *EltTy = VecType->getElementType();
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
   BasicBlock *IfBlock = CI->getParent();
 
   Builder.SetInsertPoint(InsertPt);
static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
 
   // Adjust alignment for the scalar instruction.
   const Align AdjustedAlignVal =
       commonAlignment(AlignVal, EltTy->getPrimitiveSizeInBits() / 8);
   // Bitcast %addr from i8* to EltTy*
   Type *NewPtrType =
       EltTy->getPointerTo(Ptr->getType()->getPointerAddressSpace());
   Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
-  unsigned VectorWidth = VecType->getNumElements();
+  unsigned VectorWidth = cast<FixedVectorType>(VecType)->getNumElements();
 
   // The result vector
   Value *VResult = Src0;
 
   if (isConstantIntVector(Mask)) {
     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
       if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
         continue;

 //   . . .
 static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
   Value *Mask = CI->getArgOperand(3);
 
   const Align AlignVal = cast<ConstantInt>(Alignment)->getAlignValue();
-  VectorType *VecType = cast<VectorType>(Src->getType());
+  auto *VecType = cast<VectorType>(Src->getType());
 
   Type *EltTy = VecType->getElementType();
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
   BasicBlock *IfBlock = CI->getParent();
   Builder.SetInsertPoint(InsertPt);
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 
   // Short-cut if the mask is all-true.
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
     Builder.CreateAlignedStore(Src, Ptr, AlignVal);
     CI->eraseFromParent();
     return;
   }
 
   // Adjust alignment for the scalar instruction.
   const Align AdjustedAlignVal =
       commonAlignment(AlignVal, EltTy->getPrimitiveSizeInBits() / 8);
   // Bitcast %addr from i8* to EltTy*
   Type *NewPtrType =
       EltTy->getPointerTo(Ptr->getType()->getPointerAddressSpace());
   Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
-  unsigned VectorWidth = VecType->getNumElements();
+  unsigned VectorWidth = cast<FixedVectorType>(VecType)->getNumElements();
 
   if (isConstantIntVector(Mask)) {
     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
       if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
         continue;
       Value *OneElt = Builder.CreateExtractElement(Src, Idx);
       Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx);
       Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);

 // %Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src
 // ret <16 x i32> %Result
 static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
   Value *Ptrs = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
   Value *Src0 = CI->getArgOperand(3);
 
-  VectorType *VecType = cast<VectorType>(CI->getType());
+  auto *VecType = cast<FixedVectorType>(CI->getType());
   Type *EltTy = VecType->getElementType();
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
   BasicBlock *IfBlock = CI->getParent();
   Builder.SetInsertPoint(InsertPt);
   MaybeAlign AlignVal = cast<ConstantInt>(Alignment)->getMaybeAlignValue();
 

 // store i32 %Elt1, i32* %Ptr1, align 4
 // br label %else2
 //   . . .
 static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptrs = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
   Value *Mask = CI->getArgOperand(3);
 
-  assert(isa<VectorType>(Src->getType()) &&
-         "Unexpected data type in masked scatter intrinsic");
+  auto *SrcFVTy = cast<FixedVectorType>(Src->getType());
+
   assert(
       isa<VectorType>(Ptrs->getType()) &&
       isa<PointerType>(cast<VectorType>(Ptrs->getType())->getElementType()) &&
       "Vector of pointers is expected in masked scatter intrinsic");
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
   BasicBlock *IfBlock = CI->getParent();
   Builder.SetInsertPoint(InsertPt);
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 
   MaybeAlign AlignVal = cast<ConstantInt>(Alignment)->getMaybeAlignValue();
-  unsigned VectorWidth = cast<VectorType>(Src->getType())->getNumElements();
+  unsigned VectorWidth = SrcFVTy->getNumElements();
 
   // Shorten the way if the mask is a vector of constants.
   if (isConstantIntVector(Mask)) {
     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
       if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
         continue;
       Value *OneElt =
           Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
   ModifiedDT = true;
 }
 
 static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Mask = CI->getArgOperand(1);
   Value *PassThru = CI->getArgOperand(2);
 
-  VectorType *VecType = cast<VectorType>(CI->getType());
+  auto *VecType = cast<FixedVectorType>(CI->getType());
 
   Type *EltTy = VecType->getElementType();
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
   BasicBlock *IfBlock = CI->getParent();
 
   Builder.SetInsertPoint(InsertPt);
static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
   ModifiedDT = true;
 }
 
 static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
 
-  VectorType *VecType = cast<VectorType>(Src->getType());
+  auto *VecType = cast<FixedVectorType>(Src->getType());
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
   BasicBlock *IfBlock = CI->getParent();
 
   Builder.SetInsertPoint(InsertPt);
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
   // Handle splat constant pointer.
   if (auto *C = dyn_cast<Constant>(Ptr)) {
     C = C->getSplatValue();
     if (!C)
       return false;
 
     Base = SDB->getValue(C);
 
-    unsigned NumElts = cast<VectorType>(Ptr->getType())->getNumElements();
+    unsigned NumElts = cast<FixedVectorType>(Ptr->getType())->getNumElements();
     EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
     Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
     IndexType = ISD::SIGNED_SCALED;
     Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
     return true;
   }
 
   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);

llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp

if (isa<UndefValue>(C)) {
     return APIntToHexString(APInt::getNullValue(Ty->getPrimitiveSizeInBits()));
   } else if (const auto *CFP = dyn_cast<ConstantFP>(C)) {
     return APIntToHexString(CFP->getValueAPF().bitcastToAPInt());
   } else if (const auto *CI = dyn_cast<ConstantInt>(C)) {
     return APIntToHexString(CI->getValue());
   } else {
     unsigned NumElements;
     if (auto *VTy = dyn_cast<VectorType>(Ty))
-      NumElements = VTy->getNumElements();
+      NumElements = cast<FixedVectorType>(VTy)->getNumElements();
     else
       NumElements = Ty->getArrayNumElements();
     std::string HexString;
     for (int I = NumElements - 1, E = -1; I != E; --I)
       HexString += scalarConstantToHexString(C->getAggregateElement(I));
     return HexString;
   }
 }

llvm/lib/CodeGen/ValueTypes.cpp

 
 EVT EVT::getExtendedVectorElementType() const {
   assert(isExtended() && "Type is not extended!");
   return EVT::getEVT(cast<VectorType>(LLVMTy)->getElementType());
 }
 
 unsigned EVT::getExtendedVectorNumElements() const {
   assert(isExtended() && "Type is not extended!");
-  return cast<VectorType>(LLVMTy)->getNumElements();
+  return cast<FixedVectorType>(LLVMTy)->getNumElements();
 }
 
 ElementCount EVT::getExtendedVectorElementCount() const {
   assert(isExtended() && "Type is not extended!");
   return cast<VectorType>(LLVMTy)->getElementCount();
 }
 
 TypeSize EVT::getExtendedSizeInBits() const {