Index: llvm/include/llvm/Analysis/TargetTransformInfo.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfo.h +++ llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -606,6 +606,10 @@ /// Return true if the target supports masked expand load. bool isLegalMaskedExpandLoad(Type *DataType) const; + /// Returns true if the target machine can represent a vectorized version + /// of \p V as a masked load or store operation. This assumes that the memory + /// is contiguous. + bool isLegalMaskedLoadStore(Value *V) const; /// Returns true if the target machine can represent a vectorized version /// of \p V as a masked gather or scatter operation. bool isLegalGatherOrScatter(Value *V) const; Index: llvm/lib/Analysis/TargetTransformInfo.cpp =================================================================== --- llvm/lib/Analysis/TargetTransformInfo.cpp +++ llvm/lib/Analysis/TargetTransformInfo.cpp @@ -333,6 +333,17 @@ return TTIImpl->isLegalMaskedExpandLoad(DataType); } +bool TargetTransformInfo::isLegalMaskedLoadStore(Value *V) const { + LoadInst *LI = dyn_cast(V); + StoreInst *SI = dyn_cast(V); + if (!LI && !SI) + return false; + Type *Ty = LI ? LI->getType() : SI->getValueOperand()->getType(); + MaybeAlign Align = getLoadStoreAlignment(V); + return (LI && isLegalMaskedLoad(Ty, Align)) || + (SI && isLegalMaskedStore(Ty, Align)); +} + bool TargetTransformInfo::isLegalGatherOrScatter(Value *V) const { LoadInst *LI = dyn_cast(V); StoreInst *SI = dyn_cast(V); @@ -341,7 +352,7 @@ Type *Ty = LI ? LI->getType() : SI->getValueOperand()->getType(); MaybeAlign Align = getLoadStoreAlignment(V); return (LI && isLegalMaskedGather(Ty, Align)) || - (SI && isLegalMaskedScatter(Ty, Align)); + (SI && isLegalMaskedScatter(Ty, Align)); } bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1193,20 +1193,6 @@ collectLoopScalars(VF); } - /// Returns true if the target machine supports masked store operation - /// for the given \p DataType and kind of access to \p Ptr. - bool isLegalMaskedStore(Type *DataType, Value *Ptr, MaybeAlign Alignment) { - return Legal->isConsecutivePtr(Ptr) && - TTI.isLegalMaskedStore(DataType, Alignment); - } - - /// Returns true if the target machine supports masked load operation - /// for the given \p DataType and kind of access to \p Ptr. - bool isLegalMaskedLoad(Type *DataType, Value *Ptr, MaybeAlign Alignment) { - return Legal->isConsecutivePtr(Ptr) && - TTI.isLegalMaskedLoad(DataType, Alignment); - } - /// Returns true if \p I is an instruction that will be scalarized with /// predication. Such instructions include conditional stores and /// instructions that may divide by zero. @@ -4583,7 +4569,6 @@ if (!Legal->isMaskRequired(I)) return false; auto *Ptr = getLoadStorePointerOperand(I); - auto *Ty = getMemInstValueType(I); // We have already decided how to vectorize this instruction, get that // result. if (VF > 1) { @@ -4592,11 +4577,10 @@ "Widening decision should be ready at this moment"); return WideningDecision == CM_Scalarize; } - const MaybeAlign Alignment = getLoadStoreAlignment(I); - bool LegalGather = TTI.isLegalGatherOrScatter(I); - return !(LegalGather || isa(I) - ? isLegalMaskedLoad(Ty, Ptr, Alignment) - : isLegalMaskedStore(Ty, Ptr, Alignment)); + bool LegalLoadStore = + Legal->isConsecutivePtr(Ptr) && TTI.isLegalMaskedLoadStore(I); + bool LegalGatherScatter = TTI.isLegalGatherOrScatter(I); + return !(LegalLoadStore || LegalGatherScatter); } case Instruction::UDiv: case Instruction::SDiv: @@ -4638,10 +4622,7 @@ assert(useMaskedInterleavedAccesses(TTI) && "Masked interleave-groups for predicated accesses are not enabled."); - auto *Ty = getMemInstValueType(I); - const MaybeAlign Alignment = getLoadStoreAlignment(I); - return isa(I) ? TTI.isLegalMaskedLoad(Ty, Alignment) - : TTI.isLegalMaskedStore(Ty, Alignment); + return TTI.isLegalMaskedLoadStore(I); } bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,