diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -597,9 +597,9 @@ bool isLegalNTLoad(Type *DataType, Align Alignment) const; /// Return true if the target supports masked scatter. - bool isLegalMaskedScatter(Type *DataType) const; + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) const; /// Return true if the target supports masked gather. - bool isLegalMaskedGather(Type *DataType) const; + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) const; /// Return true if the target supports masked compress store. bool isLegalMaskedCompressStore(Type *DataType) const; @@ -1237,8 +1237,8 @@ virtual bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) = 0; virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0; virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0; - virtual bool isLegalMaskedScatter(Type *DataType) = 0; - virtual bool isLegalMaskedGather(Type *DataType) = 0; + virtual bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) = 0; + virtual bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) = 0; virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0; virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0; @@ -1537,11 +1537,11 @@ bool isLegalNTLoad(Type *DataType, Align Alignment) override { return Impl.isLegalNTLoad(DataType, Alignment); } - bool isLegalMaskedScatter(Type *DataType) override { - return Impl.isLegalMaskedScatter(DataType); + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) override { + return Impl.isLegalMaskedScatter(DataType, Alignment); } - bool isLegalMaskedGather(Type *DataType) override { - return Impl.isLegalMaskedGather(DataType); + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) override { + return Impl.isLegalMaskedGather(DataType, Alignment); } bool isLegalMaskedCompressStore(Type *DataType) override { return Impl.isLegalMaskedCompressStore(DataType); diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -272,9 +272,13 @@ return Alignment >= DataSize && isPowerOf2_32(DataSize); } - bool isLegalMaskedScatter(Type *DataType) { return false; } + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) { + return false; + } - bool isLegalMaskedGather(Type *DataType) { return false; } + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) { + return false; + } bool isLegalMaskedCompressStore(Type *DataType) { return false; } diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -315,12 +315,14 @@ return TTIImpl->isLegalNTLoad(DataType, Alignment); } -bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const { - return TTIImpl->isLegalMaskedGather(DataType); +bool TargetTransformInfo::isLegalMaskedGather(Type *DataType, + MaybeAlign Alignment) const { + return TTIImpl->isLegalMaskedGather(DataType, Alignment); } -bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const { - return TTIImpl->isLegalMaskedScatter(DataType); +bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType, + MaybeAlign Alignment) const { + return TTIImpl->isLegalMaskedScatter(DataType, Alignment); } bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const { diff --git a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp --- a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp +++ b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp @@ -849,21 +849,20 @@ bool &ModifiedDT) { IntrinsicInst *II = dyn_cast(CI); if (II) { + unsigned Alignment; switch (II->getIntrinsicID()) { default: break; case Intrinsic::masked_load: { // Scalarize unsupported vector masked load - unsigned Alignment = - cast(CI->getArgOperand(1))->getZExtValue(); + Alignment = cast(CI->getArgOperand(1))->getZExtValue(); if (TTI->isLegalMaskedLoad(CI->getType(), MaybeAlign(Alignment))) return false; scalarizeMaskedLoad(CI, ModifiedDT); return true; } case Intrinsic::masked_store: { - unsigned Alignment = - cast(CI->getArgOperand(2))->getZExtValue(); + Alignment = cast(CI->getArgOperand(2))->getZExtValue(); if (TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType(), MaybeAlign(Alignment))) return false; @@ -871,12 +870,15 @@ return true; } case Intrinsic::masked_gather: - if (TTI->isLegalMaskedGather(CI->getType())) + Alignment = cast(CI->getArgOperand(1))->getZExtValue(); + if (TTI->isLegalMaskedGather(CI->getType(), MaybeAlign(Alignment))) return false; scalarizeMaskedGather(CI, ModifiedDT); return true; case Intrinsic::masked_scatter: - if (TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType())) + Alignment = cast(CI->getArgOperand(2))->getZExtValue(); + if (TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType(), + MaybeAlign(Alignment))) return false; scalarizeMaskedScatter(CI, ModifiedDT); return true; diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -159,6 +159,10 @@ return isLegalMaskedLoad(DataTy, Alignment); } + bool isLegalMaskedGather(Type *Ty, MaybeAlign Alignment) { return false; } + + bool isLegalMaskedScatter(Type *Ty, MaybeAlign Alignment) { return false; } + int getMemcpyCost(const Instruction *I); int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -189,8 +189,8 @@ bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment); bool isLegalNTLoad(Type *DataType, Align Alignment); bool isLegalNTStore(Type *DataType, Align Alignment); - bool isLegalMaskedGather(Type *DataType); - bool isLegalMaskedScatter(Type *DataType); + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment); + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment); bool isLegalMaskedExpandLoad(Type *DataType); bool isLegalMaskedCompressStore(Type *DataType); bool hasDivRemOp(Type *DataType, bool IsSigned); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3296,8 +3296,10 @@ unsigned AddressSpace = PtrTy->getAddressSpace(); bool Scalarize = false; - if ((Opcode == Instruction::Load && !isLegalMaskedGather(SrcVTy)) || - (Opcode == Instruction::Store && !isLegalMaskedScatter(SrcVTy))) + if ((Opcode == Instruction::Load && + !isLegalMaskedGather(SrcVTy, MaybeAlign(Alignment))) || + (Opcode == Instruction::Store && + !isLegalMaskedScatter(SrcVTy, MaybeAlign(Alignment)))) Scalarize = true; // Gather / Scatter for vector 2 is not profitable on KNL / SKX // Vector-4 of gather/scatter instruction does not exist on KNL. @@ -3420,7 +3422,7 @@ return isLegalMaskedExpandLoad(DataTy); } -bool X86TTIImpl::isLegalMaskedGather(Type *DataTy) { +bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, MaybeAlign Alignment) { // Some CPUs have better gather performance than others. // TODO: Remove the explicit ST->hasAVX512()?, That would mean we would only // enable gather with a -march. @@ -3458,11 +3460,11 @@ return IntWidth == 32 || IntWidth == 64; } -bool X86TTIImpl::isLegalMaskedScatter(Type *DataType) { +bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) { // AVX2 doesn't support scatter if (!ST->hasAVX512()) return false; - return isLegalMaskedGather(DataType); + return isLegalMaskedGather(DataType, Alignment); } bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) { diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1210,14 +1210,14 @@ /// Returns true if the target machine supports masked scatter operation /// for the given \p DataType. - bool isLegalMaskedScatter(Type *DataType) { - return TTI.isLegalMaskedScatter(DataType); + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) { + return TTI.isLegalMaskedScatter(DataType, Alignment); } /// Returns true if the target machine supports masked gather operation /// for the given \p DataType. - bool isLegalMaskedGather(Type *DataType) { - return TTI.isLegalMaskedGather(DataType); + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) { + return TTI.isLegalMaskedGather(DataType, Alignment); } /// Returns true if the target machine can represent \p V as a masked gather @@ -1228,7 +1228,9 @@ if (!LI && !SI) return false; auto *Ty = getMemInstValueType(V); - return (LI && isLegalMaskedGather(Ty)) || (SI && isLegalMaskedScatter(Ty)); + MaybeAlign Align = getLoadStoreAlignment(V); + return (LI && isLegalMaskedGather(Ty, Align)) || + (SI && isLegalMaskedScatter(Ty, Align)); } /// Returns true if \p I is an instruction that will be scalarized with @@ -4579,9 +4581,10 @@ return WideningDecision == CM_Scalarize; } const MaybeAlign Alignment = getLoadStoreAlignment(I); - return isa(I) ? - !(isLegalMaskedLoad(Ty, Ptr, Alignment) || isLegalMaskedGather(Ty)) - : !(isLegalMaskedStore(Ty, Ptr, Alignment) || isLegalMaskedScatter(Ty)); + return isa(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) || + isLegalMaskedGather(Ty, Alignment)) + : !(isLegalMaskedStore(Ty, Ptr, Alignment) || + isLegalMaskedScatter(Ty, Alignment)); } case Instruction::UDiv: case Instruction::SDiv: