diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1049,9 +1049,9 @@ /// extraction shuffle kinds to show the insert/extract point and the type of /// the subvector being inserted/extracted. /// NOTE: For subvector extractions Tp represents the source type. - int getShuffleCost(ShuffleKind Kind, VectorType *Tp, - ArrayRef Mask = None, int Index = 0, - VectorType *SubTp = nullptr) const; + InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask = None, int Index = 0, + VectorType *SubTp = nullptr) const; /// Represents a hint about the context in which a cast is used. /// @@ -1569,9 +1569,9 @@ OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo, OperandValueProperties Opd2PropInfo, ArrayRef Args, const Instruction *CxtI = nullptr) = 0; - virtual int getShuffleCost(ShuffleKind Kind, VectorType *Tp, - ArrayRef Mask, int Index, - VectorType *SubTp) = 0; + virtual InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp) = 0; virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH, TTI::TargetCostKind CostKind, @@ -2038,8 +2038,9 @@ return Impl.getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo, Args, CxtI); } - int getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, - int Index, VectorType *SubTp) override { + InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp) override { return Impl.getShuffleCost(Kind, Tp, Mask, Index, SubTp); } InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -463,9 +463,9 @@ return 1; } - unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Ty, - ArrayRef Mask, int Index, - VectorType *SubTp) const { + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Ty, + ArrayRef Mask, int Index, + VectorType *SubTp) const { return 1; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -717,8 +717,9 @@ return OpCost; } - unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, - ArrayRef Mask, int Index, VectorType *SubTp) { + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp) { switch (Kind) { case TTI::SK_Broadcast: @@ -1934,8 +1935,8 @@ CmpInst::BAD_ICMP_PREDICATE, CostKind); } unsigned NumReduxLevels = Log2_32(NumVecElts); - unsigned ArithCost = 0; - unsigned ShuffleCost = 0; + InstructionCost ArithCost = 0; + InstructionCost ShuffleCost = 0; std::pair LT = thisT()->getTLI()->getTypeLegalizationCost(DL, Ty); unsigned LongVectorCount = 0; @@ -1991,7 +1992,7 @@ CmpOpcode = Instruction::ICmp; } InstructionCost MinMaxCost = 0; - unsigned ShuffleCost = 0; + InstructionCost ShuffleCost = 0; std::pair LT = thisT()->getTLI()->getTypeLegalizationCost(DL, Ty); unsigned LongVectorCount = 0; diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -711,10 +711,12 @@ return Cost; } -int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, VectorType *Ty, - ArrayRef Mask, int Index, - VectorType *SubTp) const { - int Cost = TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp); +InstructionCost TargetTransformInfo::getShuffleCost(ShuffleKind Kind, + VectorType *Ty, + ArrayRef Mask, + int Index, + VectorType *SubTp) const { + InstructionCost Cost = TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -282,8 +282,9 @@ unsigned Opcode, VectorType *Ty, bool IsPairwiseForm, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput); - int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, - int Index, VectorType *SubTp); + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp); /// @} }; diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -1336,9 +1336,10 @@ CostKind); } -int AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, - ArrayRef Mask, int Index, - VectorType *SubTp) { +InstructionCost AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp) { if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose || Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc || Kind == TTI::SK_Reverse) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h @@ -197,8 +197,9 @@ unsigned getVectorSplitCost() { return 0; } - unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, - ArrayRef Mask, int Index, VectorType *SubTp); + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp); bool areInlineCompatible(const Function *Caller, const Function *Callee) const; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -1114,9 +1114,9 @@ } } -unsigned GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *VT, - ArrayRef Mask, int Index, - VectorType *SubTp) { +InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *VT, ArrayRef Mask, + int Index, VectorType *SubTp) { if (ST->hasVOP3PInsts()) { if (cast(VT)->getNumElements() == 2 && DL.getTypeSizeInBits(VT->getElementType()) == 16) { diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -187,8 +187,9 @@ int getNumMemOps(const IntrinsicInst *I) const; - int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, - int Index, VectorType *SubTp); + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp); bool preferInLoopReduction(unsigned Opcode, Type *Ty, TTI::ReductionFlags Flags) const; diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1133,9 +1133,9 @@ return NumOps; } -int ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, - ArrayRef Mask, int Index, - VectorType *SubTp) { +InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *Tp, ArrayRef Mask, + int Index, VectorType *SubTp) { if (ST->hasNEON()) { if (Kind == TTI::SK_Broadcast) { static const CostTblEntry NEONDupTbl[] = { diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -123,8 +123,8 @@ getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency); - unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, ArrayRef Mask, - int Index, Type *SubTp); + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, + ArrayRef Mask, int Index, Type *SubTp); InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -219,9 +219,9 @@ CostKind); } -unsigned HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, - ArrayRef Mask, int Index, - Type *SubTp) { +InstructionCost HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, + ArrayRef Mask, int Index, + Type *SubTp) { return 1; } diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h @@ -108,8 +108,8 @@ TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, ArrayRef Args = ArrayRef(), const Instruction *CxtI = nullptr); - int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, ArrayRef Mask, - int Index, Type *SubTp); + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, + ArrayRef Mask, int Index, Type *SubTp); InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -987,8 +987,9 @@ return *vectorCostAdjustment(Cost, Opcode, Ty, nullptr).getValue(); } -int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, - ArrayRef Mask, int Index, Type *SubTp) { +InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, + ArrayRef Mask, int Index, + Type *SubTp) { // Legalize the type. std::pair LT = TLI->getTypeLegalizationCost(DL, Tp); @@ -997,9 +998,8 @@ // instruction). We need one such shuffle instruction for each actual // register (this is not true for arbitrary shuffles, but is true for the // structured types of shuffles covered by TTI::ShuffleKind). - return *vectorCostAdjustment(LT.first, Instruction::ShuffleVector, Tp, - nullptr) - .getValue(); + return vectorCostAdjustment(LT.first, Instruction::ShuffleVector, Tp, + nullptr); } InstructionCost PPCTTIImpl::getCFInstrCost(unsigned Opcode, diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h @@ -87,8 +87,9 @@ TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, ArrayRef Args = ArrayRef(), const Instruction *CxtI = nullptr); - int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, - int Index, VectorType *SubTp); + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp); unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy); unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy); unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst, diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -554,9 +554,10 @@ Opd1PropInfo, Opd2PropInfo, Args, CxtI); } -int SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, - ArrayRef Mask, int Index, - VectorType *SubTp) { +InstructionCost SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp) { if (ST->hasVector()) { unsigned NumVectors = getNumVectorRegs(Tp); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -127,8 +127,9 @@ TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, ArrayRef Args = ArrayRef(), const Instruction *CxtI = nullptr); - int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, - int Index, VectorType *SubTp); + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, + ArrayRef Mask, int Index, + VectorType *SubTp); InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -963,9 +963,10 @@ return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info); } -int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp, - ArrayRef Mask, int Index, - VectorType *SubTp) { +InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, + VectorType *BaseTp, + ArrayRef Mask, int Index, + VectorType *SubTp) { // 64-bit packed float vectors (v2f32) are widened to type v4f32. // 64-bit packed integer vectors (v2i32) are widened to type v4i32. std::pair LT = TLI->getTypeLegalizationCost(DL, BaseTp); @@ -1011,8 +1012,8 @@ auto *SubTy = FixedVectorType::get(BaseTp->getElementType(), SubLT.second.getVectorNumElements()); int ExtractIndex = alignDown((Index % NumElts), NumSubElts); - int ExtractCost = getShuffleCost(TTI::SK_ExtractSubvector, VecTy, None, - ExtractIndex, SubTy); + InstructionCost ExtractCost = getShuffleCost( + TTI::SK_ExtractSubvector, VecTy, None, ExtractIndex, SubTy); // If the original size is 32-bits or more, we can use pshufd. Otherwise // if we have SSSE3 we can use pshufb. @@ -3088,7 +3089,7 @@ // subvector move(s). // If the vector type is already less than 128-bits then don't reduce it. // TODO: Under what circumstances should we shuffle using the full width? - int ShuffleCost = 1; + InstructionCost ShuffleCost = 1; if (Opcode == Instruction::InsertElement) { auto *SubTy = cast(Val); EVT VT = TLI->getValueType(DL, Val); @@ -3098,7 +3099,7 @@ getShuffleCost(TTI::SK_PermuteTwoSrc, SubTy, None, 0, SubTy); } int IntOrFpCost = ScalarType->isFloatingPointTy() ? 0 : 1; - return ShuffleCost + IntOrFpCost + RegisterFileMoveCost; + return *ShuffleCost.getValue() + IntOrFpCost + RegisterFileMoveCost; } // Add to the base cost if we know that the extracted element of a vector is @@ -3296,7 +3297,7 @@ // Legalize the type. std::pair LT = TLI->getTypeLegalizationCost(DL, SrcVTy); auto VT = TLI->getValueType(DL, SrcVTy); - int Cost = 0; + InstructionCost Cost = 0; if (VT.isSimple() && LT.second != VT.getSimpleVT() && LT.second.getVectorNumElements() == NumElem) // Promotion requires expand/truncate for data and a shuffle for mask. @@ -3530,7 +3531,7 @@ return BaseT::getArithmeticReductionCost(Opcode, ValVTy, IsPairwise, CostKind); - unsigned ReductionCost = 0; + InstructionCost ReductionCost = 0; auto *Ty = ValVTy; if (LT.first != 1 && MTy.isVector() && @@ -4691,7 +4692,7 @@ TTI::ShuffleKind ShuffleKind = (NumOfMemOps > 1) ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc; - unsigned ShuffleCost = + InstructionCost ShuffleCost = getShuffleCost(ShuffleKind, SingleMemOpTy, None, 0, nullptr); unsigned NumOfLoadsInInterleaveGrp = @@ -4747,7 +4748,7 @@ // There is no strided stores meanwhile. And store can't be folded in // shuffle. unsigned NumOfSources = Factor; // The number of values to be merged. - unsigned ShuffleCost = + InstructionCost ShuffleCost = getShuffleCost(TTI::SK_PermuteTwoSrc, SingleMemOpTy, None, 0, nullptr); unsigned NumOfShufflesPerStore = NumOfSources - 1;