diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1125,10 +1125,11 @@ int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index = -1) const; /// \return The cost of Load and Store instructions. - int getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput, - const Instruction *I = nullptr) const; + InstructionCost + getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput, + const Instruction *I = nullptr) const; /// \return The cost of masked Load and Store instructions. InstructionCost getMaskedMemoryOpCost( @@ -1585,10 +1586,11 @@ const Instruction *I) = 0; virtual int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) = 0; - virtual int getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) = 0; + virtual InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + Align Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) = 0; virtual InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, @@ -2061,9 +2063,10 @@ int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) override { return Impl.getVectorInstrCost(Opcode, Val, Index); } - int getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, TTI::TargetCostKind CostKind, - const Instruction *I) override { + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) override { return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind, I); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -533,9 +533,10 @@ return 1; } - unsigned getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, TTI::TargetCostKind CostKind, - const Instruction *I) const { + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) const { return 1; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -955,10 +955,10 @@ return LT.first; } - unsigned getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) { + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr) { assert(!Src->isVoidTy() && "Invalid type"); // Assume types, such as structs, are expensive. if (getTLI()->getValueType(DL, Src, true) == MVT::Other) @@ -966,7 +966,7 @@ std::pair LT = getTLI()->getTypeLegalizationCost(DL, Src); // Assuming that all loads of legal types cost 1. - unsigned Cost = LT.first; + InstructionCost Cost = LT.first; if (CostKind != TTI::TCK_RecipThroughput) return Cost; @@ -1009,7 +1009,7 @@ // // First, compute the cost of extracting the individual addresses and the // individual memory operations. - int LoadCost = + InstructionCost LoadCost = VT->getNumElements() * (getVectorInstrCost( Instruction::ExtractElement, diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -807,14 +807,13 @@ return Cost; } -int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src, - Align Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) const { +InstructionCost TargetTransformInfo::getMemoryOpCost( + unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, const Instruction *I) const { assert((I == nullptr || I->getOpcode() == Opcode) && "Opcode should reflect passed instruction."); - int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind, I); + InstructionCost Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, + AddressSpace, CostKind, I); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -175,10 +175,10 @@ bool IsZeroCmp) const; bool useNeonVector(const Type *Ty) const; - int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); int getCostOfKeepingLiveOverCall(ArrayRef Tys); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -930,7 +930,7 @@ Optional MaxNumVScale = getMaxVScale(); assert(MaxNumVScale && "Expected valid max vscale value"); - unsigned MemOpCost = + InstructionCost MemOpCost = getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I); unsigned MaxNumElementsPerGather = MaxNumVScale.getValue() * LegalVF.getKnownMinValue(); @@ -941,10 +941,11 @@ return isa(Ty) && !ST->useSVEForFixedLengthVectors(); } -int AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, - MaybeAlign Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) { +InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, + MaybeAlign Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) { // TODO: Handle other cost kinds. if (CostKind != TTI::TCK_RecipThroughput) return 1; @@ -1017,7 +1018,7 @@ } int AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef Tys) { - int Cost = 0; + InstructionCost Cost = 0; TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; for (auto *I : Tys) { if (!I->isVectorTy()) @@ -1027,7 +1028,7 @@ Cost += getMemoryOpCost(Instruction::Store, I, Align(128), 0, CostKind) + getMemoryOpCost(Instruction::Load, I, Align(128), 0, CostKind); } - return Cost; + return *Cost.getValue(); } unsigned AArch64TTIImpl::getMaxInterleaveFactor(unsigned VF) { diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -226,10 +226,10 @@ ArrayRef Args = ArrayRef(), const Instruction *CxtI = nullptr); - int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1381,10 +1381,11 @@ return BaseCost; } -int ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, - MaybeAlign Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) { +InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) { // TODO: Handle other cost kinds. if (CostKind != TTI::TCK_RecipThroughput) return 1; diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -114,10 +114,10 @@ TTI::TargetCostKind CostKind); unsigned getAddressComputationCost(Type *Tp, ScalarEvolution *SE, const SCEV *S); - unsigned getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -154,11 +154,11 @@ return 0; } -unsigned HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, - MaybeAlign Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) { +InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) { assert(Opcode == Instruction::Load || Opcode == Instruction::Store); // TODO: Handle other cost kinds. if (CostKind != TTI::TCK_RecipThroughput) @@ -240,8 +240,9 @@ Alignment, AddressSpace, CostKind, UseMaskForCond, UseMaskForGaps); - return getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace, - CostKind); + return *getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace, + CostKind) + .getValue(); } InstructionCost HexagonTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h @@ -120,10 +120,10 @@ TTI::TargetCostKind CostKind, const Instruction *I = nullptr); int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); - int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); int getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, Align Alignment, unsigned AddressSpace, diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -1104,10 +1104,11 @@ return Cost; } -int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, - MaybeAlign Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) { +InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) { if (TLI->getValueType(DL, Src, true) == MVT::Other) return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); @@ -1116,8 +1117,8 @@ assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && "Invalid Opcode"); - int Cost = BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind); + InstructionCost Cost = + BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); // TODO: Handle other cost kinds. if (CostKind != TTI::TCK_RecipThroughput) return Cost; @@ -1200,9 +1201,8 @@ std::pair LT = TLI->getTypeLegalizationCost(DL, VecTy); // Firstly, the cost of load/store operation. - int Cost = - getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace, - CostKind); + InstructionCost Cost = getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), + AddressSpace, CostKind); // PPC, for both Altivec/VSX, support cheap arbitrary permutations // (at least in the sense that there need only be one non-loop-invariant @@ -1211,7 +1211,7 @@ // because it does not need to take itself). Cost += Factor*(LT.first-1); - return Cost; + return *Cost.getValue(); } InstructionCost diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h @@ -103,9 +103,10 @@ const Instruction *I = nullptr); int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue); - int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, - unsigned AddressSpace, TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); int getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -246,7 +246,7 @@ // Find out if L contains a call, what the machine instruction count // estimate is, and how many stores there are. bool HasCall = false; - unsigned NumStores = 0; + InstructionCost NumStores = 0; for (auto &BB : L->blocks()) for (auto &I : *BB) { if (isa(&I) || isa(&I)) { @@ -270,7 +270,8 @@ // The z13 processor will run out of store tags if too many stores // are fed into it too quickly. Therefore make sure there are not // too many stores in the resulting unrolled loop. - unsigned const Max = (NumStores ? (12 / NumStores) : UINT_MAX); + unsigned const NumStoresVal = *NumStores.getValue(); + unsigned const Max = (NumStoresVal ? (12 / NumStoresVal) : UINT_MAX); if (HasCall) { // Only allow full unrolling if loop has any calls. @@ -1059,10 +1060,11 @@ return false; } -int SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, - MaybeAlign Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) { +InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) { assert(!Src->isVoidTy() && "Invalid type"); // TODO: Handle other cost kinds. diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -140,10 +140,10 @@ int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); unsigned getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract); - int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, - unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, @@ -239,8 +239,9 @@ InstructionCost getGSScalarCost(unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment, unsigned AddressSpace); - int getGSVectorCost(unsigned Opcode, Type *DataTy, const Value *Ptr, - Align Alignment, unsigned AddressSpace); + InstructionCost getGSVectorCost(unsigned Opcode, Type *DataTy, + const Value *Ptr, Align Alignment, + unsigned AddressSpace); int getGatherOverhead() const; int getScatterOverhead() const; diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3192,10 +3192,11 @@ return Cost; } -int X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, - MaybeAlign Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I) { +InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, + MaybeAlign Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) { // TODO: Handle other cost kinds. if (CostKind != TTI::TCK_RecipThroughput) { if (auto *SI = dyn_cast_or_null(I)) { @@ -3227,8 +3228,8 @@ // Assume that all other non-power-of-two numbers are scalarized. if (!isPowerOf2_32(NumElem)) { APInt DemandedElts = APInt::getAllOnesValue(NumElem); - int Cost = BaseT::getMemoryOpCost(Opcode, VTy->getScalarType(), Alignment, - AddressSpace, CostKind); + InstructionCost Cost = BaseT::getMemoryOpCost( + Opcode, VTy->getScalarType(), Alignment, AddressSpace, CostKind); int SplitCost = getScalarizationOverhead(VTy, DemandedElts, Opcode == Instruction::Load, Opcode == Instruction::Store); @@ -3286,11 +3287,10 @@ InstructionCost MaskCmpCost = NumElem * (BranchCost + ScalarCompareCost); InstructionCost ValueSplitCost = getScalarizationOverhead(SrcVTy, DemandedElts, IsLoad, IsStore); - int MemopCost = + InstructionCost MemopCost = NumElem * BaseT::getMemoryOpCost(Opcode, SrcVTy->getScalarType(), Alignment, AddressSpace, CostKind); - return *(MemopCost + ValueSplitCost + MaskSplitCost + MaskCmpCost) - .getValue(); + return MemopCost + ValueSplitCost + MaskSplitCost + MaskCmpCost; } // Legalize the type. @@ -4116,8 +4116,9 @@ // Return an average cost of Gather / Scatter instruction, maybe improved later. // FIXME: Add TargetCostKind support. -int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, const Value *Ptr, - Align Alignment, unsigned AddressSpace) { +InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, + const Value *Ptr, Align Alignment, + unsigned AddressSpace) { assert(isa(SrcVTy) && "Unexpected type in getGSVectorCost"); unsigned VF = cast(SrcVTy)->getNumElements(); @@ -4210,9 +4211,9 @@ } // The cost of the scalar loads/stores. - int MemoryOpCost = VF * getMemoryOpCost(Opcode, SrcVTy->getScalarType(), - MaybeAlign(Alignment), AddressSpace, - CostKind); + InstructionCost MemoryOpCost = + VF * getMemoryOpCost(Opcode, SrcVTy->getScalarType(), + MaybeAlign(Alignment), AddressSpace, CostKind); int InsertExtractCost = 0; if (Opcode == Instruction::Load) @@ -4562,9 +4563,8 @@ // Get the cost of one memory operation. auto *SingleMemOpTy = FixedVectorType::get(VecTy->getElementType(), LegalVT.getVectorNumElements()); - unsigned MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy, - MaybeAlign(Alignment), AddressSpace, - CostKind); + InstructionCost MemOpCost = getMemoryOpCost( + Opcode, SingleMemOpTy, MaybeAlign(Alignment), AddressSpace, CostKind); auto *VT = FixedVectorType::get(ScalarTy, VF); EVT ETy = TLI->getValueType(DL, VT); @@ -4620,13 +4620,13 @@ if (Opcode == Instruction::Load) { if (const auto *Entry = CostTableLookup(AVX2InterleavedLoadTbl, Factor, ETy.getSimpleVT())) - return NumOfMemOps * MemOpCost + Entry->Cost; + return NumOfMemOps * *MemOpCost.getValue() + Entry->Cost; } else { assert(Opcode == Instruction::Store && "Expected Store Instruction at this point"); if (const auto *Entry = CostTableLookup(AVX2InterleavedStoreTbl, Factor, ETy.getSimpleVT())) - return NumOfMemOps * MemOpCost + Entry->Cost; + return NumOfMemOps * *MemOpCost.getValue() + Entry->Cost; } return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, @@ -4661,9 +4661,8 @@ // Get the cost of one memory operation. auto *SingleMemOpTy = FixedVectorType::get(VecTy->getElementType(), LegalVT.getVectorNumElements()); - unsigned MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy, - MaybeAlign(Alignment), AddressSpace, - CostKind); + InstructionCost MemOpCost = getMemoryOpCost( + Opcode, SingleMemOpTy, MaybeAlign(Alignment), AddressSpace, CostKind); unsigned VF = VecTy->getNumElements() / Factor; MVT VT = MVT::getVectorVT(MVT::getVT(VecTy->getScalarType()), VF); @@ -4683,7 +4682,7 @@ if (const auto *Entry = CostTableLookup(AVX512InterleavedLoadTbl, Factor, VT)) - return NumOfMemOps * MemOpCost + Entry->Cost; + return NumOfMemOps * *MemOpCost.getValue() + Entry->Cost; //If an entry does not exist, fallback to the default implementation. // Kind of shuffle depends on number of loaded values. @@ -4720,7 +4719,7 @@ NumOfMoves = NumOfResults * NumOfShufflesPerResult / 2; int Cost = NumOfResults * NumOfShufflesPerResult * ShuffleCost + - NumOfUnfoldedLoads * MemOpCost + NumOfMoves; + NumOfUnfoldedLoads * *MemOpCost.getValue() + NumOfMoves; return Cost; } @@ -4742,7 +4741,7 @@ if (const auto *Entry = CostTableLookup(AVX512InterleavedStoreTbl, Factor, VT)) - return NumOfMemOps * MemOpCost + Entry->Cost; + return NumOfMemOps * *MemOpCost.getValue() + Entry->Cost; //If an entry does not exist, fallback to the default implementation. // There is no strided stores meanwhile. And store can't be folded in @@ -4755,7 +4754,8 @@ // The SK_MergeTwoSrc shuffle clobbers one of src operands. // We need additional instructions to keep sources. unsigned NumOfMoves = NumOfMemOps * NumOfShufflesPerStore / 2; - int Cost = NumOfMemOps * (MemOpCost + NumOfShufflesPerStore * ShuffleCost) + + int Cost = NumOfMemOps * + (*MemOpCost.getValue() + NumOfShufflesPerStore * ShuffleCost) + NumOfMoves; return Cost; }