diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -851,6 +851,19 @@ case Instruction::ZExt: case Instruction::AddrSpaceCast: return TargetTTI->getCastInstrCost(Opcode, Ty, OpTy, CostKind, I); + case Instruction::Store: { + auto *SI = cast(U); + Type *ValTy = U->getOperand(0)->getType(); + return TargetTTI->getMemoryOpCost(Opcode, ValTy, SI->getAlign(), + SI->getPointerAddressSpace(), + CostKind, I); + } + case Instruction::Load: { + auto *LI = cast(U); + return TargetTTI->getMemoryOpCost(Opcode, U->getType(), LI->getAlign(), + LI->getPointerAddressSpace(), + CostKind, I); + } } // By default, just classify everything as 'basic'. return TTI::TCC_Basic; diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -888,6 +888,8 @@ // Assuming that all loads of legal types cost 1. unsigned Cost = LT.first; + if (CostKind != TTI::TCK_RecipThroughput) + return Cost; if (Src->isVectorTy() && Src->getPrimitiveSizeInBits() < LT.second.getSizeInBits()) { diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -1307,17 +1307,9 @@ return getCmpSelInstrCost(I->getOpcode(), ValTy, I->getType(), CostKind, I); } - case Instruction::Store: { - const StoreInst *SI = cast(I); - Type *ValTy = SI->getValueOperand()->getType(); - return getMemoryOpCost(I->getOpcode(), ValTy, SI->getAlign(), - SI->getPointerAddressSpace(), CostKind, I); - } - case Instruction::Load: { - const LoadInst *LI = cast(I); - return getMemoryOpCost(I->getOpcode(), I->getType(), LI->getAlign(), - LI->getPointerAddressSpace(), CostKind, I); - } + case Instruction::Store: + case Instruction::Load: + return getUserCost(I, CostKind); case Instruction::ZExt: case Instruction::SExt: case Instruction::FPToUI: diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -671,6 +671,10 @@ MaybeAlign Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, const Instruction *I) { + // TODO: Handle other cost kinds. + if (CostKind != TTI::TCK_RecipThroughput) + return 1; + auto LT = TLI->getTypeLegalizationCost(DL, Ty); if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store && diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -878,6 +878,10 @@ MaybeAlign Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, const Instruction *I) { + // TODO: Handle other cost kinds. + if (CostKind != TTI::TCK_RecipThroughput) + return 1; + std::pair LT = TLI->getTypeLegalizationCost(DL, Src); if (ST->hasNEON() && Src->isVectorTy() && diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -152,6 +152,10 @@ TTI::TargetCostKind CostKind, const Instruction *I) { assert(Opcode == Instruction::Load || Opcode == Instruction::Store); + // TODO: Handle other cost kinds. + if (CostKind != TTI::TCK_RecipThroughput) + return 1; + if (Opcode == Instruction::Store) return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind, I); diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -212,8 +212,12 @@ unsigned PPCTTIImpl::getUserCost(const User *U, ArrayRef Operands, TTI::TargetCostKind CostKind) { - // We already implement getCastInstrCost and perform the vector adjustment there. - if (!isa(U) && U->getType()->isVectorTy()) { + // We already implement getCastInstrCost and getMemoryOpCost where we perform + // the vector adjustment there. + if (isa(U) || isa(U) || isa(U)) + return BaseT::getUserCost(U, Operands, CostKind); + + if (U->getType()->isVectorTy()) { // Instructions that need to be split should cost more. std::pair LT = TLI->getTypeLegalizationCost(DL, U->getType()); return LT.first * BaseT::getUserCost(U, Operands, CostKind); @@ -862,6 +866,10 @@ int Cost = BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); + // TODO: Handle other cost kinds. + if (CostKind != TTI::TCK_RecipThroughput) + return Cost; + Cost = vectorCostAdjustment(Cost, Opcode, Src, nullptr); bool IsAltivecType = ST->hasAltivec() && diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -1029,6 +1029,10 @@ const Instruction *I) { assert(!Src->isVoidTy() && "Invalid type"); + // TODO: Handle other cost kinds. + if (CostKind != TTI::TCK_RecipThroughput) + return 1; + if (!Src->isVectorTy() && Opcode == Instruction::Load && I != nullptr) { // Store the load or its truncated or extended value in FoldedValue. const Instruction *FoldedValue = nullptr; diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -191,9 +191,6 @@ int getIntImmCost(const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind); - unsigned getUserCost(const User *U, ArrayRef Operands, - TTI::TargetCostKind); - int getIntImmCostInst(unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind); int getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -2961,6 +2961,20 @@ MaybeAlign Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, const Instruction *I) { + // TODO: Handle other cost kinds. + if (CostKind != TTI::TCK_RecipThroughput) { + if (isa_and_nonnull(I)) { + Value *Ptr = I->getOperand(1); + // Store instruction with index and scale costs 2 Uops. + // Check the preceding GEP to identify non-const indices. + if (auto *GEP = dyn_cast(Ptr)) { + if (!all_of(GEP->indices(), [](Value *V) { return isa(V); })) + return TTI::TCC_Basic * 2; + } + } + return TTI::TCC_Basic; + } + // Handle non-power-of-two vectors such as <3 x float> if (VectorType *VTy = dyn_cast(Src)) { unsigned NumElem = VTy->getNumElements(); @@ -3807,22 +3821,6 @@ return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind); } -unsigned -X86TTIImpl::getUserCost(const User *U, ArrayRef Operands, - TTI::TargetCostKind CostKind) { - if (isa(U)) { - Value *Ptr = U->getOperand(1); - // Store instruction with index and scale costs 2 Uops. - // Check the preceding GEP to identify non-const indices. - if (auto GEP = dyn_cast(Ptr)) { - if (!all_of(GEP->indices(), [](Value *V) { return isa(V); })) - return TTI::TCC_Basic * 2; - } - return TTI::TCC_Basic; - } - return BaseT::getUserCost(U, Operands, CostKind); -} - // Return an average cost of Gather / Scatter instruction, maybe improved later int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, Value *Ptr, unsigned Alignment, unsigned AddressSpace) {