Index: lib/Target/NVPTX/CMakeLists.txt =================================================================== --- lib/Target/NVPTX/CMakeLists.txt +++ lib/Target/NVPTX/CMakeLists.txt @@ -9,27 +9,28 @@ add_public_tablegen_target(NVPTXCommonTableGen) set(NVPTXCodeGen_sources + NVPTXAllocaHoisting.cpp + NVPTXAsmPrinter.cpp + NVPTXAssignValidGlobalNames.cpp NVPTXFavorNonGenericAddrSpaces.cpp NVPTXFrameLowering.cpp - NVPTXInstrInfo.cpp + NVPTXGenericToNVVM.cpp NVPTXISelDAGToDAG.cpp NVPTXISelLowering.cpp + NVPTXImageOptimizer.cpp + NVPTXInstrInfo.cpp + NVPTXLowerAggrCopies.cpp + NVPTXLowerStructArgs.cpp + NVPTXMCExpr.cpp + NVPTXPrologEpilogPass.cpp NVPTXRegisterInfo.cpp + NVPTXReplaceImageHandles.cpp NVPTXSubtarget.cpp NVPTXTargetMachine.cpp - NVPTXLowerAggrCopies.cpp - NVPTXutil.cpp - NVPTXAllocaHoisting.cpp - NVPTXAsmPrinter.cpp + NVPTXTargetTransformInfo.cpp NVPTXUtilities.cpp + NVPTXutil.cpp NVVMReflect.cpp - NVPTXGenericToNVVM.cpp - NVPTXAssignValidGlobalNames.cpp - NVPTXPrologEpilogPass.cpp - NVPTXMCExpr.cpp - NVPTXReplaceImageHandles.cpp - NVPTXImageOptimizer.cpp - NVPTXLowerStructArgs.cpp ) add_llvm_target(NVPTXCodeGen ${NVPTXCodeGen_sources}) Index: lib/Target/NVPTX/NVPTX.h =================================================================== --- lib/Target/NVPTX/NVPTX.h +++ lib/Target/NVPTX/NVPTX.h @@ -59,6 +59,7 @@ llvm_unreachable("Unknown condition code"); } +ImmutablePass *createNVPTXTargetTransformInfoPass(const NVPTXTargetMachine *TM); FunctionPass * createNVPTXISelDag(NVPTXTargetMachine &TM, llvm::CodeGenOpt::Level OptLevel); ModulePass *createNVPTXAssignValidGlobalNamesPass(); Index: lib/Target/NVPTX/NVPTXTargetMachine.h =================================================================== --- lib/Target/NVPTX/NVPTXTargetMachine.h +++ lib/Target/NVPTX/NVPTXTargetMachine.h @@ -49,6 +49,9 @@ return true; } + /// \brief Register NVPTX analysis passes with a pass manager. + void addAnalysisPasses(PassManagerBase &PM) override; + }; // NVPTXTargetMachine. class NVPTXTargetMachine32 : public NVPTXTargetMachine { Index: lib/Target/NVPTX/NVPTXTargetMachine.cpp =================================================================== --- lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -121,6 +121,14 @@ return PassConfig; } +void NVPTXTargetMachine::addAnalysisPasses(PassManagerBase &PM) { + // Add first the target-independent BasicTTI pass, then our NVPTX pass. This + // allows the NVPTX pass to delegate to the target independent layer when + // appropriate. + PM.add(createBasicTargetTransformInfoPass(this)); + PM.add(createNVPTXTargetTransformInfoPass(this)); +} + void NVPTXPassConfig::addIRPasses() { // The following passes are known to not play well with virtual regs hanging // around after register allocation (which in our case, is *all* registers). Index: lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp =================================================================== --- /dev/null +++ lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp @@ -0,0 +1,125 @@ +//===-- NVPTXTargetTransformInfo.cpp - NVPTX specific TTI pass ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// \file +// This file implements a TargetTransformInfo analysis pass specific to the +// NVPTX target machine. It uses the target's detailed information to provide +// more precise answers to certain TTI queries, while letting the target +// independent and default TTI implementations handle the rest. +// +//===----------------------------------------------------------------------===// + +#include "NVPTXTargetMachine.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/Support/Debug.h" +#include "llvm/Target/CostTable.h" +#include "llvm/Target/TargetLowering.h" +using namespace llvm; + +#define DEBUG_TYPE "NVPTXtti" + +// Declare the pass initialization routine locally as target-specific passes +// don't have a target-wide initialization entry point, and so we rely on the +// pass constructor initialization. +namespace llvm { +void initializeNVPTXTTIPass(PassRegistry &); +} + +namespace { + +class NVPTXTTI final : public ImmutablePass, public TargetTransformInfo { + const NVPTXTargetMachine *TM; + const NVPTXSubtarget *ST; + const NVPTXTargetLowering *TLI; + + /// Estimate the overhead of scalarizing an instruction. Insert and Extract + /// are set if the result needs to be inserted and/or extracted from vectors. + unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const; + +public: + NVPTXTTI() : ImmutablePass(ID), TM(nullptr), ST(nullptr), TLI(nullptr) { + llvm_unreachable("This pass cannot be directly constructed"); + } + + NVPTXTTI(const NVPTXTargetMachine *TM) + : ImmutablePass(ID), TM(TM), ST(TM->getSubtargetImpl()), + TLI(TM->getSubtargetImpl()->getTargetLowering()) { + initializeNVPTXTTIPass(*PassRegistry::getPassRegistry()); + } + + void initializePass() override { pushTTIStack(this); } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + TargetTransformInfo::getAnalysisUsage(AU); + } + + /// Pass identification. + static char ID; + + /// Provide necessary pointer adjustments for the two base classes. + void *getAdjustedAnalysisPointer(const void *ID) override { + if (ID == &TargetTransformInfo::ID) + return (TargetTransformInfo *)this; + return this; + } + + bool hasBranchDivergence() const override; + + unsigned getArithmeticInstrCost( + unsigned Opcode, Type *Ty, OperandValueKind Opd1Info = OK_AnyValue, + OperandValueKind Opd2Info = OK_AnyValue, + OperandValueProperties Opd1PropInfo = OP_None, + OperandValueProperties Opd2PropInfo = OP_None) const override; + + /// @} +}; + +} // end anonymous namespace + +INITIALIZE_AG_PASS(NVPTXTTI, TargetTransformInfo, "NVPTXtti", + "NVPTX Target Transform Info", true, true, false) +char NVPTXTTI::ID = 0; + +ImmutablePass * +llvm::createNVPTXTargetTransformInfoPass(const NVPTXTargetMachine *TM) { + return new NVPTXTTI(TM); +} + +bool NVPTXTTI::hasBranchDivergence() const { return true; } + +unsigned NVPTXTTI::getArithmeticInstrCost( + unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, + OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo, + OperandValueProperties Opd2PropInfo) const { + // Legalize the type. + std::pair LT = TLI->getTypeLegalizationCost(Ty); + + int ISD = TLI->InstructionOpcodeToISD(Opcode); + + switch (ISD) { + default: + return TargetTransformInfo::getArithmeticInstrCost( + Opcode, Ty, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo); + case ISD::ADD: + case ISD::MUL: + case ISD::XOR: + case ISD::OR: + case ISD::AND: + // The machine code (SASS) simulates an i64 with two i32. Therefore, we + // estimate that arithmetic operations on i64 are twice as expensive as + // those on types that can fit into one machine register. + if (LT.second.SimpleTy == MVT::i64) + return 2 * LT.first; + // Delegate other cases to the basic TTI. + return TargetTransformInfo::getArithmeticInstrCost( + Opcode, Ty, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo); + } +} Index: lib/Transforms/Scalar/IndVarSimplify.cpp =================================================================== --- lib/Transforms/Scalar/IndVarSimplify.cpp +++ lib/Transforms/Scalar/IndVarSimplify.cpp @@ -31,6 +31,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopPass.h" #include "llvm/Analysis/ScalarEvolutionExpander.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constants.h" @@ -69,11 +70,12 @@ namespace { class IndVarSimplify : public LoopPass { - LoopInfo *LI; - ScalarEvolution *SE; - DominatorTree *DT; - const DataLayout *DL; - TargetLibraryInfo *TLI; + LoopInfo *LI; + ScalarEvolution *SE; + DominatorTree *DT; + const DataLayout *DL; + TargetLibraryInfo *TLI; + const TargetTransformInfo *TTI; SmallVector DeadInsts; bool Changed; @@ -661,7 +663,7 @@ /// extended by this sign or zero extend operation. This is used to determine /// the final width of the IV before actually widening it. static void visitIVCast(CastInst *Cast, WideIVInfo &WI, ScalarEvolution *SE, - const DataLayout *DL) { + const DataLayout *DL, const TargetTransformInfo *TTI) { bool IsSigned = Cast->getOpcode() == Instruction::SExt; if (!IsSigned && Cast->getOpcode() != Instruction::ZExt) return; @@ -671,6 +673,19 @@ if (DL && !DL->isLegalInteger(Width)) return; + // Cast is either an sext or zext up to this point. + // We should not widen an indvar if arithmetics on the wider indvar are more + // expensive than those on the narrower indvar. We check only the cost of ADD + // because at least an ADD is required to increment the induction variable. We + // could compute more comprehensively the cost of all instructions on the + // induction variable when necessary. + if (TTI && + TTI->getArithmeticInstrCost(Instruction::Add, Ty) > + TTI->getArithmeticInstrCost(Instruction::Add, + Cast->getOperand(0)->getType())) { + return; + } + if (!WI.WidestNativeType) { WI.WidestNativeType = SE->getEffectiveSCEVType(Ty); WI.IsSigned = IsSigned; @@ -1187,14 +1202,16 @@ class IndVarSimplifyVisitor : public IVVisitor { ScalarEvolution *SE; const DataLayout *DL; + const TargetTransformInfo *TTI; PHINode *IVPhi; public: WideIVInfo WI; IndVarSimplifyVisitor(PHINode *IV, ScalarEvolution *SCEV, - const DataLayout *DL, const DominatorTree *DTree): - SE(SCEV), DL(DL), IVPhi(IV) { + const DataLayout *DL, const TargetTransformInfo *TTI, + const DominatorTree *DTree) + : SE(SCEV), DL(DL), TTI(TTI), IVPhi(IV) { DT = DTree; WI.NarrowIV = IVPhi; if (ReduceLiveIVs) @@ -1202,7 +1219,9 @@ } // Implement the interface used by simplifyUsersOfIV. - void visitCast(CastInst *Cast) override { visitIVCast(Cast, WI, SE, DL); } + void visitCast(CastInst *Cast) override { + visitIVCast(Cast, WI, SE, DL, TTI); + } }; } @@ -1236,7 +1255,7 @@ PHINode *CurrIV = LoopPhis.pop_back_val(); // Information about sign/zero extensions of CurrIV. - IndVarSimplifyVisitor Visitor(CurrIV, SE, DL, DT); + IndVarSimplifyVisitor Visitor(CurrIV, SE, DL, TTI, DT); Changed |= simplifyUsersOfIV(CurrIV, SE, &LPM, DeadInsts, &Visitor); @@ -1895,6 +1914,7 @@ DataLayoutPass *DLP = getAnalysisIfAvailable(); DL = DLP ? &DLP->getDataLayout() : nullptr; TLI = getAnalysisIfAvailable(); + TTI = getAnalysisIfAvailable(); DeadInsts.clear(); Changed = false; Index: test/Transforms/IndVarSimplify/no-widen-expensive.ll =================================================================== --- /dev/null +++ test/Transforms/IndVarSimplify/no-widen-expensive.ll @@ -0,0 +1,37 @@ +; RUN: opt < %s -indvars -S | FileCheck %s + +target triple = "nvptx64-unknown-unknown" + +; For the nvptx64 architecture, the cost of an arithmetic instruction on a +; 64-bit integer is twice as expensive as that on a 32-bit integer, because the +; hardware needs to simulate a 64-bit integer using two 32-bit integers. +; Therefore, in this particular architecture, we should not widen induction +; variables to 64-bit integers even though i64 is a legal type in the 64-bit +; PTX ISA. + +define void @indvar_32_bit(i32 %n, i32* nocapture %output) { +; CHECK-LABEL: @indvar_32_bit +entry: + %cmp5 = icmp sgt i32 %n, 0 + br i1 %cmp5, label %for.body.preheader, label %for.end + +for.body.preheader: ; preds = %entry + br label %for.body + +for.body: ; preds = %for.body.preheader, %for.body + %i.06 = phi i32 [ 0, %for.body.preheader ], [ %add, %for.body ] +; CHECK: phi i32 + %mul = mul nsw i32 %i.06, %i.06 + %0 = sext i32 %i.06 to i64 + %arrayidx = getelementptr inbounds i32* %output, i64 %0 + store i32 %mul, i32* %arrayidx, align 4 + %add = add nsw i32 %i.06, 3 + %cmp = icmp slt i32 %add, %n + br i1 %cmp, label %for.body, label %for.end.loopexit + +for.end.loopexit: ; preds = %for.body + br label %for.end + +for.end: ; preds = %for.end.loopexit, %entry + ret void +}