diff --git a/llvm/include/llvm/Analysis/UniformityAnalysis.h b/llvm/include/llvm/Analysis/UniformityAnalysis.h --- a/llvm/include/llvm/Analysis/UniformityAnalysis.h +++ b/llvm/include/llvm/Analysis/UniformityAnalysis.h @@ -19,8 +19,12 @@ namespace llvm { +/// Note: the following is used instead of a type alias to make +/// UniformityInfo forward-declarable. extern template class GenericUniformityInfo; -using UniformityInfo = GenericUniformityInfo; +struct UniformityInfo : public GenericUniformityInfo { + using GenericUniformityInfo::GenericUniformityInfo; +}; /// Analysis pass which computes \ref UniformityInfo. class UniformityInfoAnalysis diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h --- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -35,7 +35,7 @@ class Argument; class BasicBlock; class BranchProbabilityInfo; -class LegacyDivergenceAnalysis; +struct UniformityInfo; class Function; class Instruction; class MachineFunction; @@ -56,7 +56,7 @@ const TargetLowering *TLI; MachineRegisterInfo *RegInfo; BranchProbabilityInfo *BPI; - const LegacyDivergenceAnalysis *DA; + const UniformityInfo *UA; /// CanLowerReturn - true iff the function's return value can be lowered to /// registers. bool CanLowerReturn; diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -71,7 +71,7 @@ class FunctionVarLocs; class GlobalValue; struct KnownBits; -class LegacyDivergenceAnalysis; +struct UniformityInfo; class LLVMContext; class MachineBasicBlock; class MachineConstantPoolValue; @@ -229,7 +229,7 @@ LLVMContext *Context; CodeGenOpt::Level OptLevel; - LegacyDivergenceAnalysis * DA = nullptr; + UniformityInfo *UA = nullptr; FunctionLoweringInfo * FLI = nullptr; /// The function-level optimization remark emitter. Used to emit remarks @@ -451,7 +451,7 @@ /// Prepare this SelectionDAG to process code in the given MachineFunction. void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, - LegacyDivergenceAnalysis *Divergence, ProfileSummaryInfo *PSIin, + UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs); void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { @@ -474,7 +474,7 @@ const TargetLowering &getTargetLoweringInfo() const { return *TLI; } const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } - const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; } + const UniformityInfo *getUniformityInfo() const { return UA; } /// Returns the result of the AssignmentTrackingAnalysis pass if it's /// available, otherwise return nullptr. const FunctionVarLocs *getFunctionVarLocs() const { return FnVarLocs; } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -74,7 +74,7 @@ class IntrinsicInst; class IRBuilderBase; struct KnownBits; -class LegacyDivergenceAnalysis; +struct UniformityInfo; class LLVMContext; class MachineBasicBlock; class MachineFunction; @@ -3537,7 +3537,7 @@ virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, - LegacyDivergenceAnalysis *DA) const { + UniformityInfo *UA) const { return false; } diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -13,7 +13,7 @@ #include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/ADT/APInt.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" +#include "llvm/Analysis/UniformityAnalysis.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -83,7 +83,7 @@ TLI = MF->getSubtarget().getTargetLowering(); RegInfo = &MF->getRegInfo(); const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); - DA = DAG->getDivergenceAnalysis(); + UA = DAG->getUniformityInfo(); // Check whether the function can return without sret-demotion. SmallVector Outs; @@ -381,8 +381,8 @@ } Register FunctionLoweringInfo::CreateRegs(const Value *V) { - return CreateRegs(V->getType(), DA && DA->isDivergent(V) && - !TLI->requiresUniformRegister(*MF, V)); + return CreateRegs(V->getType(), UA && UA->isDivergent(V) && + !TLI->requiresUniformRegister(*MF, V)); } /// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1311,8 +1311,8 @@ void SelectionDAG::init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, - LegacyDivergenceAnalysis *Divergence, - ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, + UniformityInfo *NewUA, ProfileSummaryInfo *PSIin, + BlockFrequencyInfo *BFIin, FunctionVarLocs const *VarLocs) { MF = &NewMF; SDAGISelPass = PassPtr; @@ -1321,7 +1321,7 @@ TSI = getSubtarget().getSelectionDAGInfo(); LibInfo = LibraryInfo; Context = &MF->getFunction().getContext(); - DA = Divergence; + UA = NewUA; PSI = PSIin; BFI = BFIin; FnVarLocs = VarLocs; @@ -2175,7 +2175,7 @@ return SDValue(E, 0); auto *N = newSDNode(RegNo, VT); - N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA); + N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -10659,11 +10659,11 @@ bool SelectionDAG::calculateDivergence(SDNode *N) { if (TLI->isSDNodeAlwaysUniform(N)) { - assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && + assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) && "Conflicting divergence information!"); return false; } - if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA)) + if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA)) return true; for (const auto &Op : N->ops()) { if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) @@ -12158,7 +12158,7 @@ Node->NumOperands = Vals.size(); Node->OperandList = Ops; if (!TLI->isSDNodeAlwaysUniform(Node)) { - IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); + IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA); Node->SDNodeBits.IsDivergent = IsDivergent; } checkForCycles(Node); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -26,11 +26,11 @@ #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/LazyBlockFrequencyInfo.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/UniformityAnalysis.h" #include "llvm/CodeGen/AssignmentTrackingAnalysis.h" #include "llvm/CodeGen/CodeGenCommonISel.h" #include "llvm/CodeGen/FastISel.h" @@ -425,9 +425,10 @@ LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n"); - CurDAG->init(*MF, *ORE, this, LibInfo, - getAnalysisIfAvailable(), PSI, BFI, - FnVarLocs); + UniformityInfo *UA = nullptr; + if (auto *UAPass = getAnalysisIfAvailable()) + UA = &UAPass->getUniformityInfo(); + CurDAG->init(*MF, *ORE, this, LibInfo, UA, PSI, BFI, FnVarLocs); FuncInfo->set(Fn, *MF, CurDAG); SwiftError->setFunction(*MF); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -20,7 +20,7 @@ #include "MCTargetDesc/R600MCTargetDesc.h" #include "R600RegisterInfo.h" #include "SIMachineFunctionInfo.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" +#include "llvm/Analysis/UniformityAnalysis.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/SelectionDAG.h" @@ -101,7 +101,7 @@ "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo) INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis) -INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis) +INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass) #ifdef EXPENSIVE_CHECKS INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) @@ -199,7 +199,7 @@ void AMDGPUDAGToDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); - AU.addRequired(); + AU.addRequired(); #ifdef EXPENSIVE_CHECKS AU.addRequired(); AU.addRequired(); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -466,8 +466,8 @@ Align computeKnownAlignForTargetInstr(GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth = 0) const override; - bool isSDNodeSourceOfDivergence(const SDNode *N, - FunctionLoweringInfo *FLI, LegacyDivergenceAnalysis *DA) const override; + bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, + UniformityInfo *UA) const override; bool hasMemSDNodeUser(SDNode *N) const; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -19,8 +19,8 @@ #include "SIRegisterInfo.h" #include "llvm/ADT/FloatingPointMode.h" #include "llvm/ADT/Statistic.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/UniformityAnalysis.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" @@ -35,8 +35,8 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/ModRef.h" #include "llvm/Support/KnownBits.h" +#include "llvm/Support/ModRef.h" using namespace llvm; @@ -12871,9 +12871,9 @@ return false; } -bool SITargetLowering::isSDNodeSourceOfDivergence( - const SDNode *N, FunctionLoweringInfo *FLI, - LegacyDivergenceAnalysis *KDA) const { +bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N, + FunctionLoweringInfo *FLI, + UniformityInfo *UA) const { switch (N->getOpcode()) { case ISD::CopyFromReg: { const RegisterSDNode *R = cast(N->getOperand(1)); @@ -12886,7 +12886,7 @@ return !TRI->isSGPRReg(MRI, Reg); if (const Value *V = FLI->getValueFromVirtualReg(R->getReg())) - return KDA->isDivergent(V); + return UA->isDivergent(V); assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N)); return !TRI->isSGPRReg(MRI, Reg); @@ -13167,6 +13167,8 @@ // uniform values (as produced by the mask results of control flow intrinsics) // used outside of divergent blocks. The phi users need to also be treated as // always uniform. +// +// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis? static bool hasCFUser(const Value *V, SmallPtrSet &Visited, unsigned WaveSize) { // FIXME: We assume we never cast the mask results of a control flow diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -98,9 +98,8 @@ ; GCN-O0-NEXT: Safe Stack instrumentation pass ; GCN-O0-NEXT: Insert stack protectors ; GCN-O0-NEXT: Dominator Tree Construction -; GCN-O0-NEXT: Post-Dominator Tree Construction -; GCN-O0-NEXT: Natural Loop Information -; GCN-O0-NEXT: Legacy Divergence Analysis +; GCN-O0-NEXT: Cycle Info Analysis +; GCN-O0-NEXT: Uniformity Analysis ; GCN-O0-NEXT: Assignment Tracking Analysis ; GCN-O0-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection ; GCN-O0-NEXT: MachineDominator Tree Construction @@ -292,11 +291,12 @@ ; GCN-O1-NEXT: Safe Stack instrumentation pass ; GCN-O1-NEXT: Insert stack protectors ; GCN-O1-NEXT: Dominator Tree Construction -; GCN-O1-NEXT: Post-Dominator Tree Construction -; GCN-O1-NEXT: Natural Loop Information -; GCN-O1-NEXT: Legacy Divergence Analysis +; GCN-O1-NEXT: Cycle Info Analysis +; GCN-O1-NEXT: Uniformity Analysis ; GCN-O1-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-NEXT: Function Alias Analysis Results +; GCN-O1-NEXT: Natural Loop Information +; GCN-O1-NEXT: Post-Dominator Tree Construction ; GCN-O1-NEXT: Branch Probability Analysis ; GCN-O1-NEXT: Assignment Tracking Analysis ; GCN-O1-NEXT: Lazy Branch Probability Analysis @@ -590,11 +590,12 @@ ; GCN-O1-OPTS-NEXT: Safe Stack instrumentation pass ; GCN-O1-OPTS-NEXT: Insert stack protectors ; GCN-O1-OPTS-NEXT: Dominator Tree Construction -; GCN-O1-OPTS-NEXT: Post-Dominator Tree Construction -; GCN-O1-OPTS-NEXT: Natural Loop Information -; GCN-O1-OPTS-NEXT: Legacy Divergence Analysis +; GCN-O1-OPTS-NEXT: Cycle Info Analysis +; GCN-O1-OPTS-NEXT: Uniformity Analysis ; GCN-O1-OPTS-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-OPTS-NEXT: Function Alias Analysis Results +; GCN-O1-OPTS-NEXT: Natural Loop Information +; GCN-O1-OPTS-NEXT: Post-Dominator Tree Construction ; GCN-O1-OPTS-NEXT: Branch Probability Analysis ; GCN-O1-OPTS-NEXT: Assignment Tracking Analysis ; GCN-O1-OPTS-NEXT: Lazy Branch Probability Analysis @@ -897,11 +898,12 @@ ; GCN-O2-NEXT: Safe Stack instrumentation pass ; GCN-O2-NEXT: Insert stack protectors ; GCN-O2-NEXT: Dominator Tree Construction -; GCN-O2-NEXT: Post-Dominator Tree Construction -; GCN-O2-NEXT: Natural Loop Information -; GCN-O2-NEXT: Legacy Divergence Analysis +; GCN-O2-NEXT: Cycle Info Analysis +; GCN-O2-NEXT: Uniformity Analysis ; GCN-O2-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O2-NEXT: Function Alias Analysis Results +; GCN-O2-NEXT: Natural Loop Information +; GCN-O2-NEXT: Post-Dominator Tree Construction ; GCN-O2-NEXT: Branch Probability Analysis ; GCN-O2-NEXT: Assignment Tracking Analysis ; GCN-O2-NEXT: Lazy Branch Probability Analysis @@ -1216,11 +1218,12 @@ ; GCN-O3-NEXT: Safe Stack instrumentation pass ; GCN-O3-NEXT: Insert stack protectors ; GCN-O3-NEXT: Dominator Tree Construction -; GCN-O3-NEXT: Post-Dominator Tree Construction -; GCN-O3-NEXT: Natural Loop Information -; GCN-O3-NEXT: Legacy Divergence Analysis +; GCN-O3-NEXT: Cycle Info Analysis +; GCN-O3-NEXT: Uniformity Analysis ; GCN-O3-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O3-NEXT: Function Alias Analysis Results +; GCN-O3-NEXT: Natural Loop Information +; GCN-O3-NEXT: Post-Dominator Tree Construction ; GCN-O3-NEXT: Branch Probability Analysis ; GCN-O3-NEXT: Assignment Tracking Analysis ; GCN-O3-NEXT: Lazy Branch Probability Analysis