Index: llvm/trunk/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ llvm/trunk/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -661,7 +661,9 @@ bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); - return BB->getTerminator()->getMetadata("amdgpu.uniform"); + const Instruction *Term = BB->getTerminator(); + return Term->getMetadata("amdgpu.uniform") || + Term->getMetadata("structurizecfg.uniform"); } const char *AMDGPUDAGToDAGISel::getPassName() const { Index: llvm/trunk/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp +++ llvm/trunk/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp @@ -69,6 +69,8 @@ LoopInfo *LI; + bool isUniform(BranchInst *T); + bool isTopOfStack(BasicBlock *BB); Value *popSaved(); @@ -162,6 +164,13 @@ return false; } +/// \brief Is the branch condition uniform or did the StructurizeCFG pass +/// consider it as such? +bool SIAnnotateControlFlow::isUniform(BranchInst *T) { + return DA->isUniform(T->getCondition()) || + T->getMetadata("structurizecfg.uniform") != nullptr; +} + /// \brief Is BB the last block saved on the stack ? bool SIAnnotateControlFlow::isTopOfStack(BasicBlock *BB) { return !Stack.empty() && Stack.back().first == BB; @@ -204,7 +213,7 @@ /// \brief Open a new "If" block void SIAnnotateControlFlow::openIf(BranchInst *Term) { - if (DA->isUniform(Term->getCondition())) { + if (isUniform(Term)) { return; } Value *Ret = CallInst::Create(If, Term->getCondition(), "", Term); @@ -214,7 +223,7 @@ /// \brief Close the last "If" block and open a new "Else" block void SIAnnotateControlFlow::insertElse(BranchInst *Term) { - if (DA->isUniform(Term->getCondition())) { + if (isUniform(Term)) { return; } Value *Ret = CallInst::Create(Else, popSaved(), "", Term); @@ -316,7 +325,7 @@ /// \brief Handle a back edge (loop) void SIAnnotateControlFlow::handleLoop(BranchInst *Term) { - if (DA->isUniform(Term->getCondition())) { + if (isUniform(Term)) { return; } Index: llvm/trunk/lib/Transforms/Scalar/StructurizeCFG.cpp =================================================================== --- llvm/trunk/lib/Transforms/Scalar/StructurizeCFG.cpp +++ llvm/trunk/lib/Transforms/Scalar/StructurizeCFG.cpp @@ -951,6 +951,21 @@ // TODO: We could probably be smarter here with how we handle sub-regions. if (hasOnlyUniformBranches(R)) { DEBUG(dbgs() << "Skipping region with uniform control flow: " << *R << '\n'); + + // Mark all direct child block terminators as having been treated as + // uniform. To account for a possible future in which non-uniform + // sub-regions are treated more cleverly, indirect children are not + // marked as uniform. + MDNode *MD = MDNode::get(R->getEntry()->getParent()->getContext(), {}); + Region::element_iterator E = R->element_end(); + for (Region::element_iterator I = R->element_begin(); I != E; ++I) { + if (I->isSubRegion()) + continue; + + if (Instruction *Term = I->getEntry()->getTerminator()) + Term->setMetadata("structurizecfg.uniform", MD); + } + return false; } }