Index: include/llvm/Analysis/CFGPrinter.h =================================================================== --- include/llvm/Analysis/CFGPrinter.h +++ include/llvm/Analysis/CFGPrinter.h @@ -154,14 +154,11 @@ if (TI->getNumSuccessors() == 1) return ""; - MDNode *WeightsNode = TI->getMetadata(LLVMContext::MD_prof); + MDNode *WeightsNode = TI->getProfMetadata( + LLVMContext::MD_PROF_branch_weights); if (!WeightsNode) return ""; - MDString *MDName = cast(WeightsNode->getOperand(0)); - if (MDName->getString() != "branch_weights") - return ""; - unsigned OpNo = I.getSuccessorIndex() + 1; if (OpNo >= WeightsNode->getNumOperands()) return ""; Index: include/llvm/IR/IRBuilder.h =================================================================== --- include/llvm/IR/IRBuilder.h +++ include/llvm/IR/IRBuilder.h @@ -732,9 +732,9 @@ /// instruction. /// \returns The annotated instruction. template - InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) { - if (Weights) - I->setMetadata(LLVMContext::MD_prof, Weights); + InstTy *addProfMetadata(InstTy *I, MDNode *Prof, MDNode *Unpredictable) { + if (Prof) + I->setMetadata(LLVMContext::MD_prof, Prof); if (Unpredictable) I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable); return I; @@ -773,10 +773,10 @@ /// \brief Create a conditional 'br Cond, TrueDest, FalseDest' /// instruction. BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, - MDNode *BranchWeights = nullptr, + MDNode *Prof = nullptr, MDNode *Unpredictable = nullptr) { - return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond), - BranchWeights, Unpredictable)); + return Insert(addProfMetadata(BranchInst::Create(True, False, Cond), + Prof, Unpredictable)); } /// \brief Create a conditional 'br Cond, TrueDest, FalseDest' @@ -796,10 +796,10 @@ /// and with a hint for the number of cases that will be added (for efficient /// allocation). SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10, - MDNode *BranchWeights = nullptr, + MDNode *Prof = nullptr, MDNode *Unpredictable = nullptr) { - return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases), - BranchWeights, Unpredictable)); + return Insert(addProfMetadata(SwitchInst::Create(V, Dest, NumCases), + Prof, Unpredictable)); } /// \brief Create an indirect branch instruction with the specified address @@ -1697,7 +1697,7 @@ if (MDFrom) { MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); - Sel = addBranchMetadata(Sel, Prof, Unpred); + Sel = addProfMetadata(Sel, Prof, Unpred); } return Insert(Sel, Name); } Index: include/llvm/IR/Instruction.h =================================================================== --- include/llvm/IR/Instruction.h +++ include/llvm/IR/Instruction.h @@ -241,6 +241,11 @@ /// metadata. void swapProfMetadata(); + /// Get a MD_prof metadata. + MDNode *getProfMetadata(unsigned MDProfKindId) const; + /// Set a MD_prof metadata. + void setProfMetadata(unsigned MDProfKindId, MDNode *NewNode); + /// Drop all unknown metadata except for debug locations. /// @{ /// Passes are required to drop metadata they don't understand. This is a Index: include/llvm/IR/LLVMContext.h =================================================================== --- include/llvm/IR/LLVMContext.h +++ include/llvm/IR/LLVMContext.h @@ -102,6 +102,13 @@ MD_associated = 22, // "associated" }; + // The types of the MD_prof metadata. + enum { + MD_PROF_branch_weights = 0, // "branch_weights" + MD_PROF_function_entry_count = 1, // "function_entry_count" + MD_PROF_VP = 2, // "VP" + }; + /// Known operand bundle tag IDs, which always have the same value. All /// operand bundle tags that LLVM has special knowledge of are listed here. /// Additionally, this scheme allows LLVM to efficiently check for specific @@ -120,6 +127,11 @@ /// custom metadata IDs registered in this LLVMContext. void getMDKindNames(SmallVectorImpl &Result) const; + static const char* getMDProfKindName(unsigned MDProfKindId) { + assert(MDProfKindId < sizeof(MDProfKindNames) && "out of range"); + return MDProfKindNames[MDProfKindId]; + } + /// getOperandBundleTags - Populate client supplied SmallVector with the /// bundle tags registered in this LLVMContext. The bundle tags are ordered /// by increasing bundle IDs. @@ -320,6 +332,9 @@ // Module needs access to the add/removeModule methods. friend class Module; + // The strings holding the names of the MD_Prof metadata types. + static const char *MDProfKindNames[3]; + /// addModule - Register a module as being instantiated in this context. If /// the context is deleted, the module will be deleted as well. void addModule(Module*); Index: include/llvm/IR/MDBuilder.h =================================================================== --- include/llvm/IR/MDBuilder.h +++ include/llvm/IR/MDBuilder.h @@ -62,6 +62,16 @@ /// \brief Return metadata containing a number of branch weights. MDNode *createBranchWeights(ArrayRef Weights); + /// \brief Return metadata containing two weights. + template + MDNode *createProfWeights(unsigned MDProfKindId, + WeightIntType TrueWeight, WeightIntType FalseWeight); + + /// \brief Return metadata containing a number of weights. + template + MDNode *createProfWeights(unsigned MDProfKindId, + ArrayRef Weights); + /// Return metadata specifying that a branch or switch is unpredictable. MDNode *createUnpredictable(); Index: include/llvm/Transforms/PGOInstrumentation.h =================================================================== --- include/llvm/Transforms/PGOInstrumentation.h +++ include/llvm/Transforms/PGOInstrumentation.h @@ -54,8 +54,8 @@ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; -void setProfMetadata(Module *M, Instruction *TI, ArrayRef EdgeCounts, - uint64_t MaxCount); +void setBranchWeightsMetadata(Module *M, Instruction *TI, + ArrayRef EdgeCounts, uint64_t MaxCount); } // End llvm namespace #endif Index: lib/Analysis/BranchProbabilityInfo.cpp =================================================================== --- lib/Analysis/BranchProbabilityInfo.cpp +++ lib/Analysis/BranchProbabilityInfo.cpp @@ -249,7 +249,8 @@ if (!(isa(TI) || isa(TI) || isa(TI))) return false; - MDNode *WeightsNode = TI->getMetadata(LLVMContext::MD_prof); + MDNode *WeightsNode = TI->getProfMetadata( + LLVMContext::MD_PROF_branch_weights); if (!WeightsNode) return false; Index: lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- lib/CodeGen/CodeGenPrepare.cpp +++ lib/CodeGen/CodeGenPrepare.cpp @@ -6727,14 +6727,16 @@ uint64_t NewTrueWeight = TrueWeight; uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight; scaleWeights(NewTrueWeight, NewFalseWeight); - Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) - .createBranchWeights(TrueWeight, FalseWeight)); + Br1->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(Br1->getContext()) + .createBranchWeights(TrueWeight, FalseWeight)); NewTrueWeight = TrueWeight; NewFalseWeight = 2 * FalseWeight; scaleWeights(NewTrueWeight, NewFalseWeight); - Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) - .createBranchWeights(TrueWeight, FalseWeight)); + Br2->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(Br2->getContext()) + .createBranchWeights(TrueWeight, FalseWeight)); } } else { // Codegen X & Y as: @@ -6760,14 +6762,16 @@ uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight; uint64_t NewFalseWeight = FalseWeight; scaleWeights(NewTrueWeight, NewFalseWeight); - Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) - .createBranchWeights(TrueWeight, FalseWeight)); + Br1->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(Br1->getContext()) + .createBranchWeights(TrueWeight, FalseWeight)); NewTrueWeight = 2 * TrueWeight; NewFalseWeight = FalseWeight; scaleWeights(NewTrueWeight, NewFalseWeight); - Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) - .createBranchWeights(TrueWeight, FalseWeight)); + Br2->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(Br2->getContext()) + .createBranchWeights(TrueWeight, FalseWeight)); } } Index: lib/IR/Instruction.cpp =================================================================== --- lib/IR/Instruction.cpp +++ lib/IR/Instruction.cpp @@ -590,20 +590,14 @@ } void Instruction::swapProfMetadata() { - MDNode *ProfileData = getMetadata(LLVMContext::MD_prof); - if (!ProfileData || ProfileData->getNumOperands() != 3 || - !isa(ProfileData->getOperand(0))) - return; - - MDString *MDName = cast(ProfileData->getOperand(0)); - if (MDName->getString() != "branch_weights") - return; - - // The first operand is the name. Fetch them backwards and build a new one. - Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2), - ProfileData->getOperand(1)}; - setMetadata(LLVMContext::MD_prof, - MDNode::get(ProfileData->getContext(), Ops)); + MDNode *BranchWeights = getProfMetadata(LLVMContext::MD_PROF_branch_weights); + if (BranchWeights && BranchWeights->getNumOperands() == 3) { + // The first operand is the name. Fetch them backwards and build a new one. + Metadata *Ops[] = {BranchWeights->getOperand(0), BranchWeights->getOperand(2), + BranchWeights->getOperand(1)}; + setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDNode::get(BranchWeights->getContext(), Ops)); + } } void Instruction::copyMetadata(const Instruction &SrcInst, @@ -647,20 +641,20 @@ } void Instruction::updateProfWeight(uint64_t S, uint64_t T) { - auto *ProfileData = getMetadata(LLVMContext::MD_prof); + auto *BranchWeights = getProfMetadata(LLVMContext::MD_PROF_branch_weights); + auto *VP = getProfMetadata(LLVMContext::MD_PROF_VP); + assert(((BranchWeights && !VP) || (!BranchWeights && VP) || + (!BranchWeights && !VP)) && + "Only one of BranchWeights and VP should be there"); + auto *ProfileData = BranchWeights ? BranchWeights : VP; if (ProfileData == nullptr) return; - auto *ProfDataName = dyn_cast(ProfileData->getOperand(0)); - if (!ProfDataName || (!ProfDataName->getString().equals("branch_weights") && - !ProfDataName->getString().equals("VP"))) - return; - MDBuilder MDB(getContext()); SmallVector Vals; Vals.push_back(ProfileData->getOperand(0)); APInt APS(128, S), APT(128, T); - if (ProfDataName->getString().equals("branch_weights")) + if (ProfileData == BranchWeights) { for (unsigned i = 1; i < ProfileData->getNumOperands(); i++) { // Using APInt::div may be expensive, but most cases should fit 64 bits. APInt Val(128, @@ -672,7 +666,9 @@ ConstantInt::get(Type::getInt64Ty(getContext()), Val.udiv(APT).getLimitedValue()))); } - else if (ProfDataName->getString().equals("VP")) + setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDNode::get(getContext(), Vals)); + } else if (ProfileData == VP) { for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) { // The first value is the key of the value profile, which will not change. Vals.push_back(ProfileData->getOperand(i)); @@ -686,7 +682,9 @@ ConstantInt::get(Type::getInt64Ty(getContext()), Val.udiv(APT).getLimitedValue()))); } - setMetadata(LLVMContext::MD_prof, MDNode::get(getContext(), Vals)); + setProfMetadata(LLVMContext::MD_PROF_VP, + MDNode::get(getContext(), Vals)); + } } void Instruction::setProfWeight(uint64_t W) { @@ -695,5 +693,6 @@ SmallVector Weights; Weights.push_back(W); MDBuilder MDB(getContext()); - setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDB.createBranchWeights(Weights)); } Index: lib/IR/LLVMContext.cpp =================================================================== --- lib/IR/LLVMContext.cpp +++ lib/IR/LLVMContext.cpp @@ -32,6 +32,12 @@ using namespace llvm; +const char *LLVMContext::MDProfKindNames[] = { + "branch_weights", + "function_entry_count", + "VP", +}; + LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { // Create the fixed metadata kinds. This is done in the same order as the // MD_* enum values so that they correspond. Index: lib/IR/MDBuilder.cpp =================================================================== --- lib/IR/MDBuilder.cpp +++ lib/IR/MDBuilder.cpp @@ -40,18 +40,46 @@ } MDNode *MDBuilder::createBranchWeights(ArrayRef Weights) { - assert(Weights.size() >= 1 && "Need at least one branch weights!"); + return createProfWeights(LLVMContext::MD_PROF_branch_weights, + Weights); +} + +template +MDNode *MDBuilder::createProfWeights(unsigned MDProfKindId, + WeightIntType TrueWeight, + WeightIntType FalseWeight) { + return createProfWeights(MDProfKindId, + {TrueWeight, FalseWeight}); +} + +template MDNode *MDBuilder::createProfWeights( + unsigned MDProfKindId, uint32_t TrueWeight, uint32_t FalseWeight); +template MDNode *MDBuilder::createProfWeights( + unsigned MDProfKindId, uint64_t TrueWeight, uint64_t FalseWeight); + +template +MDNode *MDBuilder::createProfWeights(unsigned MDProfKindId, + ArrayRef Weights) { + assert(Weights.size() >= 1 && "Need at least one weights!"); SmallVector Vals(Weights.size() + 1); - Vals[0] = createString("branch_weights"); + Vals[0] = createString(LLVMContext::getMDProfKindName(MDProfKindId)); - Type *Int32Ty = Type::getInt32Ty(Context); + static_assert(sizeof(WeightIntType) == 8 || sizeof(WeightIntType) == 4, + "Weight type is not 4 nor 8 byte long"); + Type *WeightType = sizeof(WeightIntType) == 8 ? + Type::getInt64Ty(Context) : Type::getInt32Ty(Context); for (unsigned i = 0, e = Weights.size(); i != e; ++i) - Vals[i + 1] = createConstant(ConstantInt::get(Int32Ty, Weights[i])); + Vals[i + 1] = createConstant(ConstantInt::get(WeightType, Weights[i])); return MDNode::get(Context, Vals); } +template MDNode *MDBuilder::createProfWeights( + unsigned MDProfKindId, ArrayRef Weights); +template MDNode *MDBuilder::createProfWeights( + unsigned MDProfKindId, ArrayRef Weights); + MDNode *MDBuilder::createUnpredictable() { return MDNode::get(Context, None); } Index: lib/IR/Metadata.cpp =================================================================== --- lib/IR/Metadata.cpp +++ lib/IR/Metadata.cpp @@ -1306,16 +1306,12 @@ (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select) && "Looking for branch weights on something besides branch or select"); - auto *ProfileData = getMetadata(LLVMContext::MD_prof); - if (!ProfileData || ProfileData->getNumOperands() != 3) + auto *BranchWeights = getProfMetadata(LLVMContext::MD_PROF_branch_weights); + if (!BranchWeights || BranchWeights->getNumOperands() != 3) return false; - auto *ProfDataName = dyn_cast(ProfileData->getOperand(0)); - if (!ProfDataName || !ProfDataName->getString().equals("branch_weights")) - return false; - - auto *CITrue = mdconst::dyn_extract(ProfileData->getOperand(1)); - auto *CIFalse = mdconst::dyn_extract(ProfileData->getOperand(2)); + auto *CITrue = mdconst::dyn_extract(BranchWeights->getOperand(1)); + auto *CIFalse = mdconst::dyn_extract(BranchWeights->getOperand(2)); if (!CITrue || !CIFalse) return false; @@ -1334,15 +1330,16 @@ "Looking for branch weights on something besides branch"); TotalVal = 0; - auto *ProfileData = getMetadata(LLVMContext::MD_prof); + auto *BranchWeights = getProfMetadata(LLVMContext::MD_PROF_branch_weights); + auto *VP = getProfMetadata(LLVMContext::MD_PROF_VP); + assert(((BranchWeights && !VP) || (!BranchWeights && VP) || + (!BranchWeights && !VP)) && + "Only up to one of BranchWeights and VP should be there"); + auto *ProfileData = BranchWeights ? BranchWeights : VP; if (!ProfileData) return false; - auto *ProfDataName = dyn_cast(ProfileData->getOperand(0)); - if (!ProfDataName) - return false; - - if (ProfDataName->getString().equals("branch_weights")) { + if (ProfileData == BranchWeights) { TotalVal = 0; for (unsigned i = 1; i < ProfileData->getNumOperands(); i++) { auto *V = mdconst::dyn_extract(ProfileData->getOperand(i)); @@ -1351,7 +1348,7 @@ TotalVal += V->getValue().getZExtValue(); } return true; - } else if (ProfDataName->getString().equals("VP") && + } else if (ProfileData == VP && ProfileData->getNumOperands() > 3) { TotalVal = mdconst::dyn_extract(ProfileData->getOperand(2)) ->getValue() @@ -1361,6 +1358,102 @@ return false; } +MDNode *Instruction::getProfMetadata(unsigned MDProfKindId) const { + MDNode *ProfNode = getMetadata(LLVMContext::MD_prof); + if (!ProfNode) + return nullptr; + if (ProfNode->getOperand(0) != ProfNode) { + // Single-level. + MDString *MDName = cast(ProfNode->getOperand(0)); + if (MDName->getString() == LLVMContext::getMDProfKindName(MDProfKindId)) + return ProfNode; + return nullptr; + } + // Two-level. + for (unsigned I = 1; I < ProfNode->getNumOperands(); ++I) { + MDNode *MD = cast(ProfNode->getOperand(I)); + MDString *MDName = cast(MD->getOperand(0)); + if (MDName->getString() == LLVMContext::getMDProfKindName(MDProfKindId)) + return MD; + } + return nullptr; +} + +void Instruction::setProfMetadata(unsigned MDProfKindId, + MDNode *NewNode) { + DEBUG({ + if (NewNode) { + MDString *MDName = cast(NewNode->getOperand(0)); + assert(MDName->getString() == + LLVMContext::getMDProfKindName(MDProfKindId) && + "Mismatching MD prof kind name/id"); + } + }); + MDNode *ProfNode = getMetadata(LLVMContext::MD_prof); + if (!ProfNode) { + // Insert as a single-level. + setMetadata(LLVMContext::MD_prof, NewNode); + return; + } + if (ProfNode->getOperand(0) != ProfNode) { + // Single-level. + MDString *MDName = cast(ProfNode->getOperand(0)); + if (MDName->getString() == LLVMContext::getMDProfKindName(MDProfKindId)) { + // Replace. + setMetadata(LLVMContext::MD_prof, NewNode); + return; + } + if (NewNode == nullptr) + return; + // Promote to two-level. + LLVMContext &Ctx = getContext(); + auto TempNode = MDNode::getTemporary(Ctx, None); + auto NewProfNode = MDNode::get(Ctx, {TempNode.get(), ProfNode, NewNode}); + NewProfNode->replaceOperandWith(0, NewProfNode); + setMetadata(LLVMContext::MD_prof, NewProfNode); + return; + } + // Two-level. + for (unsigned I = 1; I < ProfNode->getNumOperands(); ++I) { + MDNode *MDI = cast(ProfNode->getOperand(I)); + MDString *MDName = cast(MDI->getOperand(0)); + if (MDName->getString() == LLVMContext::getMDProfKindName(MDProfKindId)) { + if (NewNode == nullptr) { + // Remove. + SmallVector Ops; + Ops.reserve(ProfNode->getNumOperands() - 1); + for (unsigned J = 1; J < ProfNode->getNumOperands(); ++J) + if (I != J) { + MDNode *MDJ = cast(ProfNode->getOperand(J)); + Ops.push_back(MDJ); + } + if (Ops.size() == 0) { + // No metadata left. Clear it. + setMetadata(LLVMContext::MD_prof, nullptr); + } else if (Ops.size() == 1) { + // One remaining. Demote to single-level. + setMetadata(LLVMContext::MD_prof, cast(Ops[0])); + } else { + setMetadata(LLVMContext::MD_prof, MDNode::get(getContext(), Ops)); + } + } else { + // Replace. + ProfNode->replaceOperandWith(I, NewNode); + } + return; + } + } + if (NewNode == nullptr) + return; + // Append. + SmallVector Ops; + Ops.reserve(ProfNode->getNumOperands() + 1); + for (Metadata *MD : ProfNode->operands()) + Ops.push_back(MD); + Ops.push_back(NewNode); + setMetadata(LLVMContext::MD_prof, MDNode::get(getContext(), Ops)); +} + void Instruction::clearMetadataHashEntries() { assert(hasMetadataHashEntry() && "Caller should check"); getContext().pImpl->InstructionMetadata.erase(this); Index: lib/ProfileData/InstrProf.cpp =================================================================== --- lib/ProfileData/InstrProf.cpp +++ lib/ProfileData/InstrProf.cpp @@ -850,7 +850,7 @@ if (--MDCount == 0) break; } - Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); + Inst.setProfMetadata(LLVMContext::MD_PROF_VP, MDNode::get(Ctx, Vals)); } bool getValueProfDataFromInst(const Instruction &Inst, @@ -858,7 +858,7 @@ uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC) { - MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); + MDNode *MD = Inst.getProfMetadata(LLVMContext::MD_PROF_VP); if (!MD) return false; @@ -867,14 +867,6 @@ if (NOps < 5) return false; - // Operand 0 is a string tag "VP": - MDString *Tag = cast(MD->getOperand(0)); - if (!Tag) - return false; - - if (!Tag->getString().equals("VP")) - return false; - // Now check kind: ConstantInt *KindInt = mdconst::dyn_extract(MD->getOperand(1)); if (!KindInt) Index: lib/Transforms/IPO/CrossDSOCFI.cpp =================================================================== --- lib/Transforms/IPO/CrossDSOCFI.cpp +++ lib/Transforms/IPO/CrossDSOCFI.cpp @@ -156,7 +156,7 @@ BitsetTestFn, {&Addr, MetadataAsValue::get( Ctx, ConstantAsMetadata::get(CaseTypeId))}); BranchInst *BI = IRBTest.CreateCondBr(Test, ExitBB, TrapBB); - BI->setMetadata(LLVMContext::MD_prof, VeryLikelyWeights); + BI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, VeryLikelyWeights); SI->addCase(CaseTypeId, TestBB); ++NumTypeIds; Index: lib/Transforms/IPO/LowerTypeTests.cpp =================================================================== --- lib/Transforms/IPO/LowerTypeTests.cpp +++ lib/Transforms/IPO/LowerTypeTests.cpp @@ -635,8 +635,9 @@ BasicBlock *Then = InitialBB->splitBasicBlock(CI->getIterator()); BasicBlock *Else = Br->getSuccessor(1); BranchInst *NewBr = BranchInst::Create(Then, Else, OffsetInRange); - NewBr->setMetadata(LLVMContext::MD_prof, - Br->getMetadata(LLVMContext::MD_prof)); + NewBr->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, + Br->getProfMetadata(LLVMContext::MD_PROF_branch_weights)); ReplaceInstWithInst(InitialBB->getTerminator(), NewBr); // Update phis in Else resulting from InitialBB being split Index: lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- lib/Transforms/IPO/SampleProfile.cpp +++ lib/Transforms/IPO/SampleProfile.cpp @@ -1263,7 +1263,8 @@ } else if (!dyn_cast(&I)) { SmallVector Weights; Weights.push_back(BlockWeights[BB]); - I.setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + I.setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDB.createBranchWeights(Weights)); } } } @@ -1312,7 +1313,7 @@ // weights, the second pass does not need to set it. if (MaxWeight > 0 && !TI->extractProfTotalWeight(TempWeight)) { DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n"); - TI->setMetadata(llvm::LLVMContext::MD_prof, + TI->setProfMetadata(llvm::LLVMContext::MD_PROF_branch_weights, MDB.createBranchWeights(Weights)); ORE->emit(OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst) << "most popular destination for conditional branches at " Index: lib/Transforms/Instrumentation/IndirectCallPromotion.cpp =================================================================== --- lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -486,7 +486,7 @@ NewInst); // Clear the value profile data. - NewInst->setMetadata(LLVMContext::MD_prof, nullptr); + NewInst->setProfMetadata(LLVMContext::MD_PROF_VP, nullptr); CallSite NewCS(NewInst); FunctionType *DirectCalleeType = DirectCallee->getFunctionType(); unsigned ParamNum = DirectCalleeType->getFunctionNumParams(); @@ -567,7 +567,8 @@ Weights.push_back(Count); MDBuilder MDB(NewInst->getContext()); if (Instruction *DI = dyn_cast(NewInst->stripPointerCasts())) - DI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + DI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDB.createBranchWeights(Weights)); } // Move Inst from MergeBB to IndirectCallBB. @@ -645,7 +646,7 @@ Changed = true; // Adjust the MD.prof metadata. First delete the old one. - I->setMetadata(LLVMContext::MD_prof, nullptr); + I->setProfMetadata(LLVMContext::MD_PROF_VP, nullptr); // If all promoted, we don't need the MD.prof metadata. if (TotalCount == 0 || NumPromoted == NumVals) continue; Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1124,7 +1124,7 @@ MaxCount = EdgeCount; EdgeCounts[SuccNum] = EdgeCount; } - setProfMetadata(M, TI, EdgeCounts, MaxCount); + setBranchWeightsMetadata(M, TI, EdgeCounts, MaxCount); } } @@ -1157,7 +1157,7 @@ SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0); uint64_t MaxCount = std::max(SCounts[0], SCounts[1]); if (MaxCount) - setProfMetadata(F.getParent(), &SI, SCounts, MaxCount); + setBranchWeightsMetadata(F.getParent(), &SI, SCounts, MaxCount); } void SelectInstVisitor::visitSelectInst(SelectInst &SI) { @@ -1478,8 +1478,9 @@ } namespace llvm { -void setProfMetadata(Module *M, Instruction *TI, ArrayRef EdgeCounts, - uint64_t MaxCount) { +void setBranchWeightsMetadata(Module *M, Instruction *TI, + ArrayRef EdgeCounts, + uint64_t MaxCount) { MDBuilder MDB(M->getContext()); assert(MaxCount > 0 && "Bad max count"); uint64_t Scale = calculateCountScale(MaxCount); @@ -1490,7 +1491,8 @@ DEBUG(dbgs() << "Weight is: "; for (const auto &W : Weights) { dbgs() << W << " "; } dbgs() << "\n";); - TI->setMetadata(llvm::LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + TI->setProfMetadata(llvm::LLVMContext::MD_PROF_branch_weights, + MDB.createBranchWeights(Weights)); if (EmitBranchProbability) { std::string BrCondStr = getBranchCondString(TI); if (BrCondStr.empty()) Index: lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp +++ lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp @@ -351,7 +351,7 @@ SwitchInst *SI = IRB.CreateSwitch(SizeVar, DefaultBB, SizeIds.size()); // Clear the value profile data. - MI->setMetadata(LLVMContext::MD_prof, nullptr); + MI->setProfMetadata(LLVMContext::MD_PROF_VP, nullptr); // If all promoted, we don't need the MD.prof metadata. if (SavedRemainCount > 0 || Version != NumVals) // Otherwise we need update with the un-promoted records back. @@ -376,7 +376,7 @@ SI->addCase(CaseSizeId, CaseBB); DEBUG(dbgs() << *CaseBB << "\n"); } - setProfMetadata(Func.getParent(), SI, CaseCounts, MaxCount); + setBranchWeightsMetadata(Func.getParent(), SI, CaseCounts, MaxCount); DEBUG(dbgs() << *BB << "\n"); DEBUG(dbgs() << *DefaultBB << "\n"); Index: lib/Transforms/Scalar/JumpThreading.cpp =================================================================== --- lib/Transforms/Scalar/JumpThreading.cpp +++ lib/Transforms/Scalar/JumpThreading.cpp @@ -266,8 +266,8 @@ Weights.push_back(BP.getCompl().getNumerator()); Weights.push_back(BP.getNumerator()); } - PredBr->setMetadata(LLVMContext::MD_prof, - MDBuilder(PredBr->getParent()->getContext()) + PredBr->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(PredBr->getParent()->getContext()) .createBranchWeights(Weights)); } } @@ -1990,14 +1990,11 @@ const TerminatorInst *TI = BB->getTerminator(); assert(TI->getNumSuccessors() > 1 && "not a split"); - MDNode *WeightsNode = TI->getMetadata(LLVMContext::MD_prof); + MDNode *WeightsNode = + TI->getProfMetadata(LLVMContext::MD_PROF_branch_weights); if (!WeightsNode) return false; - MDString *MDName = cast(WeightsNode->getOperand(0)); - if (MDName->getString() != "branch_weights") - return false; - // Ensure there are weights for all of the successors. Note that the first // operand to the metadata node is a name, not a weight. return WeightsNode->getNumOperands() == TI->getNumSuccessors() + 1; @@ -2093,8 +2090,8 @@ Weights.push_back(Prob.getNumerator()); auto TI = BB->getTerminator(); - TI->setMetadata( - LLVMContext::MD_prof, + TI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, MDBuilder(TI->getParent()->getContext()).createBranchWeights(Weights)); } } Index: lib/Transforms/Scalar/LowerExpectIntrinsic.cpp =================================================================== --- lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -77,8 +77,8 @@ else Weights[Case.getCaseIndex() + 1] = LikelyBranchWeight; - SI.setMetadata(LLVMContext::MD_prof, - MDBuilder(CI->getContext()).createBranchWeights(Weights)); + SI.setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(CI->getContext()).createBranchWeights(Weights)); SI.setCondition(ArgValue); return true; @@ -219,12 +219,12 @@ }; if (IsOpndComingFromSuccessor(BI->getSuccessor(1))) - BI->setMetadata( - LLVMContext::MD_prof, + BI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight)); else if (IsOpndComingFromSuccessor(BI->getSuccessor(0))) - BI->setMetadata( - LLVMContext::MD_prof, + BI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight)); } } @@ -288,7 +288,7 @@ else Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); - BSI.setMetadata(LLVMContext::MD_prof, Node); + BSI.setProfMetadata(LLVMContext::MD_PROF_branch_weights, Node); if (CmpI) CmpI->setOperand(0, ArgValue); Index: lib/Transforms/Scalar/LowerGuardIntrinsic.cpp =================================================================== --- lib/Transforms/Scalar/LowerGuardIntrinsic.cpp +++ lib/Transforms/Scalar/LowerGuardIntrinsic.cpp @@ -68,8 +68,9 @@ CheckBI->setMetadata(LLVMContext::MD_make_implicit, MD); MDBuilder MDB(CI->getContext()); - CheckBI->setMetadata(LLVMContext::MD_prof, - MDB.createBranchWeights(PredicatePassBranchWeight, 1)); + CheckBI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, + MDB.createBranchWeights(PredicatePassBranchWeight, 1)); IRBuilder<> B(DeoptBlockTerm); auto *DeoptCall = B.CreateCall(DeoptIntrinsic, Args, {DeoptOB}, ""); Index: lib/Transforms/Utils/BasicBlockUtils.cpp =================================================================== --- lib/Transforms/Utils/BasicBlockUtils.cpp +++ lib/Transforms/Utils/BasicBlockUtils.cpp @@ -629,7 +629,8 @@ CheckTerm->setDebugLoc(SplitBefore->getDebugLoc()); BranchInst *HeadNewTerm = BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/Tail, Cond); - HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); + HeadNewTerm->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + BranchWeights); ReplaceInstWithInst(HeadOldTerm, HeadNewTerm); if (DT) { @@ -671,7 +672,8 @@ (*ElseTerm)->setDebugLoc(SplitBefore->getDebugLoc()); BranchInst *HeadNewTerm = BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/ElseBlock, Cond); - HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); + HeadNewTerm->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + BranchWeights); ReplaceInstWithInst(HeadOldTerm, HeadNewTerm); } Index: lib/Transforms/Utils/CodeExtractor.cpp =================================================================== --- lib/Transforms/Utils/CodeExtractor.cpp +++ lib/Transforms/Utils/CodeExtractor.cpp @@ -976,8 +976,8 @@ BranchProbability BP(Weight.Amount, BranchDist.Total); BPI->setEdgeProbability(CodeReplacer, Weight.TargetNode.Index, BP); } - TI->setMetadata( - LLVMContext::MD_prof, + TI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, MDBuilder(TI->getContext()).createBranchWeights(BranchWeights)); } Index: lib/Transforms/Utils/Local.cpp =================================================================== --- lib/Transforms/Utils/Local.cpp +++ lib/Transforms/Utils/Local.cpp @@ -60,6 +60,46 @@ // Local constant propagation. // +template +static void mergeProfWeightToDefaultCase(SwitchInst *SI, + SwitchInst::CaseIt CaseIt, + MDNode *MD, unsigned MDProfKindId, + LLVMContext &Ctx) { + assert(MD == SI->getProfMetadata(MDProfKindId) && "Mismatching MD"); + // Collect branch weights into a vector. + SmallVector Weights; + for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) { + auto *CI = mdconst::extract(MD->getOperand(i)); + Weights.push_back(CI->getValue().getZExtValue()); + } + // Merge weight of this case to the default weight. + unsigned idx = CaseIt->getCaseIndex(); + Weights[0] += Weights[idx+1]; + // Remove weight for this case. + std::swap(Weights[idx+1], Weights.back()); + Weights.pop_back(); + SI->setProfMetadata(MDProfKindId, MDBuilder(Ctx). + createProfWeights( + MDProfKindId, Weights)); +} + +template +static void copyProfWeightToBranch(MDNode *MD, unsigned MDProfKindId, + BranchInst *Br, LLVMContext &Ctx) { + ConstantInt *SICase = + mdconst::dyn_extract(MD->getOperand(2)); + ConstantInt *SIDef = + mdconst::dyn_extract(MD->getOperand(1)); + assert(SICase && SIDef); + // The TrueWeight should be the weight for the single case of SI. + Br->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, + MDBuilder(Ctx). + createProfWeights(MDProfKindId, + SICase->getValue().getZExtValue(), + SIDef->getValue().getZExtValue())); +} + /// ConstantFoldTerminator - If a terminator instruction is predicated on a /// constant value, convert it into an unconditional branch to the constant /// destination. This is a nontrivial operation because the successors of this @@ -142,27 +182,16 @@ // Check to see if this branch is going to the same place as the default // dest. If so, eliminate it as an explicit compare. if (i->getCaseSuccessor() == DefaultDest) { - MDNode *MD = SI->getMetadata(LLVMContext::MD_prof); + MDNode *MDBranchWeight = + SI->getProfMetadata(LLVMContext::MD_PROF_branch_weights); unsigned NCases = SI->getNumCases(); // Fold the case metadata into the default if there will be any branches // left, unless the metadata doesn't match the switch. - if (NCases > 1 && MD && MD->getNumOperands() == 2 + NCases) { - // Collect branch weights into a vector. - SmallVector Weights; - for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e; - ++MD_i) { - auto *CI = mdconst::extract(MD->getOperand(MD_i)); - Weights.push_back(CI->getValue().getZExtValue()); - } - // Merge weight of this case to the default weight. - unsigned idx = i->getCaseIndex(); - Weights[0] += Weights[idx+1]; - // Remove weight for this case. - std::swap(Weights[idx+1], Weights.back()); - Weights.pop_back(); - SI->setMetadata(LLVMContext::MD_prof, - MDBuilder(BB->getContext()). - createBranchWeights(Weights)); + if (NCases > 1 && MDBranchWeight && + MDBranchWeight->getNumOperands() == 2 + NCases) { + mergeProfWeightToDefaultCase( + SI, i, MDBranchWeight, + LLVMContext::MD_PROF_branch_weights, BB->getContext()); } // Remove this entry. DefaultDest->removePredecessor(SI->getParent()); @@ -222,18 +251,12 @@ BranchInst *NewBr = Builder.CreateCondBr(Cond, FirstCase.getCaseSuccessor(), SI->getDefaultDest()); - MDNode *MD = SI->getMetadata(LLVMContext::MD_prof); - if (MD && MD->getNumOperands() == 3) { - ConstantInt *SICase = - mdconst::dyn_extract(MD->getOperand(2)); - ConstantInt *SIDef = - mdconst::dyn_extract(MD->getOperand(1)); - assert(SICase && SIDef); - // The TrueWeight should be the weight for the single case of SI. - NewBr->setMetadata(LLVMContext::MD_prof, - MDBuilder(BB->getContext()). - createBranchWeights(SICase->getValue().getZExtValue(), - SIDef->getValue().getZExtValue())); + MDNode *MDBranchWeight = + SI->getProfMetadata(LLVMContext::MD_PROF_branch_weights); + if (MDBranchWeight && MDBranchWeight->getNumOperands() == 3) { + copyProfWeightToBranch( + MDBranchWeight, LLVMContext::MD_PROF_branch_weights, + NewBr, BB->getContext()); } // Update make.implicit metadata to the newly-created conditional branch. Index: lib/Transforms/Utils/LoopUnrollPeel.cpp =================================================================== --- lib/Transforms/Utils/LoopUnrollPeel.cpp +++ lib/Transforms/Utils/LoopUnrollPeel.cpp @@ -252,7 +252,7 @@ MDNode *WeightNode = HeaderIdx ? MDB.createBranchWeights(ExitWeight, FallThruWeight) : MDB.createBranchWeights(FallThruWeight, ExitWeight); - LatchBR->setMetadata(LLVMContext::MD_prof, WeightNode); + LatchBR->setProfMetadata(LLVMContext::MD_PROF_branch_weights, WeightNode); } } @@ -534,7 +534,7 @@ MDNode *WeightNode = HeaderIdx ? MDB.createBranchWeights(ExitWeight, BackEdgeWeight) : MDB.createBranchWeights(BackEdgeWeight, ExitWeight); - LatchBR->setMetadata(LLVMContext::MD_prof, WeightNode); + LatchBR->setProfMetadata(LLVMContext::MD_PROF_branch_weights, WeightNode); } // If the loop is nested, we changed the parent loop, update SE. Index: lib/Transforms/Utils/SimplifyCFG.cpp =================================================================== --- lib/Transforms/Utils/SimplifyCFG.cpp +++ lib/Transforms/Utils/SimplifyCFG.cpp @@ -837,7 +837,7 @@ // Collect branch weights into a vector. SmallVector Weights; - MDNode *MD = SI->getMetadata(LLVMContext::MD_prof); + MDNode *MD = SI->getProfMetadata(LLVMContext::MD_PROF_branch_weights); bool HasWeight = MD && (MD->getNumOperands() == 2 + SI->getNumCases()); if (HasWeight) for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e; @@ -857,8 +857,8 @@ } } if (HasWeight && Weights.size() >= 2) - SI->setMetadata(LLVMContext::MD_prof, - MDBuilder(SI->getParent()->getContext()) + SI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(SI->getParent()->getContext()) .createBranchWeights(Weights)); DEBUG(dbgs() << "Leaving: " << *TI << "\n"); @@ -933,12 +933,7 @@ } static inline bool HasBranchWeights(const Instruction *I) { - MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof); - if (ProfMD && ProfMD->getOperand(0)) - if (MDString *MDS = dyn_cast(ProfMD->getOperand(0))) - return MDS->getString().equals("branch_weights"); - - return false; + return I->getProfMetadata(LLVMContext::MD_PROF_branch_weights) != nullptr; } /// Get Weights of a given TerminatorInst, the default weight is at the front @@ -946,7 +941,7 @@ /// metadata. static void GetBranchWeights(TerminatorInst *TI, SmallVectorImpl &Weights) { - MDNode *MD = TI->getMetadata(LLVMContext::MD_prof); + MDNode *MD = TI->getProfMetadata(LLVMContext::MD_PROF_branch_weights); assert(MD); for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) { ConstantInt *CI = mdconst::extract(MD->getOperand(i)); @@ -1164,8 +1159,8 @@ SmallVector MDWeights(Weights.begin(), Weights.end()); - NewSI->setMetadata( - LLVMContext::MD_prof, + NewSI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, MDBuilder(BB->getContext()).createBranchWeights(MDWeights)); } @@ -2720,11 +2715,11 @@ SmallVector MDWeights(NewWeights.begin(), NewWeights.end()); - PBI->setMetadata( - LLVMContext::MD_prof, + PBI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, MDBuilder(BI->getContext()).createBranchWeights(MDWeights)); } else - PBI->setMetadata(LLVMContext::MD_prof, nullptr); + PBI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, nullptr); } else { // Update PHI nodes in the common successors. for (unsigned i = 0, e = PHIs.size(); i != e; ++i) { @@ -3290,8 +3285,8 @@ // Halve the weights if any of them cannot fit in an uint32_t FitWeights(NewWeights); - PBI->setMetadata(LLVMContext::MD_prof, - MDBuilder(BI->getContext()) + PBI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(BI->getContext()) .createBranchWeights(NewWeights[0], NewWeights[1])); } @@ -3330,8 +3325,8 @@ FitWeights(NewWeights); - NV->setMetadata(LLVMContext::MD_prof, - MDBuilder(BI->getContext()) + NV->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(BI->getContext()) .createBranchWeights(NewWeights[0], NewWeights[1])); } } @@ -3387,8 +3382,8 @@ // Create a conditional branch sharing the condition of the select. BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB); if (TrueWeight != FalseWeight) - NewBI->setMetadata(LLVMContext::MD_prof, - MDBuilder(OldTerm->getContext()) + NewBI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(OldTerm->getContext()) .createBranchWeights(TrueWeight, FalseWeight)); } } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) { @@ -3576,8 +3571,8 @@ Weights.push_back(Weights[0]); SmallVector MDWeights(Weights.begin(), Weights.end()); - SI->setMetadata( - LLVMContext::MD_prof, + SI->setProfMetadata( + LLVMContext::MD_PROF_branch_weights, MDBuilder(SI->getContext()).createBranchWeights(MDWeights)); } } @@ -4305,8 +4300,8 @@ TrueWeight /= 2; FalseWeight /= 2; } - NewBI->setMetadata(LLVMContext::MD_prof, - MDBuilder(SI->getContext()) + NewBI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(SI->getContext()) .createBranchWeights((uint32_t)TrueWeight, (uint32_t)FalseWeight)); } @@ -4405,8 +4400,8 @@ } if (HasWeight && Weights.size() >= 2) { SmallVector MDWeights(Weights.begin(), Weights.end()); - SI->setMetadata(LLVMContext::MD_prof, - MDBuilder(SI->getParent()->getContext()) + SI->setProfMetadata(LLVMContext::MD_PROF_branch_weights, + MDBuilder(SI->getParent()->getContext()) .createBranchWeights(MDWeights)); } Index: test/Analysis/BlockFrequencyInfo/two_level_prof.ll =================================================================== --- /dev/null +++ test/Analysis/BlockFrequencyInfo/two_level_prof.ll @@ -0,0 +1,54 @@ +; RUN: opt < %s -analyze -block-freq | FileCheck %s +; RUN: opt < %s -passes='print' -disable-output 2>&1 | FileCheck %s + +define i32 @test1(i32 %i, i32 %a, i32 %b) { +; CHECK-LABEL: Printing analysis {{.*}} for function 'test1': +; CHECK-NEXT: block-frequency-info: test1 +; CHECK-NEXT: entry: float = 1.0, int = [[ENTRY:[0-9]+]] +entry: + %cond = icmp ult i32 %i, 42 + br i1 %cond, label %then, label %else, !prof !0 + +; The 'then' branch is predicted more likely via branch weight metadata. +; CHECK-NEXT: then: float = 0.9411{{[0-9]*}}, +then: + br label %exit + +; CHECK-NEXT: else: float = 0.05882{{[0-9]*}}, +else: + br label %exit + +; CHECK-NEXT: exit: float = 1.0, int = [[ENTRY]] +exit: + %result = phi i32 [ %a, %then ], [ %b, %else ] + ret i32 %result +} + +define i32 @test2(i32 %i, i32 %a, i32 %b) { +; CHECK-LABEL: Printing analysis {{.*}} for function 'test2': +; CHECK-NEXT: block-frequency-info: test2 +; CHECK-NEXT: entry: float = 1.0, int = [[ENTRY:[0-9]+]] +entry: + %cond = icmp ult i32 %i, 42 + br i1 %cond, label %then, label %else, !prof !1 + +; The 'then' branch is predicted more likely via branch weight metadata. +; CHECK-NEXT: then: float = 0.9411{{[0-9]*}}, +then: + br label %exit + +; CHECK-NEXT: else: float = 0.05882{{[0-9]*}}, +else: + br label %exit + +; CHECK-NEXT: exit: float = 1.0, int = [[ENTRY]] +exit: + %result = phi i32 [ %a, %then ], [ %b, %else ] + ret i32 %result +} + +!0 = !{!"branch_weights", i32 64, i32 4} +!1 = !{!1, !2, !3, !4} +!2 = !{!"foo"} +!3 = !{!"branch_weights", i32 64, i32 4} +!4 = !{!"bar"} Index: test/Transforms/SimplifyCFG/preserve-branchweights.ll =================================================================== --- test/Transforms/SimplifyCFG/preserve-branchweights.ll +++ test/Transforms/SimplifyCFG/preserve-branchweights.ll @@ -48,7 +48,7 @@ ; CHECK-LABEL: @test2( entry: br i1 %a, label %X, label %Y, !prof !1 -; CHECK: br i1 %or.cond, label %Z, label %Y, !prof !2 +; CHECK: br i1 %or.cond, label %Z, label %Y, !prof !4 ; CHECK-NOT: !prof X: @@ -66,7 +66,7 @@ define void @test3(i1 %a, i1 %b) { ; CHECK-LABEL: @test3( -; CHECK: br i1 %or.cond, label %Z, label %Y, !prof !1 +; CHECK: br i1 %or.cond, label %Z, label %Y, !prof !3 entry: br i1 %a, label %X, label %Y, !prof !1 @@ -85,7 +85,7 @@ define void @test4(i1 %a, i1 %b) { ; CHECK-LABEL: @test4( -; CHECK: br i1 %or.cond, label %Z, label %Y, !prof !1 +; CHECK: br i1 %or.cond, label %Z, label %Y, !prof !3 entry: br i1 %a, label %X, label %Y @@ -114,7 +114,7 @@ ; CHECK: switch i32 %N, label %sw2 [ ; CHECK: i32 3, label %sw.bb1 ; CHECK: i32 2, label %sw.bb -; CHECK: ], !prof !3 +; CHECK: ], !prof !5 sw.bb: call void @helper(i32 0) @@ -147,7 +147,7 @@ ; CHECK: i32 3, label %sw.bb1 ; CHECK: i32 2, label %sw.bb ; CHECK: i32 4, label %sw.bb5 -; CHECK: ], !prof !4 +; CHECK: ], !prof !6 sw.bb: call void @helper(i32 0) @@ -182,7 +182,7 @@ ; CHECK-LABEL: @test1_swap( entry: br i1 %a, label %Y, label %X, !prof !0 -; CHECK: br i1 %or.cond, label %Y, label %Z, !prof !5 +; CHECK: br i1 %or.cond, label %Y, label %Z, !prof !7 X: %c = or i1 %b, false @@ -202,7 +202,7 @@ entry: %c = or i1 %b, false br i1 %a, label %Y, label %X, !prof !0 -; CHECK: br i1 %brmerge, label %Y, label %Z, !prof !6 +; CHECK: br i1 %brmerge, label %Y, label %Z, !prof !8 X: br i1 %c, label %Y, label %Z, !prof !6 @@ -221,7 +221,7 @@ ; CHECK-LABEL: @test8( entry: %lt = icmp slt i64 %x, %y -; CHECK: br i1 %lt, label %a, label %b, !prof !7 +; CHECK: br i1 %lt, label %a, label %b, !prof !9 %qux = select i1 %lt, i32 0, i32 2 switch i32 %qux, label %bees [ i32 0, label %a @@ -254,7 +254,7 @@ ; CHECK: i32 1, label %end ; CHECK: i32 2, label %end ; CHECK: i32 92, label %end -; CHECK: ], !prof !8 +; CHECK: ], !prof !10 a: call void @helper(i32 0) nounwind @@ -292,7 +292,7 @@ ; CHECK-LABEL: @test10( ; CHECK: %x.off = add i32 %x, -1 ; CHECK: %switch = icmp ult i32 %x.off, 3 -; CHECK: br i1 %switch, label %lor.end, label %lor.rhs, !prof !9 +; CHECK: br i1 %switch, label %lor.end, label %lor.rhs, !prof !11 } ; Remove dead cases from the switch. @@ -304,7 +304,7 @@ ], !prof !8 ; CHECK-LABEL: @test11( ; CHECK: %cond = icmp eq i32 %i, 24 -; CHECK: br i1 %cond, label %c, label %a, !prof !10 +; CHECK: br i1 %cond, label %c, label %a, !prof !12 a: call void @helper(i32 0) nounwind @@ -367,7 +367,7 @@ @max_regno = common global i32 0, align 4 define void @test14(i32* %old, i32 %final) { ; CHECK-LABEL: @test14 -; CHECK: br i1 %or.cond, label %for.exit, label %for.inc, !prof !11 +; CHECK: br i1 %or.cond, label %for.exit, label %for.inc, !prof !13 for.cond: br label %for.cond2 for.cond2: @@ -393,7 +393,7 @@ ; CHECK-LABEL: @HoistThenElseCodeToIf( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 %n, 0 -; CHECK-NEXT: [[DOT:%.*]] = select i1 [[TOBOOL]], i32 1, i32 234, !prof !12 +; CHECK-NEXT: [[DOT:%.*]] = select i1 [[TOBOOL]], i32 1, i32 234, !prof !14 ; CHECK-NEXT: ret i32 [[DOT]] ; entry: @@ -417,8 +417,8 @@ ; CHECK-LABEL: @SimplifyCondBranchToCondBranch( ; CHECK-NEXT: block1: ; CHECK-NEXT: [[BRMERGE:%.*]] = or i1 %cmpa, %cmpb -; CHECK-NEXT: [[DOTMUX:%.*]] = select i1 %cmpa, i32 0, i32 2, !prof !13 -; CHECK-NEXT: [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !14 +; CHECK-NEXT: [[DOTMUX:%.*]] = select i1 %cmpa, i32 0, i32 2, !prof !15 +; CHECK-NEXT: [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !16 ; CHECK-NEXT: ret i32 [[OUTVAL]] ; block1: @@ -444,8 +444,8 @@ ; CHECK-NEXT: [[CMPA_NOT:%.*]] = xor i1 %cmpa, true ; CHECK-NEXT: [[CMPB_NOT:%.*]] = xor i1 %cmpb, true ; CHECK-NEXT: [[BRMERGE:%.*]] = or i1 [[CMPA_NOT]], [[CMPB_NOT]] -; CHECK-NEXT: [[DOTMUX:%.*]] = select i1 [[CMPA_NOT]], i32 0, i32 2, !prof !15 -; CHECK-NEXT: [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !16 +; CHECK-NEXT: [[DOTMUX:%.*]] = select i1 [[CMPA_NOT]], i32 0, i32 2, !prof !17 +; CHECK-NEXT: [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !18 ; CHECK-NEXT: ret i32 [[OUTVAL]] ; block1: @@ -469,8 +469,8 @@ ; CHECK-NEXT: [[CMPA_NOT:%.*]] = xor i1 %cmpa, true ; CHECK-NEXT: [[CMPB_NOT:%.*]] = xor i1 %cmpb, true ; CHECK-NEXT: [[BRMERGE:%.*]] = or i1 [[CMPA_NOT]], [[CMPB_NOT]] -; CHECK-NEXT: [[DOTMUX:%.*]] = select i1 [[CMPA_NOT]], i32 0, i32 2, !prof !17 -; CHECK-NEXT: [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !18 +; CHECK-NEXT: [[DOTMUX:%.*]] = select i1 [[CMPA_NOT]], i32 0, i32 2, !prof !19 +; CHECK-NEXT: [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !20 ; CHECK-NEXT: ret i32 [[OUTVAL]] ; block1: @@ -505,23 +505,25 @@ !14 = !{!"branch_weights", i32 4, i32 7} ; CHECK: !0 = !{!"branch_weights", i32 5, i32 11} -; CHECK: !1 = !{!"branch_weights", i32 1, i32 3} -; CHECK: !2 = !{!"branch_weights", i32 1, i32 5} -; CHECK: !3 = !{!"branch_weights", i32 7, i32 1, i32 2} -; CHECK: !4 = !{!"branch_weights", i32 49, i32 12, i32 24, i32 35} -; CHECK: !5 = !{!"branch_weights", i32 11, i32 5} -; CHECK: !6 = !{!"branch_weights", i32 17, i32 15} -; CHECK: !7 = !{!"branch_weights", i32 9, i32 7} -; CHECK: !8 = !{!"branch_weights", i32 17, i32 9, i32 8, i32 7, i32 17} -; CHECK: !9 = !{!"branch_weights", i32 24, i32 33} -; CHECK: !10 = !{!"branch_weights", i32 8, i32 33} +; CHECK: !1 = distinct !{!1, !2, !3} +; CHECK: !2 = !{!"these_are_not_the_branch_weights_you_are_looking_for", i32 3, i32 5} +; CHECK: !3 = !{!"branch_weights", i32 1, i32 3} +; CHECK: !4 = !{!"branch_weights", i32 1, i32 5} +; CHECK: !5 = !{!"branch_weights", i32 7, i32 1, i32 2} +; CHECK: !6 = !{!"branch_weights", i32 49, i32 12, i32 24, i32 35} +; CHECK: !7 = !{!"branch_weights", i32 11, i32 5} +; CHECK: !8 = !{!"branch_weights", i32 17, i32 15} +; CHECK: !9 = !{!"branch_weights", i32 9, i32 7} +; CHECK: !10 = !{!"branch_weights", i32 17, i32 9, i32 8, i32 7, i32 17} +; CHECK: !11 = !{!"branch_weights", i32 24, i32 33} +; CHECK: !12 = !{!"branch_weights", i32 8, i32 33} ;; The false weight prints out as a negative integer here, but inside llvm, we ;; treat the weight as an unsigned integer. -; CHECK: !11 = !{!"branch_weights", i32 112017436, i32 -735157296} -; CHECK: !12 = !{!"branch_weights", i32 3, i32 5} -; CHECK: !13 = !{!"branch_weights", i32 22, i32 12} -; CHECK: !14 = !{!"branch_weights", i32 34, i32 21} -; CHECK: !15 = !{!"branch_weights", i32 33, i32 14} -; CHECK: !16 = !{!"branch_weights", i32 47, i32 8} -; CHECK: !17 = !{!"branch_weights", i32 6, i32 2} -; CHECK: !18 = !{!"branch_weights", i32 8, i32 2} +; CHECK: !13 = !{!"branch_weights", i32 112017436, i32 -735157296} +; CHECK: !14 = !{!"branch_weights", i32 3, i32 5} +; CHECK: !15 = !{!"branch_weights", i32 22, i32 12} +; CHECK: !16 = !{!"branch_weights", i32 34, i32 21} +; CHECK: !17 = !{!"branch_weights", i32 33, i32 14} +; CHECK: !18 = !{!"branch_weights", i32 47, i32 8} +; CHECK: !19 = !{!"branch_weights", i32 6, i32 2} +; CHECK: !20 = !{!"branch_weights", i32 8, i32 2}