Index: docs/BranchWeightMetadata.rst =================================================================== --- docs/BranchWeightMetadata.rst +++ docs/BranchWeightMetadata.rst @@ -115,3 +115,27 @@ Branch Weight Metatada is not proof against CFG changes. If terminator operands' are changed some action should be taken. In other case some misoptimizations may occur due to incorrent branch prediction information. + +Function Entry Counts +===================== + +To allow comparing different functions durint inter-procedural +analysis and optimization, ``MD_prof`` nodes can also be assigned to a +function header. The first operator is a string indicating the name of +the associated counter. + +Currently, one counter is supported: "function_entry_count". This is a +64-bit counter that indicates how many times was this function +invoked (in the case of instrumentation-based profiles). In the case +of sampling-based profiles, this counter is an approximation of how many +times the function was invoked. + +For example, in the code below, the instrumentation for function foo() +indicates that it was called 2,590 times at runtime. + +.. code-block:: llvm + +define i32 @foo() !prof !1 { + ret i32 0 +} +!1 = !{!"function_entry_count", i64 2590} Index: include/llvm/IR/Function.h =================================================================== --- include/llvm/IR/Function.h +++ include/llvm/IR/Function.h @@ -19,6 +19,7 @@ #define LLVM_IR_FUNCTION_H #include "llvm/ADT/iterator_range.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" @@ -194,6 +195,12 @@ AttributeSet::FunctionIndex, Kind, Value)); } + /// Set the entry count for this function. + void setEntryCount(uint64_t Count); + + /// Get the entry count for this function. + Optional getEntryCount() const; + /// @brief Return true if the function has the attribute. bool hasFnAttribute(Attribute::AttrKind Kind) const { return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, Kind); Index: include/llvm/IR/MDBuilder.h =================================================================== --- include/llvm/IR/MDBuilder.h +++ include/llvm/IR/MDBuilder.h @@ -60,6 +60,9 @@ /// \brief Return metadata containing a number of branch weights. MDNode *createBranchWeights(ArrayRef Weights); + /// Return metadata containing the entry count for a function. + MDNode *createFunctionEntryCount(uint64_t Count); + //===------------------------------------------------------------------===// // Range metadata. //===------------------------------------------------------------------===// Index: lib/IR/Function.cpp =================================================================== --- lib/IR/Function.cpp +++ lib/IR/Function.cpp @@ -19,10 +19,13 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/RWMutex.h" @@ -986,3 +989,19 @@ Attrs = Attrs.addAttribute(Ctx, AttributeSet::FunctionIndex, Kind, Value); F.setAttributes(Attrs); } + +void Function::setEntryCount(uint64_t Count) { + MDBuilder MDB(getContext()); + setMetadata(LLVMContext::MD_prof, MDB.createFunctionEntryCount(Count)); +} + +Optional Function::getEntryCount() const { + MDNode *MD = getMetadata(LLVMContext::MD_prof); + if (MD && MD->getOperand(0)) + if (MDString *MDS = dyn_cast(MD->getOperand(0))) + if (MDS->getString().equals("function_entry_count")) { + ConstantInt *CI = mdconst::extract(MD->getOperand(1)); + return CI->getValue().getZExtValue(); + } + return None; +} Index: lib/IR/MDBuilder.cpp =================================================================== --- lib/IR/MDBuilder.cpp +++ lib/IR/MDBuilder.cpp @@ -53,6 +53,16 @@ return MDNode::get(Context, Vals); } +MDNode *MDBuilder::createFunctionEntryCount(uint64_t Count) { + SmallVector Vals(2); + Vals[0] = createString("function_entry_count"); + + Type *Int64Ty = Type::getInt64Ty(Context); + Vals[1] = createConstant(ConstantInt::get(Int64Ty, Count)); + + return MDNode::get(Context, Vals); +} + MDNode *MDBuilder::createRange(const APInt &Lo, const APInt &Hi) { assert(Lo.getBitWidth() == Hi.getBitWidth() && "Mismatched bitwidths!"); Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -400,6 +400,8 @@ bool isReturnValue, const Value *V); void VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, const Value *V); + void VerifyFunctionMetadata( + const SmallVector, 4> MDs); void VerifyConstantExprBitcastType(const ConstantExpr *CE); void VerifyStatepoint(ImmutableCallSite CS); @@ -1463,6 +1465,33 @@ } } +void Verifier::VerifyFunctionMetadata( + const SmallVector, 4> MDs) { + if (MDs.empty()) + return; + + for (unsigned i = 0; i < MDs.size(); i++) { + if (MDs[i].first == LLVMContext::MD_prof) { + MDNode *MD = MDs[i].second; + Assert(MD->getNumOperands() >= 2, + "!prof annotations should have at least 2 operands", MD); + Assert(isa(MD->getOperand(0)), + "expected string with name of the !prof annotation", MD); + MDString *MDS = dyn_cast(MD->getOperand(0)); + StringRef ProfName = MDS->getString(); + if (ProfName.equals("function_entry_count")) { + Assert(MD->getNumOperands() == 2, + "expected exactly one argument for function_entry_count", MD); + Assert(isa(MD->getOperand(1)), + "expected integer argument to function_entry_count", MD); + } else { + Assert(false, StringRef("unexpected profile annotation ") + ProfName, + MD); + } + } + } +} + void Verifier::VerifyConstantExprBitcastType(const ConstantExpr *CE) { if (CE->getOpcode() != Instruction::BitCast) return; @@ -1691,6 +1720,7 @@ SmallVector, 4> MDs; F.getAllMetadata(MDs); assert(F.hasMetadata() != MDs.empty() && "Bit out-of-sync"); + VerifyFunctionMetadata(MDs); if (F.isMaterializable()) { // Function has a body somewhere we can't see. Index: test/Verifier/function-metadata.ll =================================================================== --- /dev/null +++ test/Verifier/function-metadata.ll @@ -0,0 +1,35 @@ +; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s + +define i32 @bad1() !prof !0 { + ret i32 0 +} + +!0 = !{i32 123, i32 3} +; CHECK: assembly parsed, but does not verify as correct! +; CHECK-NEXT: expected string with name of the !prof annotation +; CHECK-NEXT: !0 = !{i32 123, i32 3} + +define i32 @bad2() !prof !1 { + ret i32 0 +} + +!1 = !{!"function_entry_count"} +; CHECK-NEXT: !prof annotations should have at least 2 operands +; CHECK-NEXT: !1 = !{!"function_entry_count"} + + +define i32 @bad3() !prof !2 { + ret i32 0 +} + +!2 = !{!"function_entry_count", i32 -3, i64 200} +; CHECK-NEXT: expected exactly one argument for function_entry_count +; CHECK-NEXT: !2 = !{!"function_entry_count", i32 -3, i64 200} + +define i32 @bad4() !prof !3 { + ret i32 0 +} + +!3 = !{!"function_entry_count", !"string"} +; CHECK-NEXT: expected integer argument to function_entry_count +; CHECK-NEXT: !3 = !{!"function_entry_count", !"string"} Index: unittests/IR/MetadataTest.cpp =================================================================== --- unittests/IR/MetadataTest.cpp +++ unittests/IR/MetadataTest.cpp @@ -2272,4 +2272,12 @@ EXPECT_FALSE(verifyFunction(*F)); } +TEST_F(FunctionAttachmentTest, EntryCount) { + Function *F = getFunction("foo"); + EXPECT_FALSE(F->getEntryCount().hasValue()); + F->setEntryCount(12304); + EXPECT_TRUE(F->getEntryCount().hasValue()); + EXPECT_EQ(12304u, *F->getEntryCount()); +} + }