Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -25,6 +25,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Module.h" +#include "llvm/Transforms/IPO/FunctionAttrs.h" #include #include #include @@ -287,11 +288,24 @@ std::vector Args; }; + /// Function attribute flags. Used to track if a function accesses memory, + /// recurses or aliases. + struct FFlags { + unsigned ReadNone : 1; + unsigned ReadOnly : 1; + unsigned NoRecurse : 1; + unsigned ReturnDoesNotAlias : 1; + }; + private: /// Number of instructions (ignoring debug instructions, e.g.) computed /// during the initial compile step when the summary index is first built. unsigned InstCount; + /// Function attribute flags. Used to track if a function accesses memory, + /// recurses or aliases. + FFlags FunFlags; + /// List of call edge pairs from this function. std::vector CallGraphEdgeList; @@ -317,15 +331,16 @@ std::unique_ptr TIdInfo; public: - FunctionSummary(GVFlags Flags, unsigned NumInsts, std::vector Refs, - std::vector CGEdges, + FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags, + std::vector Refs, std::vector CGEdges, std::vector TypeTests, std::vector TypeTestAssumeVCalls, std::vector TypeCheckedLoadVCalls, std::vector TypeTestAssumeConstVCalls, std::vector TypeCheckedLoadConstVCalls) : GlobalValueSummary(FunctionKind, Flags, std::move(Refs)), - InstCount(NumInsts), CallGraphEdgeList(std::move(CGEdges)) { + InstCount(NumInsts), FunFlags(FunFlags), + CallGraphEdgeList(std::move(CGEdges)) { if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() || !TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() || !TypeCheckedLoadConstVCalls.empty()) @@ -341,6 +356,48 @@ return GVS->getSummaryKind() == FunctionKind; } + /// Get function attribute flags. + FFlags &fflags() { return FunFlags; } + + /// Iterator for a callees in a function summary. + class const_iterator + : public std::iterator { + std::vector::const_iterator I; + const FunctionSummary *fsumFromEdge(const EdgeTy &P) { + if (P.first.Ref && P.first.getSummaryList().size()) + return cast(P.first.getSummaryList().front().get()); + + // Create an empty functionsummary in the case of an external function + // (since scc_iterator doesn't accept nullptrs) + auto F = llvm::make_unique( + GVFlags(GlobalValue::LinkageTypes::AvailableExternallyLinkage, true, + false), + 0, FFlags{}, std::vector(), std::vector(), + std::vector(), std::vector(), + std::vector(), std::vector(), + std::vector()); + F->setOriginalName(P.first.Ref ? P.first.getGUID() : 0); + return F.get(); + } + + public: + const_iterator(std::vector::const_iterator I) : I(I){}; + const_iterator operator++(int) { + I++; + return *this; + } + bool operator==(const const_iterator &rhs) const { return I == rhs.I; } + bool operator!=(const const_iterator &rhs) const { return I != rhs.I; } + const FunctionSummary *operator*() { return fsumFromEdge(*I); } + }; + + const_iterator call_summaries_begin() const { + return const_iterator(CallGraphEdgeList.begin()); + } + const_iterator call_summaries_end() const { + return const_iterator(CallGraphEdgeList.end()); + } + /// Get the instruction count recorded for this function. unsigned instCount() const { return InstCount; } @@ -754,6 +811,39 @@ StringMap &ModuleToDefinedGVSummaries) const; }; +/// GraphTraits definition to build SCC for the index +template <> struct GraphTraits { + typedef const FunctionSummary *NodeRef; + typedef FunctionSummary::const_iterator ChildIteratorType; + + // Use the first callee as the entry node + static NodeRef getEntryNode(const FunctionSummary *F) { + if (F->call_summaries_begin() != F->call_summaries_end()) + return *(F->call_summaries_begin()); + + // use a dummy functionsummary when there is no entry node + auto S = llvm::make_unique( + FunctionSummary::GVFlags( + GlobalValue::LinkageTypes::AvailableExternallyLinkage, true, false), + 0, FunctionSummary::FFlags{}, std::vector(), + std::vector(), + std::vector(), + std::vector(), + std::vector(), + std::vector(), + std::vector()); + return S.get(); + } + + static ChildIteratorType child_begin(NodeRef N) { + return N->call_summaries_begin(); + } + + static ChildIteratorType child_end(NodeRef N) { + return N->call_summaries_end(); + } +}; + } // end namespace llvm #endif // LLVM_IR_MODULESUMMARYINDEX_H Index: include/llvm/IR/ModuleSummaryIndexYAML.h =================================================================== --- include/llvm/IR/ModuleSummaryIndexYAML.h +++ include/llvm/IR/ModuleSummaryIndexYAML.h @@ -206,8 +206,9 @@ GlobalValueSummary::GVFlags( static_cast(FSum.Linkage), FSum.NotEligibleToImport, FSum.Live), - 0, ArrayRef{}, ArrayRef{}, - std::move(FSum.TypeTests), std::move(FSum.TypeTestAssumeVCalls), + 0, FunctionSummary::FFlags{}, ArrayRef{}, + ArrayRef{}, std::move(FSum.TypeTests), + std::move(FSum.TypeTestAssumeVCalls), std::move(FSum.TypeCheckedLoadVCalls), std::move(FSum.TypeTestAssumeConstVCalls), std::move(FSum.TypeCheckedLoadConstVCalls))); Index: lib/Analysis/ModuleSummaryAnalysis.cpp =================================================================== --- lib/Analysis/ModuleSummaryAnalysis.cpp +++ lib/Analysis/ModuleSummaryAnalysis.cpp @@ -276,10 +276,16 @@ F.isVarArg(); GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport, /* Live = */ false); + FunctionSummary::FFlags FunFlags{ + F.hasFnAttribute(Attribute::ReadNone), + F.hasFnAttribute(Attribute::ReadOnly), + F.hasFnAttribute(Attribute::NoRecurse), + F.returnDoesNotAlias(), + }; auto FuncSummary = llvm::make_unique( - Flags, NumInsts, RefEdges.takeVector(), CallGraphEdges.takeVector(), - TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(), - TypeCheckedLoadVCalls.takeVector(), + Flags, NumInsts, FunFlags, RefEdges.takeVector(), + CallGraphEdges.takeVector(), TypeTests.takeVector(), + TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(), TypeTestAssumeConstVCalls.takeVector(), TypeCheckedLoadConstVCalls.takeVector()); if (NonRenamableLocal) @@ -427,11 +433,16 @@ /* Live = */ true); CantBePromoted.insert(GlobalValue::getGUID(Name)); // Create the appropriate summary type. - if (isa(GV)) { + if (Function *F = dyn_cast(GV)) { std::unique_ptr Summary = llvm::make_unique( - GVFlags, 0, ArrayRef{}, - ArrayRef{}, + GVFlags, 0, + FunctionSummary::FFlags{ + F->hasFnAttribute(Attribute::ReadNone), + F->hasFnAttribute(Attribute::ReadOnly), + F->hasFnAttribute(Attribute::NoRecurse), + F->returnDoesNotAlias()}, + ArrayRef{}, ArrayRef{}, ArrayRef{}, ArrayRef{}, ArrayRef{}, Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -860,6 +860,15 @@ } } +static FunctionSummary::FFlags getDecodedFFlags(uint64_t RawFlags) { + FunctionSummary::FFlags Flags; + Flags.ReadNone = RawFlags & 0x1; + Flags.ReadOnly = (RawFlags >> 1) & 0x1; + Flags.NoRecurse = (RawFlags >> 2) & 0x1; + Flags.ReturnDoesNotAlias = (RawFlags >> 3) & 0x1; + return Flags; +} + /// Decode the flags for GlobalValue in the summary. static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, uint64_t Version) { @@ -5036,9 +5045,9 @@ } const uint64_t Version = Record[0]; const bool IsOldProfileFormat = Version == 1; - if (Version < 1 || Version > 3) + if (Version < 1 || Version > 4) return error("Invalid summary version " + Twine(Version) + - ", 1, 2 or 3 expected"); + ", 1, 2, 3 or 4 expected"); Record.clear(); // Keep around the last seen summary to be used when we see an optional @@ -5088,9 +5097,9 @@ std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID); break; } - // FS_PERMODULE: [valueid, flags, instcount, numrefs, numrefs x valueid, - // n x (valueid)] - // FS_PERMODULE_PROFILE: [valueid, flags, instcount, numrefs, + // FS_PERMODULE: [valueid, flags, instcount, fflags, numrefs, + // numrefs x valueid, n x (valueid)] + // FS_PERMODULE_PROFILE: [valueid, flags, instcount, fflags, numrefs, // numrefs x valueid, // n x (valueid, hotness)] case bitc::FS_PERMODULE: @@ -5098,14 +5107,21 @@ unsigned ValueID = Record[0]; uint64_t RawFlags = Record[1]; unsigned InstCount = Record[2]; + uint64_t RawFunFlags = 0; unsigned NumRefs = Record[3]; + int RefListStartIndex = 4; + if (Version >= 4) { + RawFunFlags = Record[3]; + NumRefs = Record[4]; + RefListStartIndex = 5; + } + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); // The module path string ref set in the summary must be owned by the // index's module string table. Since we don't have a module path // string table section in the per-module index, we create a single // module path string table entry with an empty (0) ID to take // ownership. - static int RefListStartIndex = 4; int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; assert(Record.size() >= RefListStartIndex + NumRefs && "Record size inconsistent with number of references"); @@ -5116,8 +5132,9 @@ ArrayRef(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile); auto FS = llvm::make_unique( - Flags, InstCount, std::move(Refs), std::move(Calls), - std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), + Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), + std::move(Calls), std::move(PendingTypeTests), + std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), std::move(PendingTypeCheckedLoadConstVCalls)); @@ -5176,9 +5193,9 @@ TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); break; } - // FS_COMBINED: [valueid, modid, flags, instcount, numrefs, + // FS_COMBINED: [valueid, modid, flags, instcount, fflags, numrefs, // numrefs x valueid, n x (valueid)] - // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, numrefs, + // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, fflags, numrefs, // numrefs x valueid, n x (valueid, hotness)] case bitc::FS_COMBINED: case bitc::FS_COMBINED_PROFILE: { @@ -5186,9 +5203,17 @@ uint64_t ModuleId = Record[1]; uint64_t RawFlags = Record[2]; unsigned InstCount = Record[3]; + uint64_t RawFunFlags = 0; unsigned NumRefs = Record[4]; + int RefListStartIndex = 5; + + if (Version >= 4) { + RawFunFlags = Record[4]; + NumRefs = Record[5]; + RefListStartIndex = 6; + } + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); - static int RefListStartIndex = 5; int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; assert(Record.size() >= RefListStartIndex + NumRefs && "Record size inconsistent with number of references"); @@ -5200,8 +5225,9 @@ IsOldProfileFormat, HasProfile); ValueInfo VI = getValueInfoFromValueId(ValueID).first; auto FS = llvm::make_unique( - Flags, InstCount, std::move(Refs), std::move(Edges), - std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls), + Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs), + std::move(Edges), std::move(PendingTypeTests), + std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), std::move(PendingTypeCheckedLoadConstVCalls)); Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -895,6 +895,15 @@ return getEncodedLinkage(GV.getLinkage()); } +static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) { + uint64_t RawFlags = 0; + RawFlags |= Flags.ReadNone; + RawFlags |= (Flags.ReadOnly << 1); + RawFlags |= (Flags.NoRecurse << 2); + RawFlags |= (Flags.ReturnDoesNotAlias << 3); + return RawFlags; +} + // Decode the flags for GlobalValue in the summary static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { uint64_t RawFlags = 0; @@ -1703,7 +1712,7 @@ Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getVariable())); Record.push_back(VE.getMetadataOrNullID(N->getExpression())); - + Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR_EXPR, Record, Abbrev); Record.clear(); } @@ -3293,6 +3302,7 @@ NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); + NameVals.push_back(getEncodedFFlags(FS->fflags())); NameVals.push_back(FS->refs().size()); for (auto &RI : FS->refs()) @@ -3346,7 +3356,7 @@ // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 3; +static const uint64_t INDEX_VERSION = 4; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. @@ -3379,6 +3389,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3391,6 +3402,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3476,6 +3488,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3489,6 +3502,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -3574,6 +3588,7 @@ NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); + NameVals.push_back(getEncodedFFlags(FS->fflags())); // Fill in below NameVals.push_back(0); @@ -3585,7 +3600,7 @@ NameVals.push_back(*RefValueId); Count++; } - NameVals[4] = Count; + NameVals[5] = Count; bool HasProfileData = false; for (auto &EI : FS->calls()) { Index: test/Bitcode/summary_version.ll =================================================================== --- test/Bitcode/summary_version.ll +++ test/Bitcode/summary_version.ll @@ -2,7 +2,7 @@ ; RUN: opt -module-summary %s -o - | llvm-bcanalyzer -dump | FileCheck %s ; CHECK: +; CHECK: Index: test/Bitcode/thinlto-alias.ll =================================================================== --- test/Bitcode/thinlto-alias.ll +++ test/Bitcode/thinlto-alias.ll @@ -14,7 +14,7 @@ ; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: ; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK-NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph-pgo.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-pgo.ll +++ test/Bitcode/thinlto-function-summary-callgraph-pgo.ll @@ -17,7 +17,7 @@ ; CHECK: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll +++ test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll @@ -29,7 +29,7 @@ ; CHECK-NEXT: ; op4=hot1 op6=cold op8=hot2 op10=hot4 op12=none1 op14=hot3 op16=none2 op18=none3 op20=123 -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll +++ test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll @@ -29,7 +29,7 @@ ; CHECK-NEXT: ; op4=hot1 op6=cold op8=hot2 op10=hot4 op12=none1 op14=hot3 op16=none2 op18=none3 op20=123 -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; COMBINED-NEXT: ; COMBINED_NEXT: Index: test/Bitcode/thinlto-function-summary-callgraph.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph.ll +++ test/Bitcode/thinlto-function-summary-callgraph.ll @@ -18,7 +18,7 @@ ; CHECK: ; CHECK: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-functionattrs.ll =================================================================== --- /dev/null +++ test/Bitcode/thinlto-function-summary-functionattrs.ll @@ -0,0 +1,29 @@ +; RUN: opt -module-summary %s -o %t.o +; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s + +; CHECK: +; CHECK-DAG: ; Function W contains a call to func3 as well as a reference to globalvar: ; op0=W op4=globalvar op5=func3 -; CHECK-DAG: +; CHECK-DAG: ; Function X contains call to foo, as well as address reference to foo ; which is in the same instruction as the call: ; op0=X op4=foo op5=foo -; CHECK-DAG: +; CHECK-DAG: ; Function Y contains call to func2, and ensures we don't incorrectly add ; a reference to it when reached while earlier analyzing the phi using its ; return value: ; op0=Y op4=func2 -; CHECK-DAG: +; CHECK-DAG: ; Function Z contains call to func2, and ensures we don't incorrectly add ; a reference to it when reached while analyzing subsequent use of its return ; value: ; op0=Z op4=func2 -; CHECK-DAG: +; CHECK-DAG: ; Variable bar initialization contains address reference to func: ; op0=bar op2=func ; CHECK-DAG: