diff --git a/llvm/lib/Transforms/IPO/PGHOContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/PGHOContextDisambiguation.cpp --- a/llvm/lib/Transforms/IPO/PGHOContextDisambiguation.cpp +++ b/llvm/lib/Transforms/IPO/PGHOContextDisambiguation.cpp @@ -107,6 +107,10 @@ /// Main entry point to perform analysis and transformations on graph. bool process(); + /// Perform cloning on the graph necessary to uniquely identify the allocation + /// behavior of an allocation based on its context. + void identifyClones(); + void dump() const; void print(raw_ostream &OS) const; @@ -275,6 +279,10 @@ } }; + /// Helper to remove callee edges that have allocation type None (due to not + /// carrying any context ids) after transformations. + void removeNoneTypeCalleeEdges(ContextNode *Node); + /// Saves edge removed from graph in the RemovedEdges set for subsequent /// deletion by deleteRemovedEdges at a point where it is no longer saved /// in any intermediate data structures. @@ -380,6 +388,17 @@ /// unioning their recorded alloc types. uint8_t computeAllocType(DenseSet &ContextIds); + /// Returns the alloction type of the intersection of the contexts of two + /// nodes (based on their provided context id sets), optimized for the case + /// when Node1Ids is smaller than Node2Ids. + uint8_t intersectAllocTypesImpl(const DenseSet &Node1Ids, + const DenseSet &Node2Ids); + + /// Returns the alloction type of the intersection of the contexts of two + /// nodes (based on their provided context id sets). + uint8_t intersectAllocTypes(const DenseSet &Node1Ids, + const DenseSet &Node2Ids); + /// Create a clone of Edge's callee and move Edge to that new callee node, /// performing the necessary context id and allocation type updates. /// If callee's caller edge iterator is supplied, it is updated when removing @@ -395,6 +414,12 @@ EdgeIter *CallerEdgeI = nullptr, bool NewClone = false); + /// Recursively perform cloning on the graph for the given Node and its + /// callers, in order to uniquely identify the allocation behavior of an + /// allocation given its context. + void identifyClones(ContextNode *Node, + DenseSet &Visited); + /// Map from each context ID to the AllocationType assigned to that context. std::map ContextIdToAllocationType; @@ -597,6 +622,25 @@ Caller->CalleeEdges.push_back(Edge); } +template +void CallsiteContextGraph< + DerivedCCG, FuncTy, CallTy>::removeNoneTypeCalleeEdges(ContextNode *Node) { + for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) { + auto *Edge = *EI; + if (Edge->AllocTypes == (uint8_t)AllocationType::None) { + assert(Edge->ContextIds.empty()); + Edge->Callee->eraseCallerEdge(Edge); + EI = Node->CalleeEdges.erase(EI); + // Save edges for later deletion, as they may be saved in data structures. + RemovedEdges.insert(Edge); + // Denote that the edge is removed by setting the nodes to null + Edge->Callee = nullptr; + Edge->Caller = nullptr; + } else + ++EI; + } +} + template void CallsiteContextGraph::recordRemovedEdge( ContextEdge *Edge) { @@ -666,6 +710,33 @@ return AllocType; } +template +uint8_t +CallsiteContextGraph::intersectAllocTypesImpl( + const DenseSet &Node1Ids, const DenseSet &Node2Ids) { + uint8_t BothTypes = + (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold; + uint8_t AllocType = (uint8_t)AllocationType::None; + for (auto Id : Node1Ids) { + if (!Node2Ids.count(Id)) + continue; + AllocType |= (uint8_t)ContextIdToAllocationType[Id]; + // Bail early if alloc type reached both, no further refinement. + if (AllocType == BothTypes) + return AllocType; + } + return AllocType; +} + +template +uint8_t CallsiteContextGraph::intersectAllocTypes( + const DenseSet &Node1Ids, const DenseSet &Node2Ids) { + if (Node1Ids.size() < Node2Ids.size()) + return intersectAllocTypesImpl(Node1Ids, Node2Ids); + else + return intersectAllocTypesImpl(Node2Ids, Node1Ids); +} + template ContextNode * CallsiteContextGraph::addAllocNode(CallInfo Call) { @@ -1363,6 +1434,7 @@ template CallsiteContextGraph::~CallsiteContextGraph() { + deleteRemovedEdges(); auto DeleteCallerEdges = [](ContextNode *Node) { for (auto *Edge : Node->CallerEdges) delete Edge; @@ -1801,6 +1873,196 @@ } } +template +void CallsiteContextGraph::identifyClones() { + DenseSet Visited; + for (auto &Entry : AllocationCallToContextNodeMap) + identifyClones(Entry.second, Visited); + + // Clean up edges removed during the cloning. + deleteRemovedEdges(); +} + +template +void CallsiteContextGraph::identifyClones( + ContextNode *Node, DenseSet &Visited) { + if (VerifyNodes) + checkNode(Node, /*CheckEdges=*/true); + assert(!Node->CloneOf); + + // If Node as a null call, then either it wasn't found in the module (regular + // LTO) or summary index (ThinLTO), or there were other conditions blocking + // cloning (e.g. recursion, calls multiple targets, etc). + // Do this here so that we don't try to recursively clone callers below, which + // isn't useful at least for this node. + if (!Node->hasCall()) + return; + +#ifndef NDEBUG + auto Insert = +#endif + Visited.insert(Node); + // We should not have visited this node yet. + assert(Insert.second); + // The recursive call to identifyClones may delete the current edge from the + // CallerEdges vector. Make a copy and iterate that, simpler than passing in + // an iterator and having recursive call erase from it. Other edges may also + // get removed during the recursion, which will have null Callee and Caller + // pointers (and are deleted later), so we skip those below. + auto CallerEdges = Node->CallerEdges; + for (auto &Edge : CallerEdges) { + // Skip any that have been removed by an earlier recursive call. + if (Edge->Callee == nullptr && Edge->Caller == nullptr) + continue; + // Ignore any caller we previously visited via another edge. + if (!Visited.count(Edge->Caller) && !Edge->Caller->CloneOf) { + identifyClones(Edge->Caller, Visited); + } + } + + // Check if we reached an unambiguous call or have no more callers. + if (hasSingleAllocType(Node->AllocTypes) || Node->CallerEdges.size() <= 1) + return; + + // We need to clone. + + // Helper to check if the alloc types for all edges recorded in the + // InAllocTypes vector match the alloc types for all edges in the Edges + // vector. + auto AllocTypesMatch = [](const std::vector &InAllocTypes, + const std::vector &Edges) { + return std::equal(InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(), + [](const uint8_t &l, const ContextEdge *r) { + // Can share if one of the edges is None type - don't + // care about the type along that edge as it doesn't + // exist for those context ids. + if (l == (uint8_t)AllocationType::None || + r->AllocTypes == (uint8_t)AllocationType::None) + return true; + return allocTypeToUse(l) == + allocTypeToUse(r->AllocTypes); + }); + }; + + // Try to keep the original version as alloc type NotCold. This will make + // cases with indirect calls or any other situation with an unknown call to + // the original function get the default behavior. We do this by sorting the + // CallerEdges of the Node we will clone by alloc type. + // + // Give NotCold edge the lowest sort priority so those edges are at the end of + // the caller edges vector, and stay on the original version (since the below + // code clones greedily until it finds all remaining edges have the same type + // and leaves the remaining ones on the original Node). + // + // We shouldn't actually have any None type edges, so the sorting priority for + // that is arbitrary, and we assert in that case below. + unsigned AllocTypeCloningPriority[] = {/*None*/ 3, /*NotCold*/ 4, /*Cold*/ 1, + /*NotColdCold*/ 2}; + std::sort(Node->CallerEdges.begin(), Node->CallerEdges.end(), + [&](const ContextEdge *A, const ContextEdge *B) { + assert(A->AllocTypes != (uint8_t)AllocationType::None && + B->AllocTypes != (uint8_t)AllocationType::None); + if (A->AllocTypes == B->AllocTypes) + // Use the first context id for each edge as a tie-breaker. + return *A->ContextIds.begin() < *B->ContextIds.begin(); + return AllocTypeCloningPriority[A->AllocTypes] < + AllocTypeCloningPriority[B->AllocTypes]; + }); + + assert(Node->AllocTypes != (uint8_t)AllocationType::None); + + // We need to be able to remove Edge from CallerEdges, so need to adjust + // iterator inside the loop. + for (auto EI = Node->CallerEdges.begin(); EI != Node->CallerEdges.end();) { + auto *Edge = *EI; + + // See if cloning the prior caller edge left this node with a single alloc + // type or a single caller. In that case no more cloning of Node is needed. + if (hasSingleAllocType(Node->AllocTypes) || Node->CallerEdges.size() <= 1) + break; + + // Compute the node callee edge alloc types corresponding to the context ids + // for this caller edge. + std::vector CalleeEdgeAllocTypesForCallerEdge; + for (auto *CalleeEdge : Node->CalleeEdges) + CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes( + CalleeEdge->getContextIds(), Edge->getContextIds())); + + // Don't clone if doing so will not disambiguate any alloc types amongst + // caller edges (including the callee edges that would be cloned). + // Otherwise we will simply move all edges to the clone. + // + // First check if by cloning we will disambiguate the caller allocation + // type from node's allocation type. Query allocTypeToUse so that we don't + // bother cloning to distinguish NotCold+Cold from NotCold. Note that + // neither of these should be None type. + // + // Then check if by cloning node at least one of the callee edges will be + // disambiguated by splitting out different context ids. + assert(Edge->AllocTypes != (uint8_t)AllocationType::None); + assert(Node->AllocTypes != (uint8_t)AllocationType::None); + if (allocTypeToUse(Edge->AllocTypes) == allocTypeToUse(Node->AllocTypes) && + AllocTypesMatch(CalleeEdgeAllocTypesForCallerEdge, Node->CalleeEdges)) { + ++EI; + continue; + } + + // First see if we can use an existing clone. Check each clone and its + // callee edges for matching alloc types. + ContextNode *Clone = nullptr; + for (auto *CurClone : Node->Clones) { + if (allocTypeToUse(CurClone->AllocTypes) != + allocTypeToUse(Edge->AllocTypes)) + continue; + + if (!AllocTypesMatch(CalleeEdgeAllocTypesForCallerEdge, + CurClone->CalleeEdges)) + continue; + Clone = CurClone; + break; + } + + if (Clone) + moveEdgeToExistingCalleeClone(Edge, Clone, &EI); + else + Clone = moveEdgeToNewCalleeClone(Edge, &EI); + + assert(EI == Node->CallerEdges.end() || + Node->AllocTypes != (uint8_t)AllocationType::None); + // Sanity check that no alloc types on clone or its edges are None. + assert(Clone->AllocTypes != (uint8_t)AllocationType::None); + assert(llvm::none_of(Clone->CallerEdges, [&](ContextEdge *E) { + return E->AllocTypes == (uint8_t)AllocationType::None; + })); + } + + // Cloning may have resulted in some cloned callee edges with type None, + // because they aren't carrying any contexts. Remove those edges. + for (auto *Clone : Node->Clones) { + removeNoneTypeCalleeEdges(Clone); + if (VerifyNodes) + checkNode(Clone, /*CheckEdges=*/true); + } + // We should still have some context ids on the original Node. + assert(!Node->ContextIds.empty()); + + // Remove any callee edges that ended up with alloc type None after creating + // clones and updating callee edges. + removeNoneTypeCalleeEdges(Node); + + // Sanity check that no alloc types on node or edges are None. + assert(Node->AllocTypes != (uint8_t)AllocationType::None); + assert(llvm::none_of(Node->CalleeEdges, [&](ContextEdge *E) { + return E->AllocTypes == (uint8_t)AllocationType::None; + })); + assert(llvm::none_of(Node->CallerEdges, [&](ContextEdge *E) { + return E->AllocTypes == (uint8_t)AllocationType::None; + })); + + if (VerifyNodes) + checkNode(Node, /*CheckEdges=*/true); +} + template bool CallsiteContextGraph::process() { if (DumpCCG) { @@ -1814,6 +2076,19 @@ check(); } + identifyClones(); + + if (VerifyCCG) { + check(); + } + + if (DumpCCG) { + dbgs() << "CCG after cloning:\n"; + dbgs() << *this; + } + if (ExportToDot) + exportToDot("ccg.cloned.dot"); + return false; } diff --git a/llvm/test/ThinLTO/X86/pgho-basic.ll b/llvm/test/ThinLTO/X86/pgho-basic.ll --- a/llvm/test/ThinLTO/X86/pgho-basic.ll +++ b/llvm/test/ThinLTO/X86/pgho-basic.ll @@ -40,6 +40,8 @@ ; RUN: -o %t.out 2>&1 | FileCheck %s --check-prefix=DUMP ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOT +;; We should have cloned bar, baz, and foo, for the cold memory allocation. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'pgho-basic.ll' source_filename = "pgho-basic.ll" @@ -196,6 +198,82 @@ ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 ; DUMP: CallerEdges: +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[BAR:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 1 StackIds: 2, 3, 0 +; DUMP: AllocType 2 StackIds: 2, 3, 1 +; DUMP: (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[BAZ:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[BAZ]] +; DUMP: Callee: 10756268697391741933 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[BAZ]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[FOO]] +; DUMP: Callee: 17547784407117670007 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[MAIN1]] +; DUMP: Callee: 6988045695824228603 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: + +; DUMP: Node [[BAR2:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 1 StackIds: 2, 3, 0 +; DUMP: AllocType 2 StackIds: 2, 3, 1 +; DUMP: (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[BAZ2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[BAZ2]] +; DUMP: Callee: 10756268697391741933 (_Z3barv) Clones: 0 StackIds: 2 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAZ2]] to Caller: [[FOO2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[FOO2]] +; DUMP: Callee: 17547784407117670007 (_Z3bazv) Clones: 0 StackIds: 3 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAZ2]] to Caller: [[FOO2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[MAIN2]] +; DUMP: Callee: 6988045695824228603 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: + ; DOT: digraph CallsiteContextGraph { ; DOT: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> alloc",tooltip="N[[BAR]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -209,3 +287,22 @@ ; DOT: N[[MAIN2]] -> N[[FOO]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default ; DOT: N[[MAIN1]] -> N[[FOO]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold ; DOT: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[BAR2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> alloc",tooltip="N[[BAR2]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[BAZ2:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3bazv -\> _Z3barv",tooltip="N[[BAZ2]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[FOO2:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3foov -\> _Z3bazv",tooltip="N[[FOO2]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[MAIN2:0x[a-z0-9]+]] [shape="record",label="OrigId: 15025054523792398438\nmain -\> _Z3foov",tooltip="N[[MAIN2]] ContextIds: 2",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> alloc",tooltip="N[[BAR]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[BAZ:0x[a-z0-9]+]] [shape="record",label="OrigId: 12481870273128938184\n_Z3bazv -\> _Z3barv",tooltip="N[[BAZ]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: 2732490490862098848\n_Z3foov -\> _Z3bazv",tooltip="N[[FOO]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[MAIN1:0x[a-z0-9]+]] [shape="record",label="OrigId: 8632435727821051414\nmain -\> _Z3foov",tooltip="N[[MAIN1]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: // Edges: +; DOTCLONED: N[[BAZ2]] -> N[[BAR2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[FOO2]] -> N[[BAZ2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN2]] -> N[[FOO2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[BAZ]] -> N[[BAR]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[FOO]] -> N[[BAZ]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN1]] -> N[[FOO]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: } diff --git a/llvm/test/ThinLTO/X86/pgho-duplicate-context-ids.ll b/llvm/test/ThinLTO/X86/pgho-duplicate-context-ids.ll --- a/llvm/test/ThinLTO/X86/pgho-duplicate-context-ids.ll +++ b/llvm/test/ThinLTO/X86/pgho-duplicate-context-ids.ll @@ -62,6 +62,8 @@ ; RUN: cat %t.ccg.prestackupdate.dot | FileCheck %s --check-prefix=DOTPRE ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOTPOST +;; We should clone D once for the cold allocations via C. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'duplicate-context-ids.ll' source_filename = "duplicate-context-ids.ll" @@ -286,6 +288,64 @@ ; DUMP: Edge from Callee [[D]] to Caller: [[E]] AllocTypes: Cold ContextIds: 1 ; DUMP: CallerEdges: +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[D]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 2 StackIds: 0 +; DUMP: AllocType 1 StackIds: 1 +; DUMP: (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[D]] to Caller: [[F]] AllocTypes: NotCold ContextIds: 2 + +; DUMP: Node [[F]] +; DUMP: Callee: 4881081444663423788 (_Z1Dv) Clones: 0 StackIds: 1 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D]] to Caller: [[F]] AllocTypes: NotCold ContextIds: 2 +; DUMP: CallerEdges: + +; DUMP: Node [[D2:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 2 StackIds: 0 +; DUMP: AllocType 1 StackIds: 1 +; DUMP: (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 1 3 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[E:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 1 +; DUMP: Edge from Callee [[D2]] to Caller: [[C2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 3 +; DUMP: Edge from Callee [[D2]] to Caller: [[B:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 4 + +; DUMP: Node [[E]] +; DUMP: Callee: 4881081444663423788 (_Z1Dv) Clones: 0 StackIds: 0, 3 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[E]] AllocTypes: Cold ContextIds: 1 +; DUMP: CallerEdges: + +; DUMP: Node [[C2]] +; DUMP: Callee: 4881081444663423788 (_Z1Dv) Clones: 0 StackIds: 0 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[C2]] AllocTypes: Cold ContextIds: 3 +; DUMP: CallerEdges: + +; DUMP: Node [[B]] +; DUMP: Callee: 4881081444663423788 (_Z1Dv) Clones: 0 StackIds: 0, 2 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[B]] AllocTypes: Cold ContextIds: 4 +; DUMP: CallerEdges: + ; DOTPRE: digraph CallsiteContextGraph { ; DOTPRE: N[[D:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z1Dv -\> alloc",tooltip="N[[D]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -309,3 +369,18 @@ ; DOTPOST: N[[B]] -> N[[D]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold ; DOTPOST: N[[E]] -> N[[D]][tooltip=" ContextIds: 1",fillcolor="cyan"]; // cold ; DOTPOST: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[D2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z1Dv -\> alloc",tooltip="N[[D2]] ContextIds: 4 1 3",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[B:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z1Bv -\> _Z1Dv",tooltip="N[[B]] ContextIds: 4",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[C:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z1Cv -\> _Z1Dv",tooltip="N[[C]] ContextIds: 3",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[E:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z1Ev -\> _Z1Dv",tooltip="N[[E]] ContextIds: 1",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[D:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z1Dv -\> alloc",tooltip="N[[D]] ContextIds: 2",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[F:0x[a-z0-9]+]] [shape="record",label="OrigId: 13543580133643026784\n_Z1Fv -\> _Z1Dv",tooltip="N[[F]] ContextIds: 2",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: // Edges: +; DOTCLONED: N[[E]] -> N[[D2]][tooltip=" ContextIds: 1",fillcolor="cyan"]; // cold +; DOTCLONED: N[[C]] -> N[[D2]][tooltip=" ContextIds: 3",fillcolor="cyan"]; // cold +; DOTCLONED: N[[B]] -> N[[D2]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold +; DOTCLONED: N[[F]] -> N[[D]][tooltip=" ContextIds: 2",fillcolor="brown1"]; // default +; DOTCLONED: } diff --git a/llvm/test/ThinLTO/X86/pgho-indirectcall.ll b/llvm/test/ThinLTO/X86/pgho-indirectcall.ll --- a/llvm/test/ThinLTO/X86/pgho-indirectcall.ll +++ b/llvm/test/ThinLTO/X86/pgho-indirectcall.ll @@ -62,6 +62,9 @@ ; RUN: -o %t.out 2>&1 | FileCheck %s --check-prefix=DUMP ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOT +;; We should only create a single clone of foo, for the direct call +;; from main allocating cold memory. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'indirectcall.ll' source_filename = "indirectcall.ll" @@ -422,6 +425,119 @@ ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 6 ; DUMP: CallerEdges: +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[FOO:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 1 StackIds: 6, 8, 4 +; DUMP: AllocType 2 StackIds: 6, 8, 5 +; DUMP: AllocType 1 StackIds: 0 +; DUMP: AllocType 2 StackIds: 7, 8, 2 +; DUMP: AllocType 1 StackIds: 7, 8, 3 +; DUMP: AllocType 2 StackIds: 1 +; DUMP: (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 4 1 3 5 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[AX:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 +; DUMP: Edge from Callee [[FOO]] to Caller: [[BX:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 4 5 +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 3 + +; DUMP: Node [[AX]] +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 6 (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[AX]] AllocTypes: NotColdCold ContextIds: 1 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[AX]] to Caller: [[BAR:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 + +; DUMP: Node [[BAR]] +; DUMP: null Call +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 4 1 5 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[AX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 1 2 +; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN3:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN4:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN5:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 4 +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 5 + +; DUMP: Node [[MAIN3]] +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 4 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN3]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: + +; DUMP: Node [[MAIN4]] +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 5 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN4]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: + +; DUMP: Node [[MAIN5]] +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 2 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN5]] AllocTypes: Cold ContextIds: 4 +; DUMP: CallerEdges: + +; DUMP: Node [[MAIN6]] +; DUMP: Callee: 4095956691517954349 (_Z3barP1A) Clones: 0 StackIds: 3 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 5 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6]] AllocTypes: NotCold ContextIds: 5 +; DUMP: CallerEdges: + +; DUMP: Node [[BX]] +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 7 (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 4 5 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[BX]] AllocTypes: NotColdCold ContextIds: 4 5 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 + +; DUMP: Node [[MAIN1]] +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 0 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 3 +; DUMP: CallerEdges: + +; DUMP: Node [[FOO2:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 1 StackIds: 6, 8, 4 +; DUMP: AllocType 2 StackIds: 6, 8, 5 +; DUMP: AllocType 1 StackIds: 0 +; DUMP: AllocType 2 StackIds: 7, 8, 2 +; DUMP: AllocType 1 StackIds: 7, 8, 3 +; DUMP: AllocType 2 StackIds: 1 +; DUMP: (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 6 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 6 + +; DUMP: Node [[MAIN2]] +; DUMP: Callee: 12914368124089294956 (_Z3foov) Clones: 0 StackIds: 1 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 6 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 6 +; DUMP: CallerEdges: + ; DOT: digraph CallsiteContextGraph { ; DOT: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3foov -\> alloc",tooltip="N[[FOO]] ContextIds: 2 4 6 1 3 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -447,3 +563,30 @@ ; DOT: N[[MAIN2]] -> N[[BAR]][tooltip=" ContextIds: 5",fillcolor="brown1"]; // default ; DOT: N[[BAR]] -> N[[AX]][tooltip=" ContextIds: 1 2",fillcolor="mediumorchid1"]; // default|cold ; DOT: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[FOO2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3foov -\> alloc",tooltip="N[[FOO2]] ContextIds: 6",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[MAIN1:0x[a-z0-9]+]] [shape="record",label="OrigId: 15025054523792398438\nmain -\> _Z3foov",tooltip="N[[MAIN1]] ContextIds: 6",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3foov -\> alloc",tooltip="N[[FOO]] ContextIds: 2 4 1 3 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[MAIN6:0x[a-z0-9]+]] [shape="record",label="OrigId: 8632435727821051414\nmain -\> _Z3foov",tooltip="N[[MAIN6]] ContextIds: 3",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[BX:0x[a-z0-9]+]] [shape="record",label="OrigId: 13614864978754796978\n_ZN1B1xEv -\> _Z3foov",tooltip="N[[BX]] ContextIds: 4 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +;; Bar contains an indirect call, with multiple targets. It's call should be null. +; DOTCLONED: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: 13626499562959447861\nnull call (external)",tooltip="N[[BAR]] ContextIds: 2 4 1 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[MAIN2:0x[a-z0-9]+]] [shape="record",label="OrigId: 15737101490731057601\nmain -\> _Z3barP1A",tooltip="N[[MAIN2]] ContextIds: 5",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[MAIN3:0x[a-z0-9]+]] [shape="record",label="OrigId: 6792096022461663180\nmain -\> _Z3barP1A",tooltip="N[[MAIN3]] ContextIds: 4",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[MAIN4:0x[a-z0-9]+]] [shape="record",label="OrigId: 12699492813229484831\nmain -\> _Z3barP1A",tooltip="N[[MAIN4]] ContextIds: 2",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[MAIN5:0x[a-z0-9]+]] [shape="record",label="OrigId: 748269490701775343\nmain -\> _Z3barP1A",tooltip="N[[MAIN5]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[AX:0x[a-z0-9]+]] [shape="record",label="OrigId: 8256774051149711748\n_ZN1A1xEv -\> _Z3foov",tooltip="N[[AX]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: // Edges: +; DOTCLONED: N[[MAIN1]] -> N[[FOO2]][tooltip=" ContextIds: 6",fillcolor="cyan"]; // cold +; DOTCLONED: N[[AX]] -> N[[FOO]][tooltip=" ContextIds: 1 2",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[BX]] -> N[[FOO]][tooltip=" ContextIds: 4 5",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[MAIN6]] -> N[[FOO]][tooltip=" ContextIds: 3",fillcolor="brown1"]; // default +; DOTCLONED: N[[BAR]] -> N[[BX]][tooltip=" ContextIds: 4 5",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[MAIN5]] -> N[[BAR]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN4]] -> N[[BAR]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN3]] -> N[[BAR]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN2]] -> N[[BAR]][tooltip=" ContextIds: 5",fillcolor="brown1"]; // default +; DOTCLONED: N[[BAR]] -> N[[AX]][tooltip=" ContextIds: 1 2",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: } diff --git a/llvm/test/ThinLTO/X86/pgho-inlined.ll b/llvm/test/ThinLTO/X86/pgho-inlined.ll --- a/llvm/test/ThinLTO/X86/pgho-inlined.ll +++ b/llvm/test/ThinLTO/X86/pgho-inlined.ll @@ -49,6 +49,9 @@ ; RUN: -o %t.out 2>&1 | FileCheck %s --check-prefix=DUMP ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOT +;; We should create clones for foo and bar for the call from main to allocate +;; cold memory. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'inlined.ll' source_filename = "inlined.ll" @@ -226,6 +229,87 @@ ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1 ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[BAR:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 1 StackIds: 0, 1, 2 +; DUMP: AllocType 2 StackIds: 0, 1, 3 +; DUMP: (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 3 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[FOO:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 3 + +; DUMP: Node [[FOO]] +; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[FOO]] AllocTypes: NotCold ContextIds: 3 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 3 + +; DUMP: Node [[MAIN1]] +; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 2 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2:0x[a-z0-9]+]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1 +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 3 +; DUMP: CallerEdges: + +; DUMP: Node [[BAR2:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 1 StackIds: 0, 1, 2 +; DUMP: AllocType 2 StackIds: 0, 1, 3 +; DUMP: (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[FOO3:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 4 + +; DUMP: Node [[FOO3]] +; DUMP: Callee: 16064618363798697104 (_Z3barv) Clones: 0 StackIds: 0, 1 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[FOO3]] AllocTypes: Cold ContextIds: 4 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO3]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 4 + +; DUMP: Node [[MAIN2]] +; DUMP: Callee: 2229562716906371625 (_Z3foov) Clones: 0 StackIds: 3 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 4 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 +; DUMP: Edge from Callee [[FOO3]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 4 +; DUMP: CallerEdges: + +; DUMP: Node [[BAZ:0x[a-z0-9]+]] +; DUMP: Versions: 1 MIB: +; DUMP: AllocType 1 StackIds: 1, 2 +; DUMP: AllocType 2 StackIds: 1, 3 +; DUMP: (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 1 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 1 2 + +; DUMP: Node [[FOO2]] +; DUMP: null Call +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 1 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1 +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 + ; DOT: digraph CallsiteContextGraph { ; DOT: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3bazv -\> alloc",tooltip="N[[BAR]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -242,3 +326,23 @@ ; DOT: N[[MAIN1]] -> N[[FOO2]][tooltip=" ContextIds: 3",fillcolor="brown1"]; // default ; DOT: N[[MAIN2]] -> N[[FOO2]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold ; DOT: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[BAZ:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3bazv -\> alloc",tooltip="N[[BAZ]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[FOO2:0x[a-z0-9]+]] [shape="record",label="OrigId: 2732490490862098848\nnull call (external)",tooltip="N[[FOO2]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[MAIN2:0x[a-z0-9]+]] [shape="record",label="OrigId: 15025054523792398438\nmain -\> _Z3foov",tooltip="N[[MAIN2]] ContextIds: 2 4",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[MAIN1:0x[a-z0-9]+]] [shape="record",label="OrigId: 8632435727821051414\nmain -\> _Z3foov",tooltip="N[[MAIN1]] ContextIds: 1 3",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[BAR2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> alloc",tooltip="N[[BAR2]] ContextIds: 4",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[FOO3:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3foov -\> _Z3barv",tooltip="N[[FOO3]] ContextIds: 4",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc2\n_Z3barv -\> alloc",tooltip="N[[BAR]] ContextIds: 3",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3foov -\> _Z3barv",tooltip="N[[FOO]] ContextIds: 3",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: // Edges: +; DOTCLONED: N[[FOO2]] -> N[[BAZ]][tooltip=" ContextIds: 1 2",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[MAIN1]] -> N[[FOO2]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN2]] -> N[[FOO2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[FOO3]] -> N[[BAR2]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN2]] -> N[[FOO3]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold +; DOTCLONED: N[[FOO]] -> N[[BAR]][tooltip=" ContextIds: 3",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN1]] -> N[[FOO]][tooltip=" ContextIds: 3",fillcolor="brown1"]; // default +; DOTCLONED: } diff --git a/llvm/test/Transforms/PGHOContextDisambiguation/basic.ll b/llvm/test/Transforms/PGHOContextDisambiguation/basic.ll --- a/llvm/test/Transforms/PGHOContextDisambiguation/basic.ll +++ b/llvm/test/Transforms/PGHOContextDisambiguation/basic.ll @@ -35,6 +35,8 @@ ; RUN: %s -S 2>&1 | FileCheck %s --check-prefix=DUMP ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOT +;; We should have cloned bar, baz, and foo, for the cold memory allocation. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'basic.ll' source_filename = "basic.ll" @@ -188,6 +190,76 @@ ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 ; DUMP: CallerEdges: +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[BAR:0x[a-z0-9]+]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #6 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[BAZ:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[BAZ]] +; DUMP: %call = call noundef ptr @_Z3barv() (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[BAZ]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[FOO]] +; DUMP: %call = call noundef ptr @_Z3bazv() (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[MAIN1]] +; DUMP: %call = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: + +; DUMP: Node [[BAR2:0x[a-z0-9]+]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #6 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[BAZ2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[BAZ2]] +; DUMP: %call = call noundef ptr @_Z3barv() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[BAZ2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAZ2]] to Caller: [[FOO2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[FOO2]] +; DUMP: %call = call noundef ptr @_Z3bazv() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAZ2]] to Caller: [[FOO2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[MAIN2]] +; DUMP: %call1 = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: + ; DOT: digraph CallsiteContextGraph { ; DOT: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> _Znam",tooltip="N[[BAR]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -201,3 +273,22 @@ ; DOT: N[[MAIN2]] -> N[[FOO]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default ; DOT: N[[MAIN1]] -> N[[FOO]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold ; DOT: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[BAR2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> _Znam",tooltip="N[[BAR2]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[BAZ2:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3bazv -\> _Z3barv",tooltip="N[[BAZ2]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[FOO2:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3foov -\> _Z3bazv",tooltip="N[[FOO2]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[MAIN2:0x[a-z0-9]+]] [shape="record",label="OrigId: 15025054523792398438\nmain -\> _Z3foov",tooltip="N[[MAIN2]] ContextIds: 2",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> _Znam",tooltip="N[[BAR]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[BAZ:0x[a-z0-9]+]] [shape="record",label="OrigId: 12481870273128938184\n_Z3bazv -\> _Z3barv",tooltip="N[[BAZ]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: 2732490490862098848\n_Z3foov -\> _Z3bazv",tooltip="N[[FOO]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[MAIN1:0x[a-z0-9]+]] [shape="record",label="OrigId: 8632435727821051414\nmain -\> _Z3foov",tooltip="N[[MAIN1]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: // Edges: +; DOTCLONED: N[[BAZ2]] -> N[[BAR2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[FOO2]] -> N[[BAZ2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN2]] -> N[[FOO2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[BAZ]] -> N[[BAR]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[FOO]] -> N[[BAZ]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN1]] -> N[[FOO]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: } diff --git a/llvm/test/Transforms/PGHOContextDisambiguation/duplicate-context-ids.ll b/llvm/test/Transforms/PGHOContextDisambiguation/duplicate-context-ids.ll --- a/llvm/test/Transforms/PGHOContextDisambiguation/duplicate-context-ids.ll +++ b/llvm/test/Transforms/PGHOContextDisambiguation/duplicate-context-ids.ll @@ -57,6 +57,8 @@ ; RUN: cat %t.ccg.prestackupdate.dot | FileCheck %s --check-prefix=DOTPRE ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOTPOST +;; We should clone D once for the cold allocations via C. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'duplicate-context-ids.ll' source_filename = "duplicate-context-ids.ll" @@ -275,6 +277,58 @@ ; DUMP: Edge from Callee [[D]] to Caller: [[E]] AllocTypes: Cold ContextIds: 1 ; DUMP: CallerEdges: +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[D]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #6 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[D]] to Caller: [[F]] AllocTypes: NotCold ContextIds: 2 + +; DUMP: Node [[F]] +; DUMP: %call = call noundef ptr @_Z1Dv() (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D]] to Caller: [[F]] AllocTypes: NotCold ContextIds: 2 +; DUMP: CallerEdges: + +; DUMP: Node [[D2:0x[a-z0-9]+]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #6 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 1 3 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[E:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 1 +; DUMP: Edge from Callee [[D2]] to Caller: [[C2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 3 +; DUMP: Edge from Callee [[D2]] to Caller: [[B:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 4 + +; DUMP: Node [[E]] +; DUMP: %call.i = call noundef ptr @_Z1Dv() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[E]] AllocTypes: Cold ContextIds: 1 +; DUMP: CallerEdges: + +; DUMP: Node [[C2]] +; DUMP: %call = call noundef ptr @_Z1Dv() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[C2]] AllocTypes: Cold ContextIds: 3 +; DUMP: CallerEdges: + +; DUMP: Node [[B]] +; DUMP: %call.i = call noundef ptr @_Z1Dv() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[D2]] to Caller: [[B]] AllocTypes: Cold ContextIds: 4 +; DUMP: CallerEdges: + ; DOTPRE: digraph CallsiteContextGraph { ; DOTPRE: N[[D:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z1Dv -\> _Znam",tooltip="N[[D]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -298,3 +352,18 @@ ; DOTPOST: N[[B]] -> N[[D]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold ; DOTPOST: N[[E]] -> N[[D]][tooltip=" ContextIds: 1",fillcolor="cyan"]; // cold ; DOTPOST: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[D2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z1Dv -\> _Znam",tooltip="N[[D2]] ContextIds: 4 1 3",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[B:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z1Bv -\> _Z1Dv",tooltip="N[[B]] ContextIds: 4",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[C:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z1Cv -\> _Z1Dv",tooltip="N[[C]] ContextIds: 3",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[E:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z1Ev -\> _Z1Dv",tooltip="N[[E]] ContextIds: 1",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[D:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z1Dv -\> _Znam",tooltip="N[[D]] ContextIds: 2",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[F:0x[a-z0-9]+]] [shape="record",label="OrigId: 13543580133643026784\n_Z1Fv -\> _Z1Dv",tooltip="N[[F]] ContextIds: 2",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: // Edges: +; DOTCLONED: N[[E]] -> N[[D2]][tooltip=" ContextIds: 1",fillcolor="cyan"]; // cold +; DOTCLONED: N[[C]] -> N[[D2]][tooltip=" ContextIds: 3",fillcolor="cyan"]; // cold +; DOTCLONED: N[[B]] -> N[[D2]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold +; DOTCLONED: N[[F]] -> N[[D]][tooltip=" ContextIds: 2",fillcolor="brown1"]; // default +; DOTCLONED: } diff --git a/llvm/test/Transforms/PGHOContextDisambiguation/indirectcall.ll b/llvm/test/Transforms/PGHOContextDisambiguation/indirectcall.ll --- a/llvm/test/Transforms/PGHOContextDisambiguation/indirectcall.ll +++ b/llvm/test/Transforms/PGHOContextDisambiguation/indirectcall.ll @@ -55,6 +55,9 @@ ; RUN: %s -S 2>&1 | FileCheck %s --check-prefix=DUMP ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOT +;; We should only create a single clone of foo, for the direct call +;; from main allocating cold memory. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'indirectcall.ll' source_filename = "indirectcall.ll" @@ -408,6 +411,105 @@ ; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 6 ; DUMP: CallerEdges: +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[FOO:0x[a-z0-9]+]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #7 (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 4 1 3 5 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[AX:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 +; DUMP: Edge from Callee [[FOO]] to Caller: [[BX:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 4 5 +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 3 + +; DUMP: Node [[AX]] +; DUMP: %call = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[AX]] AllocTypes: NotColdCold ContextIds: 1 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[AX]] to Caller: [[BAR:0x[a-z0-9]+]] AllocTypes: NotColdCold ContextIds: 1 2 + +; DUMP: Node [[BAR]] +; DUMP: null Call +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 2 4 1 5 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[AX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 1 2 +; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN3:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN4:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN5:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 4 +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 5 + +; DUMP: Node [[MAIN3]] +; DUMP: %call4 = call noundef ptr @_Z3barP1A(ptr noundef %a) (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN3]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: + +; DUMP: Node [[MAIN4]] +; DUMP: %call5 = call noundef ptr @_Z3barP1A(ptr noundef %a) (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN4]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: + +; DUMP: Node [[MAIN5]] +; DUMP: %call2 = call noundef ptr @_Z3barP1A(ptr noundef %b) (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 4 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN5]] AllocTypes: Cold ContextIds: 4 +; DUMP: CallerEdges: + +; DUMP: Node [[MAIN6]] +; DUMP: %call3 = call noundef ptr @_Z3barP1A(ptr noundef %b) (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 5 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[MAIN6]] AllocTypes: NotCold ContextIds: 5 +; DUMP: CallerEdges: + +; DUMP: Node [[BX]] +; DUMP: %call = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 4 5 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[BX]] AllocTypes: NotColdCold ContextIds: 4 5 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BX]] to Caller: [[BAR]] AllocTypes: NotColdCold ContextIds: 4 5 + +; DUMP: Node [[MAIN1]] +; DUMP: %call = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 3 +; DUMP: CallerEdges: + +; DUMP: Node [[FOO2:0x[a-z0-9]+]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #7 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 6 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 6 + +; DUMP: Node [[MAIN2]] +; DUMP: %call1 = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 6 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 6 +; DUMP: CallerEdges: + ; DOT: digraph CallsiteContextGraph { ; DOT: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3foov -\> _Znam",tooltip="N[[FOO]] ContextIds: 2 4 6 1 3 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -433,3 +535,30 @@ ; DOT: N[[MAIN2]] -> N[[BAR]][tooltip=" ContextIds: 5",fillcolor="brown1"]; // default ; DOT: N[[BAR]] -> N[[AX]][tooltip=" ContextIds: 1 2",fillcolor="mediumorchid1"]; // default|cold ; DOT: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[FOO2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3foov -\> _Znam",tooltip="N[[FOO2]] ContextIds: 6",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[MAIN1:0x[a-z0-9]+]] [shape="record",label="OrigId: 15025054523792398438\nmain -\> _Z3foov",tooltip="N[[MAIN1]] ContextIds: 6",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3foov -\> _Znam",tooltip="N[[FOO]] ContextIds: 2 4 1 3 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[MAIN6:0x[a-z0-9]+]] [shape="record",label="OrigId: 8632435727821051414\nmain -\> _Z3foov",tooltip="N[[MAIN6]] ContextIds: 3",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[BX:0x[a-z0-9]+]] [shape="record",label="OrigId: 13614864978754796978\n_ZN1B1xEv -\> _Z3foov",tooltip="N[[BX]] ContextIds: 4 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +;; Bar contains an indirect call, with multiple targets. It's call should be null. +; DOTCLONED: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: 13626499562959447861\nnull call (external)",tooltip="N[[BAR]] ContextIds: 2 4 1 5",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[MAIN2:0x[a-z0-9]+]] [shape="record",label="OrigId: 15737101490731057601\nmain -\> _Z3barP1A",tooltip="N[[MAIN2]] ContextIds: 5",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[MAIN3:0x[a-z0-9]+]] [shape="record",label="OrigId: 6792096022461663180\nmain -\> _Z3barP1A",tooltip="N[[MAIN3]] ContextIds: 4",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[MAIN4:0x[a-z0-9]+]] [shape="record",label="OrigId: 12699492813229484831\nmain -\> _Z3barP1A",tooltip="N[[MAIN4]] ContextIds: 2",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[MAIN5:0x[a-z0-9]+]] [shape="record",label="OrigId: 748269490701775343\nmain -\> _Z3barP1A",tooltip="N[[MAIN5]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[AX:0x[a-z0-9]+]] [shape="record",label="OrigId: 8256774051149711748\n_ZN1A1xEv -\> _Z3foov",tooltip="N[[AX]] ContextIds: 2 1",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: // Edges: +; DOTCLONED: N[[MAIN1]] -> N[[FOO2]][tooltip=" ContextIds: 6",fillcolor="cyan"]; // cold +; DOTCLONED: N[[AX]] -> N[[FOO]][tooltip=" ContextIds: 1 2",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[BX]] -> N[[FOO]][tooltip=" ContextIds: 4 5",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[MAIN6]] -> N[[FOO]][tooltip=" ContextIds: 3",fillcolor="brown1"]; // default +; DOTCLONED: N[[BAR]] -> N[[BX]][tooltip=" ContextIds: 4 5",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[MAIN5]] -> N[[BAR]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN4]] -> N[[BAR]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN3]] -> N[[BAR]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN2]] -> N[[BAR]][tooltip=" ContextIds: 5",fillcolor="brown1"]; // default +; DOTCLONED: N[[BAR]] -> N[[AX]][tooltip=" ContextIds: 1 2",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: } diff --git a/llvm/test/Transforms/PGHOContextDisambiguation/inlined.ll b/llvm/test/Transforms/PGHOContextDisambiguation/inlined.ll --- a/llvm/test/Transforms/PGHOContextDisambiguation/inlined.ll +++ b/llvm/test/Transforms/PGHOContextDisambiguation/inlined.ll @@ -44,6 +44,9 @@ ; RUN: %s -S 2>&1 | FileCheck %s --check-prefix=DUMP ; RUN: cat %t.ccg.postbuild.dot | FileCheck %s --check-prefix=DOT +;; We should create clones for foo and bar for the call from main to allocate +;; cold memory. +; RUN: cat %t.ccg.cloned.dot | FileCheck %s --check-prefix=DOTCLONED ; ModuleID = 'inlined.ll' source_filename = "inlined.ll" @@ -215,6 +218,78 @@ ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 3 ; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 4 +; DUMP: CCG after cloning: +; DUMP: Callsite Context Graph: +; DUMP: Node [[BAR:0x[a-z0-9]+]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #7 (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[FOO:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[FOO]] +; DUMP: %call.i = call noundef ptr @_Z3barv() (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR]] to Caller: [[FOO]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1:0x[a-z0-9]+]] AllocTypes: NotCold ContextIds: 1 + +; DUMP: Node [[MAIN1]] +; DUMP: %call = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: NotCold +; DUMP: ContextIds: 1 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2:0x[a-z0-9]+]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 3 +; DUMP: Edge from Callee [[FOO]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 1 +; DUMP: CallerEdges: + +; DUMP: Node [[BAR2:0x[a-z0-9]+]] +; DUMP: %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #7 (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[FOO3:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[FOO3]] +; DUMP: %call.i = call noundef ptr @_Z3barv() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAR2]] to Caller: [[FOO3]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO3]] to Caller: [[MAIN2:0x[a-z0-9]+]] AllocTypes: Cold ContextIds: 2 + +; DUMP: Node [[MAIN2]] +; DUMP: %call1 = call noundef ptr @_Z3foov() (clone 0) +; DUMP: AllocTypes: Cold +; DUMP: ContextIds: 2 4 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 4 +; DUMP: Edge from Callee [[FOO3]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 2 +; DUMP: CallerEdges: + +; DUMP: Node [[BAZ:0x[a-z0-9]+]] +; DUMP: %call.i = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #7 (clone 0) +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 4 3 +; DUMP: CalleeEdges: +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 4 3 + +; DUMP: Node [[FOO2]] +; DUMP: null Call +; DUMP: AllocTypes: NotColdCold +; DUMP: ContextIds: 4 3 +; DUMP: CalleeEdges: +; DUMP: Edge from Callee [[BAZ]] to Caller: [[FOO2]] AllocTypes: NotColdCold ContextIds: 4 3 +; DUMP: CallerEdges: +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN1]] AllocTypes: NotCold ContextIds: 3 +; DUMP: Edge from Callee [[FOO2]] to Caller: [[MAIN2]] AllocTypes: Cold ContextIds: 4 + ; DOT: digraph CallsiteContextGraph { ; DOT: N[[BAZ:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc2\n_Z3bazv -\> _Znam",tooltip="N[[BAZ]] ContextIds: 4 3",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold @@ -231,3 +306,23 @@ ; DOT: N[[MAIN1]] -> N[[FOO]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default ; DOT: N[[MAIN2]] -> N[[FOO]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold ; DOT: } + + +; DOTCLONED: digraph CallsiteContextGraph { +; DOTCLONED: N[[BAZ:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc2\n_Z3bazv -\> _Znam",tooltip="N[[BAZ]] ContextIds: 4 3",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[FOO2:0x[a-z0-9]+]] [shape="record",label="OrigId: 2732490490862098848\nnull call (external)",tooltip="N[[FOO2]] ContextIds: 4 3",fillcolor="mediumorchid1",style="filled",style="filled"]; // callsite, default|cold +; DOTCLONED: N[[MAIN2:0x[a-z0-9]+]] [shape="record",label="OrigId: 15025054523792398438\nmain -\> _Z3foov",tooltip="N[[MAIN2]] ContextIds: 2 4",fillcolor="cyan",style="filled",style="filled"]; // callsite, cold +; DOTCLONED: N[[MAIN1:0x[a-z0-9]+]] [shape="record",label="OrigId: 8632435727821051414\nmain -\> _Z3foov",tooltip="N[[MAIN1]] ContextIds: 1 3",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[BAR2:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> _Znam",tooltip="N[[BAR2]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[FOO3:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3foov -\> _Z3barv",tooltip="N[[FOO3]] ContextIds: 2",fillcolor="cyan",style="filled",color="blue",style="filled, bold, dashed"]; // callsite, cold +; DOTCLONED: N[[BAR:0x[a-z0-9]+]] [shape="record",label="OrigId: Alloc0\n_Z3barv -\> _Znam",tooltip="N[[BAR]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: N[[FOO:0x[a-z0-9]+]] [shape="record",label="OrigId: 0\n_Z3foov -\> _Z3barv",tooltip="N[[FOO]] ContextIds: 1",fillcolor="brown1",style="filled",style="filled"]; // callsite, default +; DOTCLONED: // Edges: +; DOTCLONED: N[[FOO2]] -> N[[BAZ]][tooltip=" ContextIds: 4 3",fillcolor="mediumorchid1"]; // default|cold +; DOTCLONED: N[[MAIN1]] -> N[[FOO2]][tooltip=" ContextIds: 3",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN2]] -> N[[FOO2]][tooltip=" ContextIds: 4",fillcolor="cyan"]; // cold +; DOTCLONED: N[[FOO3]] -> N[[BAR2]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[MAIN2]] -> N[[FOO3]][tooltip=" ContextIds: 2",fillcolor="cyan"]; // cold +; DOTCLONED: N[[FOO]] -> N[[BAR]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: N[[MAIN1]] -> N[[FOO]][tooltip=" ContextIds: 1",fillcolor="brown1"]; // default +; DOTCLONED: }