diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -29,6 +29,11 @@ namespace llvm { +struct FunctionID { + StringRef DIFileName; + SmallVector FunctionNames; +}; + // The cluster information for a machine basic block. struct BBClusterInfo { // Unique ID for this basic block. @@ -39,9 +44,12 @@ unsigned PositionInCluster; }; -using ProgramBBClusterInfoMapTy = StringMap>; +struct FunctionClusterInfo { + FunctionID FuncID; + SmallVector Clusters; +}; -class BasicBlockSectionsProfileReader : public ImmutablePass { +class BasicBlockSectionsProfileReader : public ModulePass { public: static char ID; @@ -62,7 +70,7 @@ // Returns true if basic block sections profile exist for function \p // FuncName. - bool isFunctionHot(StringRef FuncName) const; + bool isFunctionHot(StringRef ModuleName, StringRef FuncName) const; // Returns a pair with first element representing whether basic block sections // profile exist for the function \p FuncName, and the second element @@ -71,17 +79,17 @@ // means unique basic block sections are desired for all basic blocks of the // function. std::pair> - getBBClusterInfoForFunction(StringRef FuncName) const; + getBBClusterInfoForFunction(FunctionID FuncID) const; /// Read profiles of basic blocks if available here. void initializePass() override; -private: - StringRef getAliasName(StringRef FuncName) const { - auto R = FuncAliasMap.find(FuncName); - return R == FuncAliasMap.end() ? FuncName : R->second; + bool runOnModule(Module &M) override { + for( + return true; } +private: // This contains the basic-block-sections profile. const MemoryBuffer *MBuf = nullptr; @@ -91,11 +99,11 @@ // some of) its basic blocks. The cluster information for every basic block // includes its cluster ID along with the position of the basic block in that // cluster. - ProgramBBClusterInfoMapTy ProgramBBClusterInfo; + std::vector ProgramBBClusterInfo; // Some functions have alias names. We use this map to find the main alias // name for which we have mapping in ProgramBBClusterInfo. - StringMap FuncAliasMap; + DenseMap FunctionProfileIndexMap. }; // Creates a BasicBlockSectionsProfileReader pass to parse the basic block diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -77,6 +77,7 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/InitializePasses.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/Target/TargetMachine.h" #include @@ -170,8 +171,20 @@ DenseMap &V) { // Find the assoicated cluster information. + std::string ModuleName = ""; + auto *Subprogram = MF.getFunction().getSubprogram(); + if (Subprogram) { + llvm::DIScope *Scope = Subprogram->getScope(); + if (Scope) { + StringRef Dir = Scope->getDirectory(); + if (Dir.empty() || Dir == ".") + ModuleName = Scope->getFilename().str(); + else + ModuleName = Dir.str() + "/" + Scope->getFilename().str(); + } + } std::pair> P = - BBSectionsProfileReader->getBBClusterInfoForFunction(MF.getName()); + BBSectionsProfileReader->getBBClusterInfoForFunction(ModuleName, MF.getName()); if (!P.first) return false; diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -28,18 +28,24 @@ "Reads and parses a basic block sections profile.", false, false) -bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const { - return getBBClusterInfoForFunction(FuncName).first; +bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef ModuleName, + StringRef FuncName) const { + return getBBClusterInfoForFunction(ModuleName, FuncName).first; } std::pair> BasicBlockSectionsProfileReader::getBBClusterInfoForFunction( - StringRef FuncName) const { + StringRef ModuleName, StringRef FuncName) const { std::pair> cluster_info(false, {}); - auto R = ProgramBBClusterInfo.find(getAliasName(FuncName)); - if (R != ProgramBBClusterInfo.end()) { - cluster_info.second = R->second; - cluster_info.first = true; + for (auto UsedModuleName : {ModuleName, StringRef("")}) { + auto R = ProgramBBClusterInfo.find(UsedModuleName); + if (R == ProgramBBClusterInfo.end()) + continue; + auto S = R->second.find(getAliasName(UsedModuleName, FuncName)); + if (S != R->second.end()) { + cluster_info.second = S->second; + cluster_info.first = true; + } } return cluster_info; } @@ -57,9 +63,10 @@ // !foo // !!1 2 // !!4 -static Error getBBClusterInfo(const MemoryBuffer *MBuf, - ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, - StringMap &FuncAliasMap) { +static Error +getBBClusterInfo(const MemoryBuffer *MBuf, + std::vector &ProgramBBClusterInfo, + StringMap> &ProfileIndexMap) { assert(MBuf); line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'); @@ -70,7 +77,7 @@ inconvertibleErrorCode()); }; - auto FI = ProgramBBClusterInfo.end(); + FunctionClusterInfo CurrentClusterInfo; // Current cluster ID corresponding to this function. unsigned CurrentCluster = 0; @@ -90,7 +97,7 @@ break; // Check for second "!" which indicates a cluster of basic blocks. if (S.consume_front("!")) { - if (FI == ProgramBBClusterInfo.end()) + if (CurrentClusterInfo.FunctionName.empty()) return invalidProfileError( "Cluster list does not follow a function name specifier."); SmallVector BBIDs; @@ -108,7 +115,7 @@ if (BBID == 0 && CurrentPosition) return invalidProfileError("Entry BB (0) does not begin a cluster."); - FI->second.emplace_back( + CurrentClusterInfo.Clusters.emplace_back( BBClusterInfo{((unsigned)BBID), CurrentCluster, CurrentPosition++}); } CurrentCluster++; @@ -116,14 +123,21 @@ // Function aliases are separated using '/'. We use the first function // name for the cluster info mapping and delegate all other aliases to // this one. + SmallVector FunctionsAndFilename; + S.split(FunctionsAndFilename, ' '); + std::optional Filename = FunctionsAndFilename.size() == 1 + ? std::nullopt + : FunctionAndModule[1]; SmallVector Aliases; - S.split(Aliases, '/'); - for (size_t i = 1; i < Aliases.size(); ++i) - FuncAliasMap.try_emplace(Aliases[i], Aliases.front()); + FunctionsAndFilename[0].split(Aliases, '/'); + for (StringRef Alias : Aliases) { + ProfileIndexMap[DIFilename][Alias] = ProgramClusterInfo.size(); + } + ProgramBBClusterInfo.push_back(CurrentClusterInfo); // Prepare for parsing clusters of this function name. // Start a new cluster map for this function name. - FI = ProgramBBClusterInfo.try_emplace(Aliases.front()).first; + FI = ProgramBBClusterInfo[ModuleName].try_emplace(Aliases.front()).first; CurrentCluster = 0; FuncBBIDs.clear(); } diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -515,7 +515,7 @@ // Use the basic-block-sections profile to promote hot functions to .text.hot // if requested. if (BBSectionsGuidedSectionPrefix && BBSectionsProfileReader && - BBSectionsProfileReader->isFunctionHot(F.getName())) { + BBSectionsProfileReader->isFunctionHot(F.getParent()->getSourceFileName(), F.getName())) { F.setSectionPrefix("hot"); } else if (ProfileGuidedSectionPrefix) { // The hot attribute overwrites profile count based hotness while profile diff --git a/llvm/test/CodeGen/X86/basic-block-sections-module1.ll b/llvm/test/CodeGen/X86/basic-block-sections-module1.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/basic-block-sections-module1.ll @@ -0,0 +1,44 @@ +;; Specify the correct module name. +; RUN: echo '!test /path/to/dir/test_filename' > %t1 +; RUN: echo '!!0' >> %t1 +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t1 | FileCheck %s -check-prefix=RIGHT-MODULE +;; Specify no module name. +; RUN: echo '!test' > %t2 +; RUN: echo '!!0' >> %t2 +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t2 | FileCheck %s -check-prefix=NO-MODULE +;; Specify incorrect module names. +; RUN: echo '!test test_filename' > %t3 +; RUN: echo '!!0' >> %t3 +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t3 | FileCheck %s -check-prefix=WRONG-MODULE +; RUN: echo '!test ./path/to/dir/test_filename' > %t4 +; RUN: echo '!!0' >> %t4 +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t4 | FileCheck %s -check-prefix=WRONG-MODULE + +define dso_local i32 @test(i32 noundef %0) #0 !dbg !10 { + %2 = alloca i32, align 4 + %3 = alloca i32, align 4 + store i32 %0, i32* %3, align 4 + %4 = load i32, i32* %3, align 4 + %5 = icmp slt i32 %4, 0 + br i1 %5, label %6, label %7 +6: ; preds = %1 + store i32 -1, i32* %2, align 4 + ret i32 0 +7: + ret i32 1 +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1) +!1 = !DIFile(filename: "test_filename", directory: "/path/to/dir") +!2 = !{i32 7, !"Dwarf Version", i32 5} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!10 = distinct !DISubprogram(name: "test", scope: !1, unit: !0) + +;; Check that the split section is created when using the correct module name, or no module name. +; +; RIGHT-MODULE: .section .text.split.test,"ax",@progbits +; NO-MODULE: .section .text.split.test,"ax",@progbits +; WRONG-MODULE-NOT: .section .text.split.test,"ax",@progbits diff --git a/llvm/test/CodeGen/X86/basic-block-sections-module2.ll b/llvm/test/CodeGen/X86/basic-block-sections-module2.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/basic-block-sections-module2.ll @@ -0,0 +1,37 @@ +;; Specify the correct module name. +; RUN: echo '!dummy' > %t1 +; RUN: echo '!!0 1' >> %1 +; RUN: echo '!test test_filename' >> %t1 +; RUN: echo '!!0' >> %t1 +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t1 | FileCheck %s -check-prefix=RIGHT-MODULE +; RUN: echo '!test ./test_filename' > %t2 +; RUN: echo '!!0' >> %t2 +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t2 | FileCheck %s -check-prefix=WRONG-MODULE + +define dso_local i32 @test(i32 noundef %0) #0 !dbg !10 { + %2 = alloca i32, align 4 + %3 = alloca i32, align 4 + store i32 %0, i32* %3, align 4 + %4 = load i32, i32* %3, align 4 + %5 = icmp slt i32 %4, 0 + br i1 %5, label %6, label %7 +6: ; preds = %1 + store i32 -1, i32* %2, align 4 + ret i32 0 +7: + ret i32 1 +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1) +!1 = !DIFile(filename: "test_filename", directory: ".") +!2 = !{i32 7, !"Dwarf Version", i32 5} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!10 = distinct !DISubprogram(name: "test", scope: !1, unit: !0) + +;; Check that the split section is created when using the correct module name, or no module name. +; +; RIGHT-MODULE: .section .text.split.test,"ax",@progbits +; WRONG-MODULE-NOT: .section .text.split.test,"ax",@progbits