diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -600,6 +600,9 @@ /// Indicates if the binary is stripped bool IsStripped{false}; + /// Indicates if the binary contains split functions. + bool HasSplitFunctions{false}; + /// Is the binary always loaded at a fixed address. Shared objects and /// position-independent executables (PIEs) are examples of binaries that /// will have HasFixedLoadAddress set to false. diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -52,6 +52,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/LEB128.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Regex.h" #include "llvm/Support/Timer.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/raw_ostream.h" @@ -1456,18 +1457,14 @@ std::optional FragName = Function.hasRestoredNameRegex(".*\\.cold(\\.[0-9]+)?"); if (FragName) { + BC->HasSplitFunctions = true; static bool PrintedWarning = false; if (!PrintedWarning) { PrintedWarning = true; errs() << "BOLT-WARNING: split function detected on input : " << *FragName; if (BC->HasRelocations) - errs() << ". The support is limited in relocation mode"; - if (opts::Lite) { - opts::Lite = false; - errs() << "\nBOLT-WARNING: disabling lite mode (-lite) when split " - << "functions are present\n"; - } + errs() << ". The support is limited in relocation mode\n"; } Function.IsFragment = true; } @@ -2764,8 +2761,18 @@ } uint64_t NumFunctionsToProcess = 0; - auto shouldProcess = [&](const BinaryFunction &Function) { + auto mustSkip = [&](const BinaryFunction &Function) { if (opts::MaxFunctions && NumFunctionsToProcess > opts::MaxFunctions) + return true; + for (std::string &Name : opts::SkipFunctionNames) + if (Function.hasNameRegex(Name)) + return true; + + return false; + }; + + auto shouldProcess = [&](const BinaryFunction &Function) { + if (mustSkip(Function)) return false; // If the list is not empty, only process functions from the list. @@ -2783,10 +2790,6 @@ return Match.has_value(); } - for (std::string &Name : opts::SkipFunctionNames) - if (Function.hasNameRegex(Name)) - return false; - if (opts::Lite) { // Forcibly include functions specified in the -function-order file. if (opts::ReorderFunctions == ReorderFunctions::RT_USER) { @@ -2807,6 +2810,9 @@ return true; }; + StringSet<> ParentFragmentsToProcess; + std::vector FuncsToIgnore; + for (auto &BFI : BC->getBinaryFunctions()) { BinaryFunction &Function = BFI.second; @@ -2820,11 +2826,41 @@ if (!shouldProcess(Function)) { LLVM_DEBUG(dbgs() << "BOLT-INFO: skipping processing of function " << Function << " per user request\n"); - Function.setIgnored(); + if (BC->HasSplitFunctions) + FuncsToIgnore.push_back(&Function); + else + Function.setIgnored(); } else { ++NumFunctionsToProcess; if (opts::MaxFunctions && NumFunctionsToProcess == opts::MaxFunctions) outs() << "BOLT-INFO: processing ending on " << Function << '\n'; + if (!BC->HasSplitFunctions) + continue; + Function.forEachName([&](StringRef Name) { + size_t ColdSuffixPos = Name.find(".cold"); + ParentFragmentsToProcess.insert(Name.substr(0, ColdSuffixPos)); + return false; + }); + } + } + if (!BC->HasSplitFunctions) + return; + for (BinaryFunction *Function : FuncsToIgnore) { + if (mustSkip(*Function)) { + Function->setIgnored(); + continue; + } + Optional Match = Function->forEachName([&](StringRef Name) { + StringRef Parent = Name.substr(0, Name.find(".cold")); + return ParentFragmentsToProcess.contains(Parent); + }); + if (!Match.has_value()) { + Function->setIgnored(); + continue; + } + if (opts::Verbosity >= 1) { + outs() << "BOLT-INFO: processing " << *Function + << " as a sibling of non-ignored function\n"; } } } diff --git a/bolt/test/X86/fragment-lite-reverse.s b/bolt/test/X86/fragment-lite-reverse.s new file mode 100644 --- /dev/null +++ b/bolt/test/X86/fragment-lite-reverse.s @@ -0,0 +1,58 @@ +# Check BOLT in lite mode processes both the main function and cold fragment, +# even if the main function has no profile. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: link_fdata %s %t.o %t.fdata +# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.out --lite=1 --data %t.fdata -v=1 2>&1 \ +# RUN: | FileCheck %s + +# CHECK: BOLT-INFO: processing main as a sibling of non-ignored function +# CHECK: BOLT-INFO: processing foo as a sibling of non-ignored function + + .globl main + .type main, %function +main: + .cfi_startproc + cmpl $0x0, %eax + je main.cold.1 + retq + .cfi_endproc +.size main, .-main + + .globl foo + .type foo, %function +foo: + .cfi_startproc + cmpl $0x0, %eax + je foo.cold.1 + retq + .cfi_endproc +.size foo, .-foo + + .section .text.cold + .globl main.cold.1 + .type main.cold.1, %function +main.cold.1: +# FDATA: 0 [unknown] 0 1 main.cold.1 0 1 0 + .cfi_startproc + pushq %rbp + movq %rsp, %rbp + movl $0x0, %eax + popq %rbp + retq + .cfi_endproc +.size main.cold.1, .-main.cold.1 + + .globl foo.cold.1 + .type foo.cold.1, %function +foo.cold.1: +# FDATA: 0 [unknown] 0 1 foo.cold.1 0 1 0 + .cfi_startproc + pushq %rbp + movq %rsp, %rbp + movl $0x0, %eax + popq %rbp + retq + .cfi_endproc +.size foo.cold.1, .-foo.cold.1 diff --git a/bolt/test/X86/fragment-lite.s b/bolt/test/X86/fragment-lite.s new file mode 100644 --- /dev/null +++ b/bolt/test/X86/fragment-lite.s @@ -0,0 +1,58 @@ +# Check BOLT in lite mode processes both the main function and cold fragment, +# even if the fragment has no profile. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: link_fdata %s %t.o %t.fdata +# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.out --lite=1 --data %t.fdata -v=1 2>&1 \ +# RUN: | FileCheck %s + +# CHECK: BOLT-INFO: processing main.cold.1 as a sibling of non-ignored function +# CHECK: BOLT-INFO: processing foo.cold.1/1 as a sibling of non-ignored function + + .globl main + .type main, %function +main: + .cfi_startproc +# FDATA: 0 [unknown] 0 1 main 0 1 0 + cmpl $0x0, %eax + je main.cold.1 + retq + .cfi_endproc +.size main, .-main + + .globl foo + .type foo, %function +foo: + .cfi_startproc +# FDATA: 1 main 0 1 foo 0 1 0 + cmpl $0x0, %eax + je foo.cold.1 + retq + .cfi_endproc +.size foo, .-foo + + .section .text.cold + .globl main.cold.1 + .type main.cold.1, %function +main.cold.1: + .cfi_startproc + pushq %rbp + movq %rsp, %rbp + movl $0x0, %eax + popq %rbp + retq + .cfi_endproc +.size main.cold.1, .-main.cold.1 + + .local foo.cold.1 + .type foo.cold.1, %function +foo.cold.1: + .cfi_startproc + pushq %rbp + movq %rsp, %rbp + movl $0x0, %eax + popq %rbp + retq + .cfi_endproc +.size foo.cold.1, .-foo.cold.1