diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h --- a/lld/MachO/Config.h +++ b/lld/MachO/Config.h @@ -14,6 +14,7 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/Support/CachePruning.h" #include "llvm/Support/GlobPattern.h" #include "llvm/Support/VersionTuple.h" #include "llvm/TextAPI/Architecture.h" @@ -139,6 +140,8 @@ llvm::StringRef thinLTOJobs; llvm::StringRef umbrella; uint32_t ltoo = 2; + llvm::CachePruningPolicy thinLTOCachePolicy; + llvm::StringRef thinLTOCacheDir; bool deadStripDylibs = false; bool demangle = false; bool deadStrip = false; diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -195,6 +195,35 @@ {"/Library/Frameworks", "/System/Library/Frameworks"}); } +static llvm::CachePruningPolicy getLTOCachePolicy(InputArgList &args) { + SmallString<128> ltoPolicy; + auto add = [<oPolicy](Twine val) { + if (!ltoPolicy.empty()) + ltoPolicy += ":"; + val.toVector(ltoPolicy); + }; + for (const Arg *arg : + args.filtered(OPT_thinlto_cache_policy, OPT_prune_interval_lto, + OPT_prune_after_lto, OPT_max_relative_cache_size_lto)) { + switch (arg->getOption().getID()) { + case OPT_thinlto_cache_policy: add(arg->getValue()); break; + case OPT_prune_interval_lto: + if (!strcmp("-1", arg->getValue())) + add("prune_interval=87600h"); // 10 years + else + add(Twine("prune_interval=") + arg->getValue() + "s"); + break; + case OPT_prune_after_lto: + add(Twine("prune_after=") + arg->getValue() + "s"); + break; + case OPT_max_relative_cache_size_lto: + add(Twine("cache_size=") + arg->getValue() + "%"); + break; + } + } + return CHECK(parseCachePruningPolicy(ltoPolicy), "invalid LTO cache policy"); +} + namespace { struct ArchiveMember { MemoryBufferRef mbref; @@ -1171,6 +1200,8 @@ config->ltoo = args::getInteger(args, OPT_lto_O, 2); if (config->ltoo > 3) error("--lto-O: invalid optimization level: " + Twine(config->ltoo)); + config->thinLTOCacheDir = args.getLastArgValue(OPT_cache_path_lto); + config->thinLTOCachePolicy = getLTOCachePolicy(args); config->runtimePaths = args::getStrings(args, OPT_rpath); config->allLoad = args.hasArg(OPT_all_load); config->archMultiple = args.hasArg(OPT_arch_multiple); diff --git a/lld/MachO/LTO.h b/lld/MachO/LTO.h --- a/lld/MachO/LTO.h +++ b/lld/MachO/LTO.h @@ -10,6 +10,7 @@ #define LLD_MACHO_LTO_H #include "llvm/ADT/SmallString.h" +#include "llvm/Support/MemoryBuffer.h" #include #include @@ -35,6 +36,7 @@ private: std::unique_ptr ltoObj; std::vector> buf; + std::vector> files; }; } // namespace macho diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp --- a/lld/MachO/LTO.cpp +++ b/lld/MachO/LTO.cpp @@ -17,6 +17,8 @@ #include "lld/Common/ErrorHandler.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" +#include "llvm/LTO/Caching.h" +#include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -97,11 +99,28 @@ std::vector BitcodeCompiler::compile() { unsigned maxTasks = ltoObj->getMaxTasks(); buf.resize(maxTasks); - - checkError(ltoObj->run([&](size_t task) { - return std::make_unique( - std::make_unique(buf[task])); - })); + files.resize(maxTasks); + + // The -cache_path_lto option specifies the path to a directory in which + // to cache native object files for ThinLTO incremental builds. If a path was + // specified, configure LTO to use it as the cache directory. + lto::NativeObjectCache cache; + if (!config->thinLTOCacheDir.empty()) + cache = check( + lto::localCache(config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr mb) { + files[task] = std::move(mb); + })); + + checkError(ltoObj->run( + [&](size_t task) { + return std::make_unique( + std::make_unique(buf[task])); + }, + cache)); + + if (!config->thinLTOCacheDir.empty()) + pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy); if (config->saveTemps) { if (!buf[0].empty()) @@ -130,6 +149,8 @@ ret.push_back(make( MemoryBufferRef(buf[i], saver.save(filePath.str())), modTime, "")); } - + for (std::unique_ptr &file : files) + if (file) + ret.push_back(make(*file, 0, "")); return ret; } diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td --- a/lld/MachO/Options.td +++ b/lld/MachO/Options.td @@ -68,6 +68,9 @@ HelpText<"Set optimization level for LTO (default: 2)">, MetaVarName<"">, Group; +def thinlto_cache_policy: Joined<["--"], "thinlto-cache-policy=">, + HelpText<"Pruning policy for the ThinLTO cache">, + Group; // This is a complete Options.td compiled from Apple's ld(1) manpage // dated 2018-03-07 and cross checked with ld64 source code in repo @@ -915,22 +918,18 @@ def cache_path_lto : Separate<["-"], "cache_path_lto">, MetaVarName<"">, HelpText<"Use as a directory for the incremental LTO cache">, - Flags<[HelpHidden]>, Group; def prune_interval_lto : Separate<["-"], "prune_interval_lto">, MetaVarName<"">, HelpText<"Prune the incremental LTO cache after (-1 disables pruning)">, - Flags<[HelpHidden]>, Group; def prune_after_lto : Separate<["-"], "prune_after_lto">, MetaVarName<"">, HelpText<"Remove LTO cache entries after ">, - Flags<[HelpHidden]>, Group; def max_relative_cache_size_lto : Separate<["-"], "max_relative_cache_size_lto">, MetaVarName<"">, HelpText<"Limit the incremental LTO cache growth to of free disk, space">, - Flags<[HelpHidden]>, Group; def page_align_data_atoms : Flag<["-"], "page_align_data_atoms">, HelpText<"Distribute global variables on separate pages so page used/dirty status can guide creation of an order file to cluster commonly used/dirty globals">, diff --git a/lld/test/MachO/lto-cache.ll b/lld/test/MachO/lto-cache.ll new file mode 100644 --- /dev/null +++ b/lld/test/MachO/lto-cache.ll @@ -0,0 +1,94 @@ +; REQUIRES: x86 +; NetBSD: noatime mounts currently inhibit 'touch' from updating atime +; UNSUPPORTED: system-netbsd + +; RUN: rm -rf %t; split-file %s %t +; RUN: opt -module-hash -module-summary %t/foo.ll -o %t/foo.o +; RUN: opt -module-hash -module-summary %t/bar.ll -o %t/bar.o + +; RUN: rm -Rf %t/cache && mkdir %t/cache +;; Create two files that would be removed by cache pruning due to age. +;; We should only remove files matching the pattern "llvmcache-*". +; RUN: touch -t 197001011200 %t/cache/llvmcache-baz %t/cache/baz +; RUN: %lld -cache_path_lto %t/cache \ +; RUN: --thinlto-cache-policy=prune_after=1h:prune_interval=0s \ +; RUN: -o %t/test %t/foo.o %t/bar.o + +;; Two cached objects, plus a timestamp file and "baz", minus the file we removed. +; RUN: ls %t/cache | count 4 + +;; Same thing, but with `-prune_after_lto` +; RUN: touch -t 197001011200 %t/cache/llvmcache-baz +; RUN: %lld -cache_path_lto %t/cache -prune_after_lto 3600 -prune_interval_lto 0 \ +; RUN: -o %t/test %t/foo.o %t/bar.o +; RUN: ls %t/cache | count 4 + +;; Create a file of size 64KB. +; RUN: %python -c "print(' ' * 65536)" > %t/cache/llvmcache-baz + +;; This should leave the file in place. +; RUN: %lld -cache_path_lto %t/cache \ +; RUN: --thinlto-cache-policy=cache_size_bytes=128k:prune_interval=0s \ +; RUN: -o %t/test %t/foo.o %t/bar.o +; RUN: ls %t/cache | count 5 + +;; Increase the age of llvmcache-baz, which will give it the oldest time stamp +;; so that it is processed and removed first. +; RUN: %python -c 'import os,sys,time; t=time.time()-120; os.utime(sys.argv[1],(t,t))' \ +; RUN: %t/cache/llvmcache-baz + +;; This should remove it. +; RUN: %lld -cache_path_lto %t/cache \ +; RUN: --thinlto-cache-policy=cache_size_bytes=32k:prune_interval=0s \ +; RUN: -o %t/test %t/foo.o %t/bar.o +; RUN: ls %t/cache | count 4 + +;; Delete everything except for the timestamp, "baz" and one cache file. +; RUN: %lld -cache_path_lto %t/cache \ +; RUN: --thinlto-cache-policy=prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s \ +; RUN: -o %t/test %t/foo.o %t/bar.o +; RUN: ls %t/cache | count 3 + +;; Check that we remove the least recently used file first. +; RUN: rm -fr %t/cache +; RUN: mkdir %t/cache +; RUN: echo xyz > %t/cache/llvmcache-old +; RUN: touch -t 198002011200 %t/cache/llvmcache-old +; RUN: echo xyz > %t/cache/llvmcache-newer +; RUN: touch -t 198002021200 %t/cache/llvmcache-newer +; RUN: %lld -cache_path_lto %t/cache \ +; RUN: --thinlto-cache-policy=prune_after=0s:cache_size=0%:cache_size_files=3:prune_interval=0s \ +; RUN: -o %t/test %t/foo.o %t/bar.o +; RUN: ls %t/cache | FileCheck %s + +;; Check that `-max_relative_cache_size_lto` is a legal argument. +; RUN: %lld -cache_path_lto %t/cache -max_relative_cache_size_lto 10 \ +; RUN: -o %t/test %t/foo.o %t/bar.o + +; CHECK-NOT: llvmcache-old +; CHECK: llvmcache-newer +; CHECK-NOT: llvmcache-old + +;--- foo.ll + +target triple = "x86_64-apple-macosx10.15.0" +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @globalfunc() #0 { +entry: + ret void +} + + +;--- bar.ll + +target triple = "x86_64-apple-macosx10.15.0" +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define i32 @main() { +entry: + call void (...) @globalfunc() + ret i32 0 +} + +declare void @globalfunc(...)