Index: lld/MachO/Config.h =================================================================== --- lld/MachO/Config.h +++ lld/MachO/Config.h @@ -14,6 +14,7 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/Support/CachePruning.h" #include "llvm/Support/GlobPattern.h" #include "llvm/Support/VersionTuple.h" #include "llvm/TextAPI/Architecture.h" @@ -139,6 +140,8 @@ llvm::StringRef thinLTOJobs; llvm::StringRef umbrella; uint32_t ltoo = 2; + llvm::CachePruningPolicy thinLTOCachePolicy; + llvm::StringRef thinLTOCacheDir; bool deadStripDylibs = false; bool demangle = false; bool deadStrip = false; Index: lld/MachO/Driver.cpp =================================================================== --- lld/MachO/Driver.cpp +++ lld/MachO/Driver.cpp @@ -1173,6 +1173,47 @@ config->ltoo = args::getInteger(args, OPT_lto_O, 2); if (config->ltoo > 3) error("--lto-O: invalid optimization level: " + Twine(config->ltoo)); + config->thinLTOCacheDir = args.getLastArgValue(OPT_cache_path_lto); + bool cachePolicyStringProvided = false; + if (const Arg *arg = args.getLastArg(OPT_thinlto_cache_policy)) { + config->thinLTOCachePolicy = + CHECK(parseCachePruningPolicy(arg->getValue()), + "--thinlto-cache-policy: invalid cache policy"); + cachePolicyStringProvided = true; + } + if (const Arg *arg = args.getLastArg(OPT_prune_interval_lto)) { + if (cachePolicyStringProvided) { + warn("-prune_interval_lto: ignored when --thinlto-cache-policy is " + "provided"); + } else { + int value = 0; + if (!llvm::to_integer(arg->getValue(), value)) + error("-prune_interval_lto: invalid value: " + Twine(arg->getValue())); + config->thinLTOCachePolicy.Interval = std::chrono::seconds(value); + } + } + if (const Arg *arg = args.getLastArg(OPT_prune_after_lto)) { + if (cachePolicyStringProvided) { + warn("-prune_after_lto: ignored when --thinlto-cache-policy is provided"); + } else { + int value = 0; + if (!llvm::to_integer(arg->getValue(), value)) + error("-prune_after_lto: invalid value: " + Twine(arg->getValue())); + config->thinLTOCachePolicy.Expiration = std::chrono::seconds(value); + } + } + if (const Arg *arg = args.getLastArg(OPT_max_relative_cache_size_lto)) { + if (cachePolicyStringProvided) { + warn("-max_relative_cache_size_lto: ignored when --thinlto-cache-policy " + "is provided"); + } else { + int value = 0; + if (!llvm::to_integer(arg->getValue(), value)) + error("-max_relative_cache_size_lto: invalid value: " + + Twine(arg->getValue())); + config->thinLTOCachePolicy.MaxSizePercentageOfAvailableSpace = value; + } + } config->runtimePaths = args::getStrings(args, OPT_rpath); config->allLoad = args.hasArg(OPT_all_load); config->archMultiple = args.hasArg(OPT_arch_multiple); Index: lld/MachO/LTO.h =================================================================== --- lld/MachO/LTO.h +++ lld/MachO/LTO.h @@ -9,6 +9,7 @@ #ifndef LLD_MACHO_LTO_H #define LLD_MACHO_LTO_H +#include "lld/Common/LLVM.h" #include "llvm/ADT/SmallString.h" #include #include @@ -35,6 +36,7 @@ private: std::unique_ptr ltoObj; std::vector> buf; + std::vector> files; }; } // namespace macho Index: lld/MachO/LTO.cpp =================================================================== --- lld/MachO/LTO.cpp +++ lld/MachO/LTO.cpp @@ -17,6 +17,8 @@ #include "lld/Common/ErrorHandler.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" +#include "llvm/LTO/Caching.h" +#include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -97,17 +99,35 @@ std::vector BitcodeCompiler::compile() { unsigned maxTasks = ltoObj->getMaxTasks(); buf.resize(maxTasks); - - checkError(ltoObj->run([&](size_t task) { - return std::make_unique( - std::make_unique(buf[task])); - })); + files.resize(maxTasks); + + // The --thinlto-cache-dir option specifies the path to a directory in which + // to cache native object files for ThinLTO incremental builds. If a path was + // specified, configure LTO to use it as the cache directory. + lto::NativeObjectCache cache; + if (!config->thinLTOCacheDir.empty()) + cache = check( + lto::localCache(config->thinLTOCacheDir, + [&](size_t task, std::unique_ptr mb) { + files[task] = std::move(mb); + })); + + checkError(ltoObj->run( + [&](size_t task) { + return std::make_unique( + std::make_unique(buf[task])); + }, + cache)); + + if (!config->thinLTOCacheDir.empty()) + pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy); if (config->saveTemps) { if (!buf[0].empty()) saveBuffer(buf[0], config->outputFile + ".lto.o"); for (unsigned i = 1; i != maxTasks; ++i) - saveBuffer(buf[i], config->outputFile + Twine(i) + ".lto.o"); + if (!buf[i].empty()) + saveBuffer(buf[i], config->outputFile + Twine(i) + ".lto.o"); } if (!config->ltoObjPath.empty()) @@ -130,6 +150,8 @@ ret.push_back(make( MemoryBufferRef(buf[i], saver.save(filePath.str())), modTime, "")); } - + for (std::unique_ptr &file : files) + if (file) + ret.push_back(make(*file, 0, "")); return ret; } Index: lld/MachO/Options.td =================================================================== --- lld/MachO/Options.td +++ lld/MachO/Options.td @@ -68,6 +68,9 @@ HelpText<"Set optimization level for LTO (default: 2)">, MetaVarName<"">, Group; +def thinlto_cache_policy: Separate<["--"], "thinlto-cache-policy">, + HelpText<"Pruning policy for the ThinLTO cache">, + Group; // This is a complete Options.td compiled from Apple's ld(1) manpage // dated 2018-03-07 and cross checked with ld64 source code in repo @@ -915,22 +918,18 @@ def cache_path_lto : Separate<["-"], "cache_path_lto">, MetaVarName<"">, HelpText<"Use as a directory for the incremental LTO cache">, - Flags<[HelpHidden]>, Group; def prune_interval_lto : Separate<["-"], "prune_interval_lto">, MetaVarName<"">, HelpText<"Prune the incremental LTO cache after (-1 disables pruning)">, - Flags<[HelpHidden]>, Group; def prune_after_lto : Separate<["-"], "prune_after_lto">, MetaVarName<"">, HelpText<"Remove LTO cache entries after ">, - Flags<[HelpHidden]>, Group; def max_relative_cache_size_lto : Separate<["-"], "max_relative_cache_size_lto">, MetaVarName<"">, HelpText<"Limit the incremental LTO cache growth to of free disk, space">, - Flags<[HelpHidden]>, Group; def page_align_data_atoms : Flag<["-"], "page_align_data_atoms">, HelpText<"Distribute global variables on separate pages so page used/dirty status can guide creation of an order file to cluster commonly used/dirty globals">, Index: lld/test/MachO/Inputs/lto-cache.ll =================================================================== --- /dev/null +++ lld/test/MachO/Inputs/lto-cache.ll @@ -0,0 +1,10 @@ +target triple = "x86_64-apple-macosx10.15.0" +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define i32 @main() { +entry: + call void (...) @globalfunc() + ret i32 0 +} + +declare void @globalfunc(...) Index: lld/test/MachO/lto-cache.ll =================================================================== --- /dev/null +++ lld/test/MachO/lto-cache.ll @@ -0,0 +1,60 @@ +; REQUIRES: x86 +; NetBSD: noatime mounts currently inhibit 'touch' from updating atime +; UNSUPPORTED: system-netbsd + +; RUN: opt -module-hash -module-summary %s -o %t.o +; RUN: opt -module-hash -module-summary %p/Inputs/lto-cache.ll -o %t2.o + +; RUN: rm -Rf %t.cache && mkdir %t.cache +; Create two files that would be removed by cache pruning due to age. +; We should only remove files matching the pattern "llvmcache-*". +; RUN: touch -t 197001011200 %t.cache/llvmcache-foo %t.cache/foo +; RUN: %lld -cache_path_lto %t.cache --thinlto-cache-policy prune_after=1h:prune_interval=0s -o %t3 %t2.o %t.o + +; Two cached objects, plus a timestamp file and "foo", minus the file we removed. +; RUN: ls %t.cache | count 4 + +; Create a file of size 64KB. +; RUN: %python -c "print(' ' * 65536)" > %t.cache/llvmcache-foo + +; This should leave the file in place. +; RUN: %lld -cache_path_lto %t.cache --thinlto-cache-policy cache_size_bytes=128k:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 5 + +; Increase the age of llvmcache-foo, which will give it the oldest time stamp +; so that it is processed and removed first. +; RUN: %python -c 'import os,sys,time; t=time.time()-120; os.utime(sys.argv[1],(t,t))' %t.cache/llvmcache-foo + +; This should remove it. +; RUN: %lld -cache_path_lto %t.cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 4 + +; Setting max number of files to 0 should disable the limit, not delete everything. +; RUN: %lld -cache_path_lto %t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 4 + +; Delete everything except for the timestamp, "foo" and one cache file. +; RUN: %lld -cache_path_lto %t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 3 + +; Check that we remove the least recently used file first. +; RUN: rm -fr %t.cache +; RUN: mkdir %t.cache +; RUN: echo xyz > %t.cache/llvmcache-old +; RUN: touch -t 198002011200 %t.cache/llvmcache-old +; RUN: echo xyz > %t.cache/llvmcache-newer +; RUN: touch -t 198002021200 %t.cache/llvmcache-newer +; RUN: %lld -cache_path_lto %t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=3:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | FileCheck %s + +; CHECK-NOT: llvmcache-old +; CHECK: llvmcache-newer +; CHECK-NOT: llvmcache-old + +target triple = "x86_64-apple-macosx10.15.0" +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @globalfunc() #0 { +entry: + ret void +}