Index: clang/docs/ThinLTO.rst =================================================================== --- clang/docs/ThinLTO.rst +++ clang/docs/ThinLTO.rst @@ -173,6 +173,10 @@ ``cache_size_bytes=1g`` on its own will cause both the 1GB and default 75% policies to be applied unless the default ``cache_size`` is overridden. +- ``cache_size_files=X``: + Set the maximum number of files in the cache directory. Set to 0 to indicate + no limit. The default is 1000000 files. + - ``prune_after=Xs``, ``prune_after=Xm``, ``prune_after=Xh``: Sets the expiration time for cache files to ``X`` seconds (or minutes, hours respectively). When a file hasn't been accessed for ``prune_after`` seconds, Index: lld/test/ELF/lto/cache.ll =================================================================== --- lld/test/ELF/lto/cache.ll +++ lld/test/ELF/lto/cache.ll @@ -23,6 +23,14 @@ ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o %t3 %t2.o %t.o ; RUN: ls %t.cache | count 4 +; Setting max number of files to 0 should disable the limit, not delete everything. +; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_files=0:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 4 + +; Delete everything except for the timestamp, "foo" and one cache file. +; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_files=1:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 3 + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/include/llvm/Support/CachePruning.h =================================================================== --- llvm/include/llvm/Support/CachePruning.h +++ llvm/include/llvm/Support/CachePruning.h @@ -46,6 +46,15 @@ /// of available space on the disk will be reduced to the amount of available /// space. A value of 0 disables the absolute size-based pruning. uint64_t MaxSizeBytes = 0; + + /// The maximum number of files in the cache directory. A value of 0 disables + /// the number of files based pruning. + /// + /// This defaults to 1000000 because with that many files there are + /// diminishing returns on the effectiveness of the cache, and some file + /// systems have a limit on how many files can be contained in a directory + /// (notably ext4, which is limited to around 6000000 files). + uint64_t MaxSizeFiles = 1000000; }; /// Parse the given string as a cache pruning policy. Defaults are taken from a Index: llvm/lib/Support/CachePruning.cpp =================================================================== --- llvm/lib/Support/CachePruning.cpp +++ llvm/lib/Support/CachePruning.cpp @@ -113,6 +113,10 @@ return make_error("'" + Value + "' not an integer", inconvertibleErrorCode()); Policy.MaxSizeBytes = Size * Mult; + } else if (Key == "cache_size_files") { + if (Value.getAsInteger(0, Policy.MaxSizeFiles)) + return make_error("'" + Value + "' not an integer", + inconvertibleErrorCode()); } else { return make_error("Unknown key: '" + Key + "'", inconvertibleErrorCode()); @@ -225,6 +229,26 @@ FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())}); } + auto FileAndSize = FileSizes.rbegin(); + size_t NumFiles = FileSizes.size(); + + auto RemoveCacheFile = [&]() { + // Remove the file. + sys::fs::remove(FileAndSize->second); + // Update size + TotalSize -= FileAndSize->first; + NumFiles--; + DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size " + << FileAndSize->first << "), new occupancy is " << TotalSize + << "%\n"); + ++FileAndSize; + }; + + // Prune for number of files. + if (Policy.MaxSizeFiles) + while (NumFiles > Policy.MaxSizeFiles) + RemoveCacheFile(); + // Prune for size now if needed if (ShouldComputeSize) { auto ErrOrSpaceInfo = sys::fs::disk_space(Path); @@ -246,18 +270,9 @@ << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace << "%, " << Policy.MaxSizeBytes << " bytes\n"); - auto FileAndSize = FileSizes.rbegin(); - // Remove the oldest accessed files first, till we get below the threshold - while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) { - // Remove the file. - sys::fs::remove(FileAndSize->second); - // Update size - TotalSize -= FileAndSize->first; - DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size " - << FileAndSize->first << "), new occupancy is " << TotalSize - << "%\n"); - ++FileAndSize; - } + // Remove the oldest accessed files first, till we get below the threshold. + while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) + RemoveCacheFile(); } return true; }