Index: cfe/trunk/docs/ThinLTO.rst =================================================================== --- cfe/trunk/docs/ThinLTO.rst +++ cfe/trunk/docs/ThinLTO.rst @@ -173,6 +173,10 @@ ``cache_size_bytes=1g`` on its own will cause both the 1GB and default 75% policies to be applied unless the default ``cache_size`` is overridden. +- ``cache_size_files=X``: + Set the maximum number of files in the cache directory. Set to 0 to indicate + no limit. The default is 1000000 files. + - ``prune_after=Xs``, ``prune_after=Xm``, ``prune_after=Xh``: Sets the expiration time for cache files to ``X`` seconds (or minutes, hours respectively). When a file hasn't been accessed for ``prune_after`` seconds, Index: lld/trunk/test/ELF/lto/cache.ll =================================================================== --- lld/trunk/test/ELF/lto/cache.ll +++ lld/trunk/test/ELF/lto/cache.ll @@ -23,6 +23,14 @@ ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o %t3 %t2.o %t.o ; RUN: ls %t.cache | count 4 +; Setting max number of files to 0 should disable the limit, not delete everything. +; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 4 + +; Delete everything except for the timestamp, "foo" and one cache file. +; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 3 + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/trunk/include/llvm/Support/CachePruning.h =================================================================== --- llvm/trunk/include/llvm/Support/CachePruning.h +++ llvm/trunk/include/llvm/Support/CachePruning.h @@ -46,6 +46,15 @@ /// of available space on the disk will be reduced to the amount of available /// space. A value of 0 disables the absolute size-based pruning. uint64_t MaxSizeBytes = 0; + + /// The maximum number of files in the cache directory. A value of 0 disables + /// the number of files based pruning. + /// + /// This defaults to 1000000 because with that many files there are + /// diminishing returns on the effectiveness of the cache, and some file + /// systems have a limit on how many files can be contained in a directory + /// (notably ext4, which is limited to around 6000000 files). + uint64_t MaxSizeFiles = 1000000; }; /// Parse the given string as a cache pruning policy. Defaults are taken from a Index: llvm/trunk/lib/Support/CachePruning.cpp =================================================================== --- llvm/trunk/lib/Support/CachePruning.cpp +++ llvm/trunk/lib/Support/CachePruning.cpp @@ -113,6 +113,10 @@ return make_error("'" + Value + "' not an integer", inconvertibleErrorCode()); Policy.MaxSizeBytes = Size * Mult; + } else if (Key == "cache_size_files") { + if (Value.getAsInteger(0, Policy.MaxSizeFiles)) + return make_error("'" + Value + "' not an integer", + inconvertibleErrorCode()); } else { return make_error("Unknown key: '" + Key + "'", inconvertibleErrorCode()); @@ -141,7 +145,7 @@ if (Policy.Expiration == seconds(0) && Policy.MaxSizePercentageOfAvailableSpace == 0 && - Policy.MaxSizeBytes == 0) { + Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) { DEBUG(dbgs() << "No pruning settings set, exit early\n"); // Nothing will be pruned, early exit return false; @@ -179,9 +183,6 @@ writeTimestampFile(TimestampFile); } - bool ShouldComputeSize = - (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0); - // Keep track of space. Needs to be kept ordered by size for determinism. std::set> FileSizes; uint64_t TotalSize = 0; @@ -211,7 +212,7 @@ // If the file hasn't been used recently enough, delete it const auto FileAccessTime = StatusOrErr->getLastAccessedTime(); auto FileAge = CurrentTime - FileAccessTime; - if (FileAge > Policy.Expiration) { + if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) { DEBUG(dbgs() << "Remove " << File->path() << " (" << duration_cast(FileAge).count() << "s old)\n"); sys::fs::remove(File->path()); @@ -219,14 +220,32 @@ } // Leave it here for now, but add it to the list of size-based pruning. - if (!ShouldComputeSize) - continue; TotalSize += StatusOrErr->getSize(); FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())}); } + auto FileAndSize = FileSizes.rbegin(); + size_t NumFiles = FileSizes.size(); + + auto RemoveCacheFile = [&]() { + // Remove the file. + sys::fs::remove(FileAndSize->second); + // Update size + TotalSize -= FileAndSize->first; + NumFiles--; + DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size " + << FileAndSize->first << "), new occupancy is " << TotalSize + << "%\n"); + ++FileAndSize; + }; + + // Prune for number of files. + if (Policy.MaxSizeFiles) + while (NumFiles > Policy.MaxSizeFiles) + RemoveCacheFile(); + // Prune for size now if needed - if (ShouldComputeSize) { + if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) { auto ErrOrSpaceInfo = sys::fs::disk_space(Path); if (!ErrOrSpaceInfo) { report_fatal_error("Can't get available size"); @@ -246,18 +265,9 @@ << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace << "%, " << Policy.MaxSizeBytes << " bytes\n"); - auto FileAndSize = FileSizes.rbegin(); - // Remove the oldest accessed files first, till we get below the threshold - while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) { - // Remove the file. - sys::fs::remove(FileAndSize->second); - // Update size - TotalSize -= FileAndSize->first; - DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size " - << FileAndSize->first << "), new occupancy is " << TotalSize - << "%\n"); - ++FileAndSize; - } + // Remove the oldest accessed files first, till we get below the threshold. + while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) + RemoveCacheFile(); } return true; }