CachePruning: Allow limiting the number of files in the cache directory.

[pcc]

Summary

The default limit is 1000000 but it can be configured with a cache
policy. The motivation is that some filesystems (notably ext4) have
a limit on the number of files that can be contained in a directory
(separate from the inode limit).

[tejohnson]

llvm/lib/Support/CachePruning.cpp

232 ↗

(On Diff #123871)

I think you will need to change the way ShouldComputeSize is set above (i.e. OR in Policy.MaxSizeFiles > 0) to ensure it is true and therefore FileSizes and TotalSize have been computed at this point.

[pcc]

• Bug fixes

[pcc]

llvm/lib/Support/CachePruning.cpp

232 ↗

(On Diff #123871)

Done, also fixed a bug where we were exiting early above, and another bug where prune_after=0s did not behave as documented.

[tejohnson]

LGTM

cfe/trunk/docs/ThinLTO.rst

- ``cache_size_bytes=X``, ``cache_size_bytes=Xk``, ``cache_size_bytes=Xm``,
   will be reduced to the amount of available space. A value of 0 disables
   the byte size-based pruning. The default is no byte size-based pruning.
 
   Note that ThinLTO will apply both size-based pruning policies simultaneously,
   and changing one does not affect the other. For example, a policy of
   ``cache_size_bytes=1g`` on its own will cause both the 1GB and default 75%
   policies to be applied unless the default ``cache_size`` is overridden.
 
+- ``cache_size_files=X``:
+  Set the maximum number of files in the cache directory. Set to 0 to indicate
+  no limit. The default is 1000000 files.
+
 - ``prune_after=Xs``, ``prune_after=Xm``, ``prune_after=Xh``: Sets the
   expiration time for cache files to ``X`` seconds (or minutes, hours
   respectively).  When a file hasn't been accessed for ``prune_after`` seconds,
   it is removed from the cache. A value of 0 disables the expiration-based
   pruning. The default is 1 week.
 
 - ``prune_interval=Xs``, ``prune_interval=Xm``, ``prune_interval=Xh``:
   Sets the pruning interval to ``X`` seconds (or minutes, hours

lld/trunk/test/ELF/lto/cache.ll

 ; This should leave the file in place.
 ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=128k:prune_interval=0s -o %t3 %t2.o %t.o
 ; RUN: ls %t.cache | count 5
 
 ; This should remove it.
 ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o %t3 %t2.o %t.o
 ; RUN: ls %t.cache | count 4
 
+; Setting max number of files to 0 should disable the limit, not delete everything.
+; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o %t3 %t2.o %t.o
+; RUN: ls %t.cache | count 4
+
+; Delete everything except for the timestamp, "foo" and one cache file.
+; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o %t3 %t2.o %t.o
+; RUN: ls %t.cache | count 3
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @globalfunc() #0 {
 entry:
   ret void
 }

llvm/trunk/include/llvm/Support/CachePruning.h

struct CachePruningPolicy {
   /// space. A value over 100 will be reduced to 100. A value of 0 disables the
   /// percentage size-based pruning.
   unsigned MaxSizePercentageOfAvailableSpace = 75;
 
   /// The maximum size for the cache directory in bytes. A value over the amount
   /// of available space on the disk will be reduced to the amount of available
   /// space. A value of 0 disables the absolute size-based pruning.
   uint64_t MaxSizeBytes = 0;
+
+  /// The maximum number of files in the cache directory. A value of 0 disables
+  /// the number of files based pruning.
+  ///
+  /// This defaults to 1000000 because with that many files there are
+  /// diminishing returns on the effectiveness of the cache, and some file
+  /// systems have a limit on how many files can be contained in a directory
+  /// (notably ext4, which is limited to around 6000000 files).
+  uint64_t MaxSizeFiles = 1000000;
 };
 
 /// Parse the given string as a cache pruning policy. Defaults are taken from a
 /// default constructed CachePruningPolicy object.
 /// For example: "prune_interval=30s:prune_after=24h:cache_size=50%"
 /// which means a pruning interval of 30 seconds, expiration time of 24 hours
 /// and maximum cache size of 50% of available disk space.
 Expected<CachePruningPolicy> parseCachePruningPolicy(StringRef PolicyStr);

llvm/trunk/lib/Support/CachePruning.cpp

if (Key == "prune_interval") {
         Value = Value.drop_back();
         break;
       }
       uint64_t Size;
       if (Value.getAsInteger(0, Size))
         return make_error<StringError>("'" + Value + "' not an integer",
                                        inconvertibleErrorCode());
       Policy.MaxSizeBytes = Size * Mult;
+    } else if (Key == "cache_size_files") {
+      if (Value.getAsInteger(0, Policy.MaxSizeFiles))
+        return make_error<StringError>("'" + Value + "' not an integer",
+                                       inconvertibleErrorCode());
     } else {
       return make_error<StringError>("Unknown key: '" + Key + "'",
                                      inconvertibleErrorCode());
     }
   }
 
   return Policy;
 }
bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
   if (!isPathDir)
     return false;
 
   Policy.MaxSizePercentageOfAvailableSpace =
       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
 
   if (Policy.Expiration == seconds(0) &&
       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
-      Policy.MaxSizeBytes == 0) {
+      Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
     DEBUG(dbgs() << "No pruning settings set, exit early\n");
     // Nothing will be pruned, early exit
     return false;
   }
 
   // Try to stat() the timestamp file.
   SmallString<128> TimestampFile(Path);
   sys::path::append(TimestampFile, "llvmcache.timestamp");
if (Policy.Interval != seconds(0)) {
       }
     }
     // Write a new timestamp file so that nobody else attempts to prune.
     // There is a benign race condition here, if two processes happen to
     // notice at the same time that the timestamp is out-of-date.
     writeTimestampFile(TimestampFile);
   }
 
-  bool ShouldComputeSize =
-      (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
-
   // Keep track of space. Needs to be kept ordered by size for determinism.
   std::set<std::pair<uint64_t, std::string>> FileSizes;
   uint64_t TotalSize = 0;
 
   // Walk the entire directory cache, looking for unused files.
   std::error_code EC;
   SmallString<128> CachePathNative;
   sys::path::native(Path, CachePathNative);
for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
     if (!StatusOrErr) {
       DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
       continue;
     }
 
     // If the file hasn't been used recently enough, delete it
     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
     auto FileAge = CurrentTime - FileAccessTime;
-    if (FileAge > Policy.Expiration) {
+    if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
       DEBUG(dbgs() << "Remove " << File->path() << " ("
                    << duration_cast<seconds>(FileAge).count() << "s old)\n");
       sys::fs::remove(File->path());
       continue;
     }
 
     // Leave it here for now, but add it to the list of size-based pruning.
-    if (!ShouldComputeSize)
-      continue;
     TotalSize += StatusOrErr->getSize();
     FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
   }
 
+  auto FileAndSize = FileSizes.rbegin();
+  size_t NumFiles = FileSizes.size();
+
+  auto RemoveCacheFile = [&]() {
+    // Remove the file.
+    sys::fs::remove(FileAndSize->second);
+    // Update size
+    TotalSize -= FileAndSize->first;
+    NumFiles--;
+    DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
+                 << FileAndSize->first << "), new occupancy is " << TotalSize
+                 << "%\n");
+    ++FileAndSize;
+  };
+
+  // Prune for number of files.
+  if (Policy.MaxSizeFiles)
+    while (NumFiles > Policy.MaxSizeFiles)
+      RemoveCacheFile();
+
   // Prune for size now if needed
-  if (ShouldComputeSize) {
+  if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
     if (!ErrOrSpaceInfo) {
       report_fatal_error("Can't get available size");
     }
     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
     auto AvailableSpace = TotalSize + SpaceInfo.free;
 
     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
       Policy.MaxSizePercentageOfAvailableSpace = 100;
     if (Policy.MaxSizeBytes == 0)
       Policy.MaxSizeBytes = AvailableSpace;
     auto TotalSizeTarget = std::min<uint64_t>(
         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
         Policy.MaxSizeBytes);
 
     DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
                  << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
                  << "%, " << Policy.MaxSizeBytes << " bytes\n");
 
-    auto FileAndSize = FileSizes.rbegin();
-    // Remove the oldest accessed files first, till we get below the threshold
-    while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
-      // Remove the file.
-      sys::fs::remove(FileAndSize->second);
-      // Update size
-      TotalSize -= FileAndSize->first;
-      DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
-                   << FileAndSize->first << "), new occupancy is " << TotalSize
-                   << "%\n");
-      ++FileAndSize;
-    }
+    // Remove the oldest accessed files first, till we get below the threshold.
+    while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
+      RemoveCacheFile();
   }
   return true;
 }