CachePruning: Allow limiting the number of files in the cache directory.

[pcc]

Summary

The default limit is 1000000 but it can be configured with a cache
policy. The motivation is that some filesystems (notably ext4) have
a limit on the number of files that can be contained in a directory
(separate from the inode limit).

[tejohnson]

llvm/lib/Support/CachePruning.cpp

232

I think you will need to change the way ShouldComputeSize is set above (i.e. OR in Policy.MaxSizeFiles > 0) to ensure it is true and therefore FileSizes and TotalSize have been computed at this point.

[pcc]

• Bug fixes

[pcc]

llvm/lib/Support/CachePruning.cpp

232

Done, also fixed a bug where we were exiting early above, and another bug where prune_after=0s did not behave as documented.

[tejohnson]

LGTM

clang/docs/ThinLTO.rst

- ``cache_size_bytes=X``, ``cache_size_bytes=Xk``, ``cache_size_bytes=Xm``,
   will be reduced to the amount of available space. A value of 0 disables
   the byte size-based pruning. The default is no byte size-based pruning.
 
   Note that ThinLTO will apply both size-based pruning policies simultaneously,
   and changing one does not affect the other. For example, a policy of
   ``cache_size_bytes=1g`` on its own will cause both the 1GB and default 75%
   policies to be applied unless the default ``cache_size`` is overridden.
 
+- ``cache_size_files=X``:
+  Set the maximum number of files in the cache directory. Set to 0 to indicate
+  no limit. The default is 1000000 files.
+
 - ``prune_after=Xs``, ``prune_after=Xm``, ``prune_after=Xh``: Sets the
   expiration time for cache files to ``X`` seconds (or minutes, hours
   respectively).  When a file hasn't been accessed for ``prune_after`` seconds,
   it is removed from the cache. A value of 0 disables the expiration-based
   pruning. The default is 1 week.
 
 - ``prune_interval=Xs``, ``prune_interval=Xm``, ``prune_interval=Xh``:
   Sets the pruning interval to ``X`` seconds (or minutes, hours

lld/test/ELF/lto/cache.ll

 ; This should leave the file in place.
 ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=128k:prune_interval=0s -o %t3 %t2.o %t.o
 ; RUN: ls %t.cache | count 5
 
 ; This should remove it.
 ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o %t3 %t2.o %t.o
 ; RUN: ls %t.cache | count 4
 
+; Setting max number of files to 0 should disable the limit, not delete everything.
+; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_files=0:prune_interval=0s -o %t3 %t2.o %t.o
+; RUN: ls %t.cache | count 4
+
+; Delete everything except for the timestamp, "foo" and one cache file.
+; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_files=1:prune_interval=0s -o %t3 %t2.o %t.o
+; RUN: ls %t.cache | count 3
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @globalfunc() #0 {
 entry:
   ret void
 }

llvm/include/llvm/Support/CachePruning.h

struct CachePruningPolicy {
   /// space. A value over 100 will be reduced to 100. A value of 0 disables the
   /// percentage size-based pruning.
   unsigned MaxSizePercentageOfAvailableSpace = 75;
 
   /// The maximum size for the cache directory in bytes. A value over the amount
   /// of available space on the disk will be reduced to the amount of available
   /// space. A value of 0 disables the absolute size-based pruning.
   uint64_t MaxSizeBytes = 0;
+
+  /// The maximum number of files in the cache directory. A value of 0 disables
+  /// the number of files based pruning.
+  ///
+  /// This defaults to 1000000 because with that many files there are
+  /// diminishing returns on the effectiveness of the cache, and some file
+  /// systems have a limit on how many files can be contained in a directory
+  /// (notably ext4, which is limited to around 6000000 files).
+  uint64_t MaxSizeFiles = 1000000;
 };
 
 /// Parse the given string as a cache pruning policy. Defaults are taken from a
 /// default constructed CachePruningPolicy object.
 /// For example: "prune_interval=30s:prune_after=24h:cache_size=50%"
 /// which means a pruning interval of 30 seconds, expiration time of 24 hours
 /// and maximum cache size of 50% of available disk space.
 Expected<CachePruningPolicy> parseCachePruningPolicy(StringRef PolicyStr);

llvm/lib/Support/CachePruning.cpp

if (Key == "prune_interval") {
         Value = Value.drop_back();
         break;
       }
       uint64_t Size;
       if (Value.getAsInteger(0, Size))
         return make_error<StringError>("'" + Value + "' not an integer",
                                        inconvertibleErrorCode());
       Policy.MaxSizeBytes = Size * Mult;
+    } else if (Key == "cache_size_files") {
+      if (Value.getAsInteger(0, Policy.MaxSizeFiles))
+        return make_error<StringError>("'" + Value + "' not an integer",
+                                       inconvertibleErrorCode());
     } else {
       return make_error<StringError>("Unknown key: '" + Key + "'",
                                      inconvertibleErrorCode());
     }
   }
 
   return Policy;
 }
for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
 
     // Leave it here for now, but add it to the list of size-based pruning.
     if (!ShouldComputeSize)
       continue;
     TotalSize += StatusOrErr->getSize();
     FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
   }
 
+  auto FileAndSize = FileSizes.rbegin();
+  size_t NumFiles = FileSizes.size();
+
+  auto RemoveCacheFile = [&]() {
+    // Remove the file.
+    sys::fs::remove(FileAndSize->second);
+    // Update size
+    TotalSize -= FileAndSize->first;
+    NumFiles--;
+    DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
+                 << FileAndSize->first << "), new occupancy is " << TotalSize
+                 << "%\n");
+    ++FileAndSize;
+  };
+
+  // Prune for number of files.
+  if (Policy.MaxSizeFiles)
+    while (NumFiles > Policy.MaxSizeFiles)
+      RemoveCacheFile();
+
   // Prune for size now if needed
   if (ShouldComputeSize) {
     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
     if (!ErrOrSpaceInfo) {
       report_fatal_error("Can't get available size");
     }
     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
     auto AvailableSpace = TotalSize + SpaceInfo.free;
 
     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
       Policy.MaxSizePercentageOfAvailableSpace = 100;
     if (Policy.MaxSizeBytes == 0)
       Policy.MaxSizeBytes = AvailableSpace;
     auto TotalSizeTarget = std::min<uint64_t>(
         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
         Policy.MaxSizeBytes);
 
     DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
                  << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
                  << "%, " << Policy.MaxSizeBytes << " bytes\n");
 
-    auto FileAndSize = FileSizes.rbegin();
-    // Remove the oldest accessed files first, till we get below the threshold
-    while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
-      // Remove the file.
-      sys::fs::remove(FileAndSize->second);
-      // Update size
-      TotalSize -= FileAndSize->first;
-      DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
-                   << FileAndSize->first << "), new occupancy is " << TotalSize
-                   << "%\n");
-      ++FileAndSize;
-    }
+    // Remove the oldest accessed files first, till we get below the threshold.
+    while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
+      RemoveCacheFile();
   }
   return true;
 }