Support: Add a cache pruning policy parser.

[pcc]

Summary

The idea is that the policy string fully specifies the policy and is portable
between clients.

[mehdi_amini]

Neat! LGTM.

llvm/include/llvm/LTO/Caching.h

23

Do all the clients of this header need this declaration? I found strange that nothing changes in the header but this is needed.

llvm/include/llvm/Support/CachePruning.h

47

Could you add an example syntax here?

llvm/lib/Support/CachePruning.cpp

37

Why "Interval"? It seems that you're parsing a "time" or a "duration"?

98

StringSwitch? I guess it does not play well with the error to return...

[pcc]

llvm/include/llvm/LTO/Caching.h

23

This was an unintentional change, I've reverted it.

llvm/lib/Support/CachePruning.cpp

37

They're synonyms for the most part, but it looks like "duration" is the term used by the stdlib and distinguishes from the pruning interval, so I've switched to it.

98

Right, I guess I'd have to go through contortions in this code that would outweigh the value of StringSwitch IMHO.

[mehdi_amini]

llvm/lib/Support/CachePruning.cpp

37

OK.
(maybe because I'm not a native speaker, I thought about "interval" as being synonym to "range", so rather having a begin and an end)

llvm/include/llvm/LTO/Caching.h

 
 #ifndef LLVM_LTO_CACHING_H
 #define LLVM_LTO_CACHING_H
 
 #include "llvm/LTO/LTO.h"
 #include <string>
 
 namespace llvm {
+
+struct CachePruningPolicy;
+
 namespace lto {
 
 /// This type defines the callback to add a pre-existing native object file
 /// (e.g. in a cache).
 ///
 /// File callbacks must be thread safe.
 typedef std::function<void(unsigned Task, StringRef Path)> AddFileFn;
 

llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h

public:
    *  - Finally, the garbage collector can be instructed to prune the cache till
    *    the occupied space goes below a threshold.
    * @{
    */
 
   struct CachingOptions {
     std::string Path;                    // Path to the cache, empty to disable.
     CachePruningPolicy Policy;
-
-    CachingOptions() {
-      Policy.Interval = std::chrono::seconds(1200);
-      Policy.Expiration = std::chrono::hours(7 * 24); // 1w
-      Policy.PercentageOfAvailableSpace = 75;
-    };
   };
 
   /// Provide a path to a directory where to store the cached files for
   /// incremental build.
   void setCacheDir(std::string Path) { CacheOptions.Path = std::move(Path); }
 
   /// Cache policy: interval (seconds) between two prune of the cache. Set to a
   /// negative value (default) to disable pruning. A value of 0 will be ignored.

llvm/include/llvm/Support/CachePruning.h

 #ifndef LLVM_SUPPORT_CACHE_PRUNING_H
 #define LLVM_SUPPORT_CACHE_PRUNING_H
 
 #include "llvm/ADT/StringRef.h"
 #include <chrono>
 
 namespace llvm {
 
+template <typename T> class Expected;
+
+/// Policy for the pruneCache() function. A default constructed
+/// CachePruningPolicy provides a reasonable default policy.
 struct CachePruningPolicy {
   /// The pruning interval. This is intended to be used to avoid scanning the
   /// directory too often. It does not impact the decision of which file to
   /// prune. A value of 0 forces the scan to occur.
-  std::chrono::seconds Interval = std::chrono::seconds::zero();
+  std::chrono::seconds Interval = std::chrono::seconds(1200);
 
   /// The expiration for a file. When a file hasn't been accessed for Expiration
   /// seconds, it is removed from the cache. A value of 0 disables the
   /// expiration-based pruning.
-  std::chrono::seconds Expiration = std::chrono::seconds::zero();
+  std::chrono::seconds Expiration = std::chrono::hours(7 * 24); // 1w
 
   /// The maximum size for the cache directory, in terms of percentage of the
   /// available space on the the disk. Set to 100 to indicate no limit, 50 to
   /// indicate that the cache size will not be left over half the available disk
   /// space. A value over 100 will be reduced to 100. A value of 0 disables the
   /// size-based pruning.
-  unsigned PercentageOfAvailableSpace = 0;
+  unsigned PercentageOfAvailableSpace = 75;
 };
 
+/// Parse the given string as a cache pruning policy. Defaults are taken from a
+/// default constructed CachePruningPolicy object.
+Expected<CachePruningPolicy> parseCachePruningPolicy(StringRef PolicyStr);
+
 /// Peform pruning using the supplied policy, returns true if pruning
 /// occured, i.e. if Policy.Interval was expired.
 bool pruneCache(StringRef Path, CachePruningPolicy Policy);
 
 } // namespace llvm
 
 #endif

llvm/lib/Support/CachePruning.cpp

 // This file implements the pruning of a directory based on least recently used.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/CachePruning.h"
 
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define DEBUG_TYPE "cache-pruning"
 
 #include <set>
 #include <system_error>
 
 using namespace llvm;
 
 /// Write a new timestamp file with the given path. This is used for the pruning
 /// interval option.
 static void writeTimestampFile(StringRef TimestampFile) {
   std::error_code EC;
   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
 }
 
+static Expected<std::chrono::seconds> parseInterval(StringRef Interval) {
+  if (Interval.empty())
+    return make_error<StringError>("Interval must not be empty",
+                                   inconvertibleErrorCode());
+
+  StringRef NumStr = Interval.slice(0, Interval.size()-1);
+  uint64_t Num;
+  if (NumStr.getAsInteger(0, Num))
+    return make_error<StringError>("'" + NumStr + "' not an integer",
+                                   inconvertibleErrorCode());
+
+  switch (Interval.back()) {
+  case 's':
+    return std::chrono::seconds(Num);
+  case 'm':
+    return std::chrono::minutes(Num);
+  case 'h':
+    return std::chrono::hours(Num);
+  default:
+    return make_error<StringError>("'" + Interval +
+                                       "' must end with one of 's', 'm' or 'h'",
+                                   inconvertibleErrorCode());
+  }
+}
+
+Expected<CachePruningPolicy>
+llvm::parseCachePruningPolicy(StringRef PolicyStr) {
+  CachePruningPolicy Policy;
+  std::pair<StringRef, StringRef> P = {"", PolicyStr};
+  while (!P.second.empty()) {
+    P = P.second.split(':');
+
+    StringRef Key, Value;
+    std::tie(Key, Value) = P.first.split('=');
+    if (Key == "prune_interval") {
+      auto IntervalOrErr = parseInterval(Value);
+      if (!IntervalOrErr)
+        return std::move(IntervalOrErr.takeError());
+      Policy.Interval = *IntervalOrErr;
+    } else if (Key == "prune_after") {
+      auto IntervalOrErr = parseInterval(Value);
+      if (!IntervalOrErr)
+        return std::move(IntervalOrErr.takeError());
+      Policy.Expiration = *IntervalOrErr;
+    } else if (Key == "cache_size") {
+      if (Value.back() != '%')
+        return make_error<StringError>("'" + Value + "' must be a percentage",
+                                       inconvertibleErrorCode());
+      StringRef SizeStr = Value.slice(0, Value.size() - 1);
+      uint64_t Size;
+      if (SizeStr.getAsInteger(0, Size))
+        return make_error<StringError>("'" + SizeStr + "' not an integer",
+                                       inconvertibleErrorCode());
+      if (Size > 100)
+        return make_error<StringError>("'" + SizeStr +
+                                           "' must be between 0 and 100",
+                                       inconvertibleErrorCode());
+      Policy.PercentageOfAvailableSpace = Size;
+    } else {
+      return make_error<StringError>("Unknown key: '" + Key + "'",
+                                     inconvertibleErrorCode());
+    }
+  }
+
+  return Policy;
+}
+
 /// Prune the cache of files that haven't been accessed in a long time.
 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
   using namespace std::chrono;
 
   if (Path.empty())
     return false;
 
   bool isPathDir;

llvm/unittests/Support/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 add_llvm_unittest(SupportTests
   AlignOfTest.cpp
   AllocatorTest.cpp
   ARMAttributeParser.cpp
   ArrayRecyclerTest.cpp
   BinaryStreamTest.cpp
   BlockFrequencyTest.cpp
   BranchProbabilityTest.cpp
+  CachePruningTest.cpp
   Casting.cpp
   Chrono.cpp
   CommandLineTest.cpp
   CompressionTest.cpp
   ConvertUTFTest.cpp
   DataExtractorTest.cpp
   DebugTest.cpp
   DwarfTest.cpp

llvm/unittests/Support/CachePruningTest.cpp

+//===- CachePruningTest.cpp -----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+TEST(CachePruningPolicyParser, Empty) {
+  auto P = parseCachePruningPolicy("");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
+  EXPECT_EQ(std::chrono::hours(7 * 24), P->Expiration);
+  EXPECT_EQ(75u, P->PercentageOfAvailableSpace);
+}
+
+TEST(CachePruningPolicyParser, Interval) {
+  auto P = parseCachePruningPolicy("prune_interval=1s");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1), P->Interval);
+  P = parseCachePruningPolicy("prune_interval=2m");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::minutes(2), P->Interval);
+  P = parseCachePruningPolicy("prune_interval=3h");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::hours(3), P->Interval);
+}
+
+TEST(CachePruningPolicyParser, Expiration) {
+  auto P = parseCachePruningPolicy("prune_after=1s");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
+}
+
+TEST(CachePruningPolicyParser, PercentageOfAvailableSpace) {
+  auto P = parseCachePruningPolicy("cache_size=100%");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(100u, P->PercentageOfAvailableSpace);
+}
+
+TEST(CachePruningPolicyParser, Multiple) {
+  auto P = parseCachePruningPolicy("prune_after=1s:cache_size=50%");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
+  EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
+  EXPECT_EQ(50u, P->PercentageOfAvailableSpace);
+}
+
+TEST(CachePruningPolicyParser, Errors) {
+  EXPECT_EQ("Interval must not be empty",
+            toString(parseCachePruningPolicy("prune_interval=").takeError()));
+  EXPECT_EQ("'foo' not an integer",
+            toString(parseCachePruningPolicy("prune_interval=foos").takeError()));
+  EXPECT_EQ("'24x' must end with one of 's', 'm' or 'h'",
+            toString(parseCachePruningPolicy("prune_interval=24x").takeError()));
+  EXPECT_EQ("'foo' must be a percentage",
+            toString(parseCachePruningPolicy("cache_size=foo").takeError()));
+  EXPECT_EQ("'foo' not an integer",
+            toString(parseCachePruningPolicy("cache_size=foo%").takeError()));
+  EXPECT_EQ("'101' must be between 0 and 100",
+            toString(parseCachePruningPolicy("cache_size=101%").takeError()));
+  EXPECT_EQ("Unknown key: 'foo'",
+            toString(parseCachePruningPolicy("foo=bar").takeError()));
+}