Support: Add a cache pruning policy parser.

[pcc]

Summary

The idea is that the policy string fully specifies the policy and is portable
between clients.

[mehdi_amini]

Neat! LGTM.

llvm/include/llvm/LTO/Caching.h

23 ↗

(On Diff #91968)

Do all the clients of this header need this declaration? I found strange that nothing changes in the header but this is needed.

llvm/include/llvm/Support/CachePruning.h

47 ↗

(On Diff #91968)

Could you add an example syntax here?

llvm/lib/Support/CachePruning.cpp

37 ↗

(On Diff #91968)

Why "Interval"? It seems that you're parsing a "time" or a "duration"?

98 ↗

(On Diff #91968)

StringSwitch? I guess it does not play well with the error to return...

[pcc]

llvm/include/llvm/LTO/Caching.h

23 ↗

(On Diff #91968)

This was an unintentional change, I've reverted it.

llvm/lib/Support/CachePruning.cpp

37 ↗

(On Diff #91968)

They're synonyms for the most part, but it looks like "duration" is the term used by the stdlib and distinguishes from the pruning interval, so I've switched to it.

98 ↗

(On Diff #91968)

Right, I guess I'd have to go through contortions in this code that would outweigh the value of StringSwitch IMHO.

[mehdi_amini]

llvm/lib/Support/CachePruning.cpp

37 ↗

(On Diff #91968)

OK.
(maybe because I'm not a native speaker, I thought about "interval" as being synonym to "range", so rather having a begin and an end)

llvm/trunk/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h

public:
    *  - Finally, the garbage collector can be instructed to prune the cache till
    *    the occupied space goes below a threshold.
    * @{
    */
 
   struct CachingOptions {
     std::string Path;                    // Path to the cache, empty to disable.
     CachePruningPolicy Policy;
-
-    CachingOptions() {
-      Policy.Interval = std::chrono::seconds(1200);
-      Policy.Expiration = std::chrono::hours(7 * 24); // 1w
-      Policy.PercentageOfAvailableSpace = 75;
-    };
   };
 
   /// Provide a path to a directory where to store the cached files for
   /// incremental build.
   void setCacheDir(std::string Path) { CacheOptions.Path = std::move(Path); }
 
   /// Cache policy: interval (seconds) between two prune of the cache. Set to a
   /// negative value (default) to disable pruning. A value of 0 will be ignored.

llvm/trunk/include/llvm/Support/CachePruning.h

 #ifndef LLVM_SUPPORT_CACHE_PRUNING_H
 #define LLVM_SUPPORT_CACHE_PRUNING_H
 
 #include "llvm/ADT/StringRef.h"
 #include <chrono>
 
 namespace llvm {
 
+template <typename T> class Expected;
+
+/// Policy for the pruneCache() function. A default constructed
+/// CachePruningPolicy provides a reasonable default policy.
 struct CachePruningPolicy {
   /// The pruning interval. This is intended to be used to avoid scanning the
   /// directory too often. It does not impact the decision of which file to
   /// prune. A value of 0 forces the scan to occur.
-  std::chrono::seconds Interval = std::chrono::seconds::zero();
+  std::chrono::seconds Interval = std::chrono::seconds(1200);
 
   /// The expiration for a file. When a file hasn't been accessed for Expiration
   /// seconds, it is removed from the cache. A value of 0 disables the
   /// expiration-based pruning.
-  std::chrono::seconds Expiration = std::chrono::seconds::zero();
+  std::chrono::seconds Expiration = std::chrono::hours(7 * 24); // 1w
 
   /// The maximum size for the cache directory, in terms of percentage of the
   /// available space on the the disk. Set to 100 to indicate no limit, 50 to
   /// indicate that the cache size will not be left over half the available disk
   /// space. A value over 100 will be reduced to 100. A value of 0 disables the
   /// size-based pruning.
-  unsigned PercentageOfAvailableSpace = 0;
+  unsigned PercentageOfAvailableSpace = 75;
 };
 
+/// Parse the given string as a cache pruning policy. Defaults are taken from a
+/// default constructed CachePruningPolicy object.
+/// For example: "prune_interval=30s:prune_after=24h:cache_size=50%"
+/// which means a pruning interval of 30 seconds, expiration time of 24 hours
+/// and maximum cache size of 50% of available disk space.
+Expected<CachePruningPolicy> parseCachePruningPolicy(StringRef PolicyStr);
+
 /// Peform pruning using the supplied policy, returns true if pruning
 /// occured, i.e. if Policy.Interval was expired.
 bool pruneCache(StringRef Path, CachePruningPolicy Policy);
 
 } // namespace llvm
 
 #endif

llvm/trunk/lib/Support/CachePruning.cpp

 // This file implements the pruning of a directory based on least recently used.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/CachePruning.h"
 
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define DEBUG_TYPE "cache-pruning"
 
 #include <set>
 #include <system_error>
 
 using namespace llvm;
 
 /// Write a new timestamp file with the given path. This is used for the pruning
 /// interval option.
 static void writeTimestampFile(StringRef TimestampFile) {
   std::error_code EC;
   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
 }
 
+static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
+  if (Duration.empty())
+    return make_error<StringError>("Duration must not be empty",
+                                   inconvertibleErrorCode());
+
+  StringRef NumStr = Duration.slice(0, Duration.size()-1);
+  uint64_t Num;
+  if (NumStr.getAsInteger(0, Num))
+    return make_error<StringError>("'" + NumStr + "' not an integer",
+                                   inconvertibleErrorCode());
+
+  switch (Duration.back()) {
+  case 's':
+    return std::chrono::seconds(Num);
+  case 'm':
+    return std::chrono::minutes(Num);
+  case 'h':
+    return std::chrono::hours(Num);
+  default:
+    return make_error<StringError>("'" + Duration +
+                                       "' must end with one of 's', 'm' or 'h'",
+                                   inconvertibleErrorCode());
+  }
+}
+
+Expected<CachePruningPolicy>
+llvm::parseCachePruningPolicy(StringRef PolicyStr) {
+  CachePruningPolicy Policy;
+  std::pair<StringRef, StringRef> P = {"", PolicyStr};
+  while (!P.second.empty()) {
+    P = P.second.split(':');
+
+    StringRef Key, Value;
+    std::tie(Key, Value) = P.first.split('=');
+    if (Key == "prune_interval") {
+      auto DurationOrErr = parseDuration(Value);
+      if (!DurationOrErr)
+        return std::move(DurationOrErr.takeError());
+      Policy.Interval = *DurationOrErr;
+    } else if (Key == "prune_after") {
+      auto DurationOrErr = parseDuration(Value);
+      if (!DurationOrErr)
+        return std::move(DurationOrErr.takeError());
+      Policy.Expiration = *DurationOrErr;
+    } else if (Key == "cache_size") {
+      if (Value.back() != '%')
+        return make_error<StringError>("'" + Value + "' must be a percentage",
+                                       inconvertibleErrorCode());
+      StringRef SizeStr = Value.slice(0, Value.size() - 1);
+      uint64_t Size;
+      if (SizeStr.getAsInteger(0, Size))
+        return make_error<StringError>("'" + SizeStr + "' not an integer",
+                                       inconvertibleErrorCode());
+      if (Size > 100)
+        return make_error<StringError>("'" + SizeStr +
+                                           "' must be between 0 and 100",
+                                       inconvertibleErrorCode());
+      Policy.PercentageOfAvailableSpace = Size;
+    } else {
+      return make_error<StringError>("Unknown key: '" + Key + "'",
+                                     inconvertibleErrorCode());
+    }
+  }
+
+  return Policy;
+}
+
 /// Prune the cache of files that haven't been accessed in a long time.
 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
   using namespace std::chrono;
 
   if (Path.empty())
     return false;
 
   bool isPathDir;

llvm/trunk/unittests/Support/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 add_llvm_unittest(SupportTests
   AlignOfTest.cpp
   AllocatorTest.cpp
   ARMAttributeParser.cpp
   ArrayRecyclerTest.cpp
   BinaryStreamTest.cpp
   BlockFrequencyTest.cpp
   BranchProbabilityTest.cpp
+  CachePruningTest.cpp
   Casting.cpp
   Chrono.cpp
   CommandLineTest.cpp
   CompressionTest.cpp
   ConvertUTFTest.cpp
   DataExtractorTest.cpp
   DebugTest.cpp
   DwarfTest.cpp

llvm/trunk/unittests/Support/CachePruningTest.cpp

+//===- CachePruningTest.cpp -----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+TEST(CachePruningPolicyParser, Empty) {
+  auto P = parseCachePruningPolicy("");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
+  EXPECT_EQ(std::chrono::hours(7 * 24), P->Expiration);
+  EXPECT_EQ(75u, P->PercentageOfAvailableSpace);
+}
+
+TEST(CachePruningPolicyParser, Interval) {
+  auto P = parseCachePruningPolicy("prune_interval=1s");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1), P->Interval);
+  P = parseCachePruningPolicy("prune_interval=2m");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::minutes(2), P->Interval);
+  P = parseCachePruningPolicy("prune_interval=3h");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::hours(3), P->Interval);
+}
+
+TEST(CachePruningPolicyParser, Expiration) {
+  auto P = parseCachePruningPolicy("prune_after=1s");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
+}
+
+TEST(CachePruningPolicyParser, PercentageOfAvailableSpace) {
+  auto P = parseCachePruningPolicy("cache_size=100%");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(100u, P->PercentageOfAvailableSpace);
+}
+
+TEST(CachePruningPolicyParser, Multiple) {
+  auto P = parseCachePruningPolicy("prune_after=1s:cache_size=50%");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
+  EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
+  EXPECT_EQ(50u, P->PercentageOfAvailableSpace);
+}
+
+TEST(CachePruningPolicyParser, Errors) {
+  EXPECT_EQ("Duration must not be empty",
+            toString(parseCachePruningPolicy("prune_interval=").takeError()));
+  EXPECT_EQ("'foo' not an integer",
+            toString(parseCachePruningPolicy("prune_interval=foos").takeError()));
+  EXPECT_EQ("'24x' must end with one of 's', 'm' or 'h'",
+            toString(parseCachePruningPolicy("prune_interval=24x").takeError()));
+  EXPECT_EQ("'foo' must be a percentage",
+            toString(parseCachePruningPolicy("cache_size=foo").takeError()));
+  EXPECT_EQ("'foo' not an integer",
+            toString(parseCachePruningPolicy("cache_size=foo%").takeError()));
+  EXPECT_EQ("'101' must be between 0 and 100",
+            toString(parseCachePruningPolicy("cache_size=101%").takeError()));
+  EXPECT_EQ("Unknown key: 'foo'",
+            toString(parseCachePruningPolicy("foo=bar").takeError()));
+}