Index: clang/docs/ThinLTO.rst =================================================================== --- clang/docs/ThinLTO.rst +++ clang/docs/ThinLTO.rst @@ -146,6 +146,18 @@ disk space. A value over 100 is invalid. A value of 0 disables the percentage size-based pruning. The default is 75%. +- ``cache_size_bytes=X``, ``cache_size_bytes=Xk``, ``cache_size_bytes=Xm``, + ``cache_size_bytes=Xg``: + Sets the maximum size for the cache directory to ``X`` bytes (or KB, MB, + GB respectively). A value over the amount of available space on the disk + will be reduced to the amount of available space. A value of 0 disables + the byte size-based pruning. The default is no byte size-based pruning. + + Note that ThinLTO will apply both size-based pruning policies simultaneously, + and changing one does not affect the other. For example, a policy of + ``cache_size_bytes=1g`` on its own will cause both the 1GB and default 75% + policies to be applied unless the default ``cache_size`` is overridden. + - ``prune_after=Xs``, ``prune_after=Xm``, ``prune_after=Xh``: Sets the expiration time for cache files to ``X`` seconds (or minutes, hours respectively). When a file hasn't been accessed for ``prune_after`` seconds, Index: lld/test/ELF/lto/cache.ll =================================================================== --- lld/test/ELF/lto/cache.ll +++ lld/test/ELF/lto/cache.ll @@ -12,6 +12,17 @@ ; Two cached objects, plus a timestamp file and "foo", minus the file we removed. ; RUN: ls %t.cache | count 4 +; Create a file of size 64KB. +; RUN: %python -c "print ' ' * 65536" > %t.cache/llvmcache-foo + +; This should leave the file in place. +; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=128k -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 5 + +; This should remove it. +; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=32k -o %t3 %t2.o %t.o +; RUN: ls %t.cache | count 4 + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h =================================================================== --- llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h +++ llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h @@ -177,7 +177,7 @@ */ void setMaxCacheSizeRelativeToAvailableSpace(unsigned Percentage) { if (Percentage) - CacheOptions.Policy.PercentageOfAvailableSpace = Percentage; + CacheOptions.Policy.MaxSizePercentageOfAvailableSpace = Percentage; } /**@}*/ Index: llvm/include/llvm/Support/CachePruning.h =================================================================== --- llvm/include/llvm/Support/CachePruning.h +++ llvm/include/llvm/Support/CachePruning.h @@ -39,8 +39,13 @@ /// available space on the the disk. Set to 100 to indicate no limit, 50 to /// indicate that the cache size will not be left over half the available disk /// space. A value over 100 will be reduced to 100. A value of 0 disables the - /// size-based pruning. - unsigned PercentageOfAvailableSpace = 75; + /// percentage size-based pruning. + unsigned MaxSizePercentageOfAvailableSpace = 75; + + /// The maximum size for the cache directory in bytes. A value over the amount + /// of available space on the disk will be reduced to the amount of available + /// space. A value of 0 disables the absolute size-based pruning. + uint64_t MaxSizeBytes = 0; }; /// Parse the given string as a cache pruning policy. Defaults are taken from a Index: llvm/lib/Support/CachePruning.cpp =================================================================== --- llvm/lib/Support/CachePruning.cpp +++ llvm/lib/Support/CachePruning.cpp @@ -79,10 +79,10 @@ return DurationOrErr.takeError(); Policy.Expiration = *DurationOrErr; } else if (Key == "cache_size") { - if (Value.back() != '%') - return make_error("'" + Value + "' must be a percentage", - inconvertibleErrorCode()); - StringRef SizeStr = Value.slice(0, Value.size() - 1); + if (Value.back() != '%') + return make_error("'" + Value + "' must be a percentage", + inconvertibleErrorCode()); + StringRef SizeStr = Value.drop_back(); uint64_t Size; if (SizeStr.getAsInteger(0, Size)) return make_error("'" + SizeStr + "' not an integer", @@ -91,7 +91,28 @@ return make_error("'" + SizeStr + "' must be between 0 and 100", inconvertibleErrorCode()); - Policy.PercentageOfAvailableSpace = Size; + Policy.MaxSizePercentageOfAvailableSpace = Size; + } else if (Key == "cache_size_bytes") { + uint64_t Mult = 1; + switch (Value.back()) { + case 'k': + Mult = 1024; + Value = Value.drop_back(); + break; + case 'm': + Mult = 1024 * 1024; + Value = Value.drop_back(); + break; + case 'g': + Mult = 1024 * 1024 * 1024; + Value = Value.drop_back(); + break; + } + uint64_t Size; + if (Value.getAsInteger(0, Size)) + return make_error("'" + Value + "' not an integer", + inconvertibleErrorCode()); + Policy.MaxSizeBytes = Size * Mult; } else { return make_error("Unknown key: '" + Key + "'", inconvertibleErrorCode()); @@ -115,11 +136,12 @@ if (!isPathDir) return false; - Policy.PercentageOfAvailableSpace = - std::min(Policy.PercentageOfAvailableSpace, 100u); + Policy.MaxSizePercentageOfAvailableSpace = + std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u); if (Policy.Expiration == seconds(0) && - Policy.PercentageOfAvailableSpace == 0) { + Policy.MaxSizePercentageOfAvailableSpace == 0 && + Policy.MaxSizeBytes == 0) { DEBUG(dbgs() << "No pruning settings set, exit early\n"); // Nothing will be pruned, early exit return false; @@ -157,7 +179,8 @@ writeTimestampFile(TimestampFile); } - bool ShouldComputeSize = (Policy.PercentageOfAvailableSpace > 0); + bool ShouldComputeSize = + (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0); // Keep track of space std::set> FileSizes; @@ -216,14 +239,22 @@ } sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get(); auto AvailableSpace = TotalSize + SpaceInfo.free; - auto FileAndSize = FileSizes.rbegin(); + + if (Policy.MaxSizePercentageOfAvailableSpace == 0) + Policy.MaxSizePercentageOfAvailableSpace = 100; + if (Policy.MaxSizeBytes == 0) + Policy.MaxSizeBytes = AvailableSpace; + auto TotalSizeTarget = std::min( + AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull, + Policy.MaxSizeBytes); + DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace) - << "% target is: " << Policy.PercentageOfAvailableSpace - << "\n"); + << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace + << "%, " << Policy.MaxSizeBytes << " bytes\n"); + + auto FileAndSize = FileSizes.rbegin(); // Remove the oldest accessed files first, till we get below the threshold - while (((100 * TotalSize) / AvailableSpace) > - Policy.PercentageOfAvailableSpace && - FileAndSize != FileSizes.rend()) { + while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) { // Remove the file. sys::fs::remove(FileAndSize->second); // Update size Index: llvm/unittests/Support/CachePruningTest.cpp =================================================================== --- llvm/unittests/Support/CachePruningTest.cpp +++ llvm/unittests/Support/CachePruningTest.cpp @@ -18,7 +18,7 @@ ASSERT_TRUE(bool(P)); EXPECT_EQ(std::chrono::seconds(1200), P->Interval); EXPECT_EQ(std::chrono::hours(7 * 24), P->Expiration); - EXPECT_EQ(75u, P->PercentageOfAvailableSpace); + EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace); } TEST(CachePruningPolicyParser, Interval) { @@ -39,10 +39,30 @@ EXPECT_EQ(std::chrono::seconds(1), P->Expiration); } -TEST(CachePruningPolicyParser, PercentageOfAvailableSpace) { +TEST(CachePruningPolicyParser, MaxSizePercentageOfAvailableSpace) { auto P = parseCachePruningPolicy("cache_size=100%"); ASSERT_TRUE(bool(P)); - EXPECT_EQ(100u, P->PercentageOfAvailableSpace); + EXPECT_EQ(100u, P->MaxSizePercentageOfAvailableSpace); + EXPECT_EQ(0u, P->MaxSizeBytes); +} + +TEST(CachePruningPolicyParser, MaxSizeBytes) { + auto P = parseCachePruningPolicy("cache_size_bytes=1"); + ASSERT_TRUE(bool(P)); + EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace); + EXPECT_EQ(1u, P->MaxSizeBytes); + P = parseCachePruningPolicy("cache_size_bytes=2k"); + ASSERT_TRUE(bool(P)); + EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace); + EXPECT_EQ(2u * 1024u, P->MaxSizeBytes); + P = parseCachePruningPolicy("cache_size_bytes=3m"); + ASSERT_TRUE(bool(P)); + EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace); + EXPECT_EQ(3u * 1024u * 1024u, P->MaxSizeBytes); + P = parseCachePruningPolicy("cache_size_bytes=4g"); + ASSERT_TRUE(bool(P)); + EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace); + EXPECT_EQ(4ull * 1024ull * 1024ull * 1024ull, P->MaxSizeBytes); } TEST(CachePruningPolicyParser, Multiple) { @@ -50,7 +70,7 @@ ASSERT_TRUE(bool(P)); EXPECT_EQ(std::chrono::seconds(1200), P->Interval); EXPECT_EQ(std::chrono::seconds(1), P->Expiration); - EXPECT_EQ(50u, P->PercentageOfAvailableSpace); + EXPECT_EQ(50u, P->MaxSizePercentageOfAvailableSpace); } TEST(CachePruningPolicyParser, Errors) { @@ -66,6 +86,12 @@ toString(parseCachePruningPolicy("cache_size=foo%").takeError())); EXPECT_EQ("'101' must be between 0 and 100", toString(parseCachePruningPolicy("cache_size=101%").takeError())); + EXPECT_EQ( + "'foo' not an integer", + toString(parseCachePruningPolicy("cache_size_bytes=foo").takeError())); + EXPECT_EQ( + "'foo' not an integer", + toString(parseCachePruningPolicy("cache_size_bytes=foom").takeError())); EXPECT_EQ("Unknown key: 'foo'", toString(parseCachePruningPolicy("foo=bar").takeError())); }