Index: lld/COFF/LTO.cpp =================================================================== --- lld/COFF/LTO.cpp +++ lld/COFF/LTO.cpp @@ -202,7 +202,7 @@ } if (!config->ltoCache.empty()) - pruneCache(config->ltoCache, config->ltoCachePolicy); + pruneCache(config->ltoCache, config->ltoCachePolicy, files); std::vector ret; for (unsigned i = 0; i != maxTasks; ++i) { Index: lld/ELF/LTO.cpp =================================================================== --- lld/ELF/LTO.cpp +++ lld/ELF/LTO.cpp @@ -367,7 +367,7 @@ } if (!config->thinLTOCacheDir.empty()) - pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy); + pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files); if (!config->ltoObjPath.empty()) { saveBuffer(buf[0], config->ltoObjPath); Index: lld/MachO/LTO.cpp =================================================================== --- lld/MachO/LTO.cpp +++ lld/MachO/LTO.cpp @@ -141,7 +141,7 @@ cache)); if (!config->thinLTOCacheDir.empty()) - pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy); + pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files); // In ThinLTO mode, Clang passes a temporary directory in -object_path_lto, // while the argument is a single file in FullLTO mode. Index: lld/test/COFF/lto-cache-warnings.ll =================================================================== --- /dev/null +++ lld/test/COFF/lto-cache-warnings.ll @@ -0,0 +1,43 @@ +; REQUIRES: x86, shell + +; RUN: opt -module-hash -module-summary %s -o %t.o +; RUN: opt -module-hash -module-summary %p/Inputs/lto-cache.ll -o %t2.o + +; RUN: rm -rf %t && mkdir %t + +;; Check cache policies of the number of files. +;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning. +; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=0 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck --implicit-check-not=warning: %s +;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning. +; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=3 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --implicit-check-not=warning: +;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given. +; RUN: lld-link /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=1 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --check-prefix=NUM + +;; Check cache policies of the cache size. +;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning. +; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=0 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Get the total size of created cache files. +; RUN: rm -rf %t && mkdir %t && cd %t +; RUN: lld-link /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=32k /out:%t3 /entry:main %t2.o %t.o 2>&1 +; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt + +;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning. +; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given. +; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --check-prefix=SIZE + +;; Check emit two warnings if pruning happens due to reach both the size and number limits. +; RUN: lld-link /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=1:cache_size_bytes=1 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE + +; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy +; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy + +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +define void @globalfunc() #0 { +entry: + ret void +} Index: lld/test/ELF/lto/cache-warnings.ll =================================================================== --- /dev/null +++ lld/test/ELF/lto/cache-warnings.ll @@ -0,0 +1,44 @@ +; REQUIRES: x86, shell + +; RUN: opt -module-hash -module-summary %s -o %t.o +; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o + +; RUN: rm -rf %t && mkdir %t + +;; Check cache policies of the number of files. +;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning. +; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: +;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning. +; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=3 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: +;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given. +; RUN: ld.lld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,WARN + +;; Check cache policies of the cache size. +;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning. +; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Get the total size of created cache files. +; RUN: rm -rf %t && mkdir %t && cd %t +; RUN: ld.lld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=32k %t2.o %t.o -o %t3 2>&1 +; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt + +;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning. +; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given. +; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN + +;; Check emit two warnings if pruning happens due to reach both the size and number limits. +; RUN: ld.lld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN + +; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy +; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy +; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @globalfunc() { +entry: + ret void +} Index: lld/test/MachO/lto-cache-warnings.ll =================================================================== --- /dev/null +++ lld/test/MachO/lto-cache-warnings.ll @@ -0,0 +1,59 @@ +; REQUIRES: x86, shell + +; RUN: rm -rf %t; split-file %s %t +; RUN: opt -module-hash -module-summary %t/foo.ll -o %t/foo.o +; RUN: opt -module-hash -module-summary %t/bar.ll -o %t/bar.o + +;; Check cache policies of the number of files. +;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning. +; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=0 -o %t/test %t/foo.o %t/bar.o 2>&1 | FileCheck %s --implicit-check-not=warning: +;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning. +; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=3 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --implicit-check-not=warning: +;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given. +; RUN: %lld -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --check-prefixes=NUM,WARN + +;; Check cache policies of the cache size. +;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning. +; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=0 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Get the total size of created cache files. +; RUN: cd %t +; RUN: %lld -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=32k %t/foo.o %t/bar.o -o %t/test 2>&1 +; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt + +;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning. +; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given. +; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN + +;; Check emit two warnings if pruning happens due to reach both the size and number limits. +; RUN: %lld -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN + +; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy +; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy +; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy + +;--- foo.ll + +target triple = "x86_64-apple-macosx10.15.0" +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @globalfunc() #0 { +entry: + ret void +} + + +;--- bar.ll + +target triple = "x86_64-apple-macosx10.15.0" +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define i32 @main() { +entry: + call void (...) @globalfunc() + ret i32 0 +} + +declare void @globalfunc(...) Index: lld/test/wasm/lto/cache-warnings.ll =================================================================== --- /dev/null +++ lld/test/wasm/lto/cache-warnings.ll @@ -0,0 +1,44 @@ +; REQUIRES: shell + +; RUN: opt -module-hash -module-summary %s -o %t.o +; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o + +; RUN: rm -rf %t && mkdir %t + +;; Check cache policies of the number of files. +;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning. +; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: +;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning. +; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=3 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: +;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given. +; RUN: wasm-ld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,WARN + +;; Check cache policies of the cache size. +;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning. +; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Get the total size of created cache files. +; RUN: rm -rf %t && mkdir %t && cd %t +; RUN: wasm-ld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=32k %t2.o %t.o -o %t3 2>&1 +; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt + +;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning. +; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning: + +;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given. +; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN + +;; Check emit two warnings if pruning happens due to reach both the size and number limits. +; RUN: wasm-ld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN + +; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy +; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy +; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy + +target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128" +target triple = "wasm32-unknown-unknown-wasm" + +define void @globalfunc() #0 { +entry: + ret void +} Index: lld/wasm/LTO.cpp =================================================================== --- lld/wasm/LTO.cpp +++ lld/wasm/LTO.cpp @@ -142,7 +142,7 @@ cache)); if (!config->thinLTOCacheDir.empty()) - pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy); + pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files); std::vector ret; for (unsigned i = 0; i != maxTasks; ++i) { Index: llvm/include/llvm/Support/CachePruning.h =================================================================== --- llvm/include/llvm/Support/CachePruning.h +++ llvm/include/llvm/Support/CachePruning.h @@ -15,6 +15,7 @@ #define LLVM_SUPPORT_CACHEPRUNING_H #include "llvm/ADT/Optional.h" +#include "llvm/Support/MemoryBuffer.h" #include namespace llvm { @@ -70,11 +71,16 @@ /// Peform pruning using the supplied policy, returns true if pruning /// occurred, i.e. if Policy.Interval was expired. /// +/// Check whether cache pruning happens using the supplied policy, adds a +/// ThinLTO warning if cache_size_bytes or cache_size_files is too small for the +/// current link job. The warning recommends the user to consider adjusting +/// --thinlto-cache-policy. +/// /// As a safeguard against data loss if the user specifies the wrong directory /// as their cache directory, this function will ignore files not matching the /// pattern "llvmcache-*". -bool pruneCache(StringRef Path, CachePruningPolicy Policy); - +bool pruneCache(StringRef Path, CachePruningPolicy Policy, + const std::vector> &Files = {}); } // namespace llvm #endif Index: llvm/lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- llvm/lib/LTO/ThinLTOCodeGenerator.cpp +++ llvm/lib/LTO/ThinLTOCodeGenerator.cpp @@ -1217,7 +1217,7 @@ } } - pruneCache(CacheOptions.Path, CacheOptions.Policy); + pruneCache(CacheOptions.Path, CacheOptions.Policy, ProducedBinaries); // If statistics were requested, print them out now. if (llvm::AreStatisticsEnabled()) Index: llvm/lib/Support/CachePruning.cpp =================================================================== --- llvm/lib/Support/CachePruning.cpp +++ llvm/lib/Support/CachePruning.cpp @@ -17,6 +17,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" #define DEBUG_TYPE "cache-pruning" @@ -141,7 +142,8 @@ } /// Prune the cache of files that haven't been accessed in a long time. -bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) { +bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy, + const std::vector> &Files) { using namespace std::chrono; if (Path.empty()) @@ -258,6 +260,17 @@ ++FileInfo; }; + // files.size() is greater the number of inputs by one. However, a timestamp + // file is created and stored in the cache directory if --thinlto-cache-policy + // option is used. Therefore, files.size() is used as ActualNums. + const size_t ActualNums = Files.size(); + if (Policy.MaxSizeFiles && ActualNums > Policy.MaxSizeFiles) + WithColor::warning() + << "ThinLTO cache pruning happens since the number of created files (" + << ActualNums << ") exceeds the maximum number of files (" + << Policy.MaxSizeFiles + << "); consider adjusting --thinlto-cache-policy\n"; + // Prune for number of files. if (Policy.MaxSizeFiles) while (NumFiles > Policy.MaxSizeFiles) @@ -285,6 +298,19 @@ << Policy.MaxSizePercentageOfAvailableSpace << "%, " << Policy.MaxSizeBytes << " bytes\n"); + size_t ActualSizes = 0; + for (const auto &File : Files) + if (File) + ActualSizes += File->getBufferSize(); + + if (ActualSizes > TotalSizeTarget) + WithColor::warning() + << "ThinLTO cache pruning happens since the total size of the cache " + "files consumed by the current link job (" + << ActualSizes << " bytes) exceeds maximum cache size (" + << TotalSizeTarget + << " bytes); consider adjusting --thinlto-cache-policy\n"; + // Remove the oldest accessed files first, till we get below the threshold. while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end()) RemoveCacheFile();