Index: lld/test/ELF/lto/cache.ll =================================================================== --- lld/test/ELF/lto/cache.ll +++ lld/test/ELF/lto/cache.ll @@ -4,13 +4,25 @@ ; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o ; RUN: rm -Rf %t.cache && mkdir %t.cache +; RUN: touch %t.cache/llvmcache.timestamp ; Create a file that will be removed by cache pruning due to age. ; RUN: touch -t 197001010000 %t.cache/foo ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=1h -o %t3 %t2.o %t.o -; Two cached objects, plus a timestamp file, minus the file we removed. +; RUN: ls %t.cache/llvmcache.timestamp +; RUN: not ls %t.cache/foo ; RUN: ls %t.cache | count 3 +; Similar case, except that the timestamp file does not exist, so we should not prune. + +; RUN: rm -Rf %t.cache2 && mkdir %t.cache2 +; RUN: touch -t 197001010000 %t.cache2/foo +; RUN: ld.lld --thinlto-cache-dir=%t.cache2 --thinlto-cache-policy prune_after=1h -o %t3 %t2.o %t.o + +; RUN: not ls %t.cache2/llvmcache.timestamp +; RUN: ls %t.cache2/foo +; RUN: ls %t.cache2 | count 3 + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/include/llvm/Support/CachePruning.h =================================================================== --- llvm/include/llvm/Support/CachePruning.h +++ llvm/include/llvm/Support/CachePruning.h @@ -20,6 +20,7 @@ namespace llvm { +class Error; template class Expected; /// Policy for the pruneCache() function. A default constructed @@ -50,6 +51,16 @@ /// and maximum cache size of 50% of available disk space. Expected parseCachePruningPolicy(StringRef PolicyStr); +/// Write a new timestamp file to the cache directory with the given path. This +/// is used to mark a directory as a cache directory and for the pruning +/// interval option. +void writeCacheTimestampFile(StringRef Path); + +/// Create a cache directory, if it does not already exist. If we created the +/// cache directory, also create a timestamp file to mark the directory as a +/// cache directory. +Error initCache(StringRef Path); + /// Peform pruning using the supplied policy, returns true if pruning /// occured, i.e. if Policy.Interval was expired. bool pruneCache(StringRef Path, CachePruningPolicy Policy); Index: llvm/lib/LTO/Caching.cpp =================================================================== --- llvm/lib/LTO/Caching.cpp +++ llvm/lib/LTO/Caching.cpp @@ -13,6 +13,7 @@ #include "llvm/LTO/Caching.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/Support/CachePruning.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" @@ -23,8 +24,8 @@ Expected lto::localCache(StringRef CacheDirectoryPath, AddBufferFn AddBuffer) { - if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath)) - return errorCodeToError(EC); + if (auto Err = initCache(CacheDirectoryPath)) + return std::move(Err); return [=](unsigned Task, StringRef Key) -> AddStreamFn { // First, see if we have a cache hit. Index: llvm/lib/LTO/ThinLTOCodeGenerator.cpp =================================================================== --- llvm/lib/LTO/ThinLTOCodeGenerator.cpp +++ llvm/lib/LTO/ThinLTOCodeGenerator.cpp @@ -822,6 +822,11 @@ ProducedBinaryFiles.resize(Modules.size()); } + // The client is responsible for creating the cache directory, so we need to + // create the timestamp file ourselves. + if (!CacheOptions.Path.empty()) + writeCacheTimestampFile(CacheOptions.Path); + if (CodeGenOnly) { // Perform only parallel codegen and return. ThreadPool Pool; Index: llvm/lib/Support/CachePruning.cpp =================================================================== --- llvm/lib/Support/CachePruning.cpp +++ llvm/lib/Support/CachePruning.cpp @@ -27,9 +27,9 @@ using namespace llvm; -/// Write a new timestamp file with the given path. This is used for the pruning -/// interval option. -static void writeTimestampFile(StringRef TimestampFile) { +void llvm::writeCacheTimestampFile(StringRef Path) { + SmallString<128> TimestampFile(Path); + sys::path::append(TimestampFile, "llvmcache.timestamp"); std::error_code EC; raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None); } @@ -101,6 +101,19 @@ return Policy; } +Error llvm::initCache(StringRef Path) { + auto EC = sys::fs::create_directories(Path, /*IgnoreExisting=*/false); + if (EC == errc::file_exists) + return Error::success(); + + // The process that actually created the directory is responsible for writing + // the timestamp file. + if (!EC) + writeCacheTimestampFile(Path); + + return errorCodeToError(EC); +} + /// Prune the cache of files that haven't been accessed in a long time. bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) { using namespace std::chrono; @@ -108,13 +121,6 @@ if (Path.empty()) return false; - bool isPathDir; - if (sys::fs::is_directory(Path, isPathDir)) - return false; - - if (!isPathDir) - return false; - Policy.PercentageOfAvailableSpace = std::min(Policy.PercentageOfAvailableSpace, 100u); @@ -125,37 +131,32 @@ return false; } - // Try to stat() the timestamp file. + // Try to stat() the timestamp file. If it does not exist we refuse to prune + // the cache. This is a safeguard against pruning directories that are not + // cache directories. SmallString<128> TimestampFile(Path); sys::path::append(TimestampFile, "llvmcache.timestamp"); sys::fs::file_status FileStatus; const auto CurrentTime = system_clock::now(); - if (auto EC = sys::fs::status(TimestampFile, FileStatus)) { - if (EC == errc::no_such_file_or_directory) { - // If the timestamp file wasn't there, create one now. - writeTimestampFile(TimestampFile); - } else { - // Unknown error? + if (sys::fs::status(TimestampFile, FileStatus)) + return false; + + if (Policy.Interval == seconds(0)) { + // Check whether the time stamp is older than our pruning interval. + // If not, do nothing. + const auto TimeStampModTime = FileStatus.getLastModificationTime(); + auto TimeStampAge = CurrentTime - TimeStampModTime; + if (TimeStampAge <= Policy.Interval) { + DEBUG(dbgs() << "Timestamp file too recent (" + << duration_cast(TimeStampAge).count() + << "s old), do not prune.\n"); return false; } - } else { - if (Policy.Interval == seconds(0)) { - // Check whether the time stamp is older than our pruning interval. - // If not, do nothing. - const auto TimeStampModTime = FileStatus.getLastModificationTime(); - auto TimeStampAge = CurrentTime - TimeStampModTime; - if (TimeStampAge <= Policy.Interval) { - DEBUG(dbgs() << "Timestamp file too recent (" - << duration_cast(TimeStampAge).count() - << "s old), do not prune.\n"); - return false; - } - } - // Write a new timestamp file so that nobody else attempts to prune. - // There is a benign race condition here, if two processes happen to - // notice at the same time that the timestamp is out-of-date. - writeTimestampFile(TimestampFile); } + // Write a new timestamp file so that nobody else attempts to prune. + // There is a benign race condition here, if two processes happen to + // notice at the same time that the timestamp is out-of-date. + writeCacheTimestampFile(Path); bool ShouldComputeSize = (Policy.PercentageOfAvailableSpace > 0); Index: llvm/lib/Support/Path.cpp =================================================================== --- llvm/lib/Support/Path.cpp +++ llvm/lib/Support/Path.cpp @@ -884,7 +884,10 @@ if (Parent.empty()) return EC; - if ((EC = create_directories(Parent, IgnoreExisting, Perms))) + // Only return an error if the specified directory does not exist, as opposed + // to a parent directory (inevitably some parent must exist, so without this + // we would always return an error if IgnoreExisting was false). + if ((EC = create_directories(Parent, /*IgnoreExisting=*/true, Perms))) return EC; return create_directory(P, IgnoreExisting, Perms); Index: llvm/test/ThinLTO/X86/cache-config.ll =================================================================== --- llvm/test/ThinLTO/X86/cache-config.ll +++ llvm/test/ThinLTO/X86/cache-config.ll @@ -16,7 +16,7 @@ ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -aa-pipeline=basic-aa ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -override-triple=x86_64-unknown-linux-gnu ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -default-triple=x86_64-unknown-linux-gnu -; RUN: ls %t.cache | count 15 +; RUN: ls %t.cache | count 16 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/test/ThinLTO/X86/cache-import-lists.ll =================================================================== --- llvm/test/ThinLTO/X86/cache-import-lists.ll +++ llvm/test/ThinLTO/X86/cache-import-lists.ll @@ -9,7 +9,7 @@ ; RUN: rm -rf %t.cache ; RUN: llvm-lto2 -cache-dir %t.cache -o %t.o %t.bc %t1.bc %t2.bc -r=%t.bc,main,plx -r=%t.bc,f1,lx -r=%t.bc,f2,lx -r=%t1.bc,f1,plx -r=%t1.bc,linkonce_odr,plx -r=%t2.bc,f2,plx -r=%t2.bc,linkonce_odr,lx ; RUN: llvm-lto2 -cache-dir %t.cache -o %t.o %t.bc %t2.bc %t1.bc -r=%t.bc,main,plx -r=%t.bc,f1,lx -r=%t.bc,f2,lx -r=%t2.bc,f2,plx -r=%t2.bc,linkonce_odr,plx -r=%t1.bc,f1,plx -r=%t1.bc,linkonce_odr,lx -; RUN: ls %t.cache | count 6 +; RUN: ls %t.cache | count 7 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/test/ThinLTO/X86/cache-typeid-resolutions.ll =================================================================== --- llvm/test/ThinLTO/X86/cache-typeid-resolutions.ll +++ llvm/test/ThinLTO/X86/cache-typeid-resolutions.ll @@ -7,20 +7,20 @@ ; Two resolutions for typeid1: Unsat, Single ; where both t and t-import are sensitive to typeid1's resolution -; so 4 distinct objects in total. +; so 4 distinct objects in total, plus the timestamp file. ; RUN: rm -rf %t.cache ; RUN: llvm-lto2 -o %t.o %t.bc %t-import.bc -cache-dir %t.cache -r=%t.bc,f1,plx -r=%t.bc,f2,plx -r=%t-import.bc,importf1,plx -r=%t-import.bc,f1,lx -r=%t-import.bc,importf2,plx -r=%t-import.bc,f2,lx ; RUN: llvm-lto2 -o %t.o %t.bc %t-import.bc %t1.bc -cache-dir %t.cache -r=%t.bc,f1,plx -r=%t.bc,f2,plx -r=%t-import.bc,importf1,plx -r=%t-import.bc,f1,lx -r=%t-import.bc,importf2,plx -r=%t-import.bc,f2,lx -r=%t1.bc,vt1,plx -; RUN: ls %t.cache | count 4 +; RUN: ls %t.cache | count 5 ; Three resolutions for typeid2: Indir, SingleImpl, UniqueRetVal ; where both t and t-import are sensitive to typeid2's resolution -; so 6 distinct objects in total. +; so 6 distinct objects in total, plus the timestamp file. ; RUN: rm -rf %t.cache ; RUN: llvm-lto2 -o %t.o %t.bc %t-import.bc -cache-dir %t.cache -r=%t.bc,f1,plx -r=%t.bc,f2,plx -r=%t-import.bc,importf1,plx -r=%t-import.bc,f1,lx -r=%t-import.bc,importf2,plx -r=%t-import.bc,f2,lx ; RUN: llvm-lto2 -o %t.o %t.bc %t-import.bc %t2.bc -cache-dir %t.cache -r=%t.bc,f1,plx -r=%t.bc,f2,plx -r=%t2.bc,vt2,plx -r=%t-import.bc,importf1,plx -r=%t-import.bc,f1,lx -r=%t-import.bc,importf2,plx -r=%t-import.bc,f2,lx ; RUN: llvm-lto2 -o %t.o %t.bc %t-import.bc %t3.bc -cache-dir %t.cache -r=%t.bc,f1,plx -r=%t.bc,f2,plx -r=%t3.bc,vt2a,plx -r=%t3.bc,vt2b,plx -r=%t-import.bc,importf1,plx -r=%t-import.bc,f1,lx -r=%t-import.bc,importf2,plx -r=%t-import.bc,f2,lx -; RUN: ls %t.cache | count 6 +; RUN: ls %t.cache | count 7 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/test/ThinLTO/X86/cache.ll =================================================================== --- llvm/test/ThinLTO/X86/cache.ll +++ llvm/test/ThinLTO/X86/cache.ll @@ -15,7 +15,8 @@ ; RUN: -r=%t2.bc,_main,plx \ ; RUN: -r=%t2.bc,_globalfunc,lx \ ; RUN: -r=%t.bc,_globalfunc,plx -; RUN: ls %t.cache | count 0 +; RUN: ls %t.cache/llvmcache.timestamp +; RUN: ls %t.cache | count 1 ; Repeat again, *with* hash this time. @@ -35,7 +36,8 @@ ; RUN: -r=%t2.bc,_main,plx \ ; RUN: -r=%t2.bc,_globalfunc,lx \ ; RUN: -r=%t.bc,_globalfunc,plx -; RUN: ls %t.cache | count 2 +; RUN: ls %t.cache/llvmcache.timestamp +; RUN: ls %t.cache | count 3 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.11.0" Index: llvm/test/ThinLTO/X86/empty_module_with_cache.ll =================================================================== --- llvm/test/ThinLTO/X86/empty_module_with_cache.ll +++ llvm/test/ThinLTO/X86/empty_module_with_cache.ll @@ -11,7 +11,8 @@ ; RUN: rm -Rf %t.cache ; RUN: llvm-lto2 -o %t.o %t2.bc %t.bc -cache-dir %t.cache \ ; RUN: -r=%t2.bc,_main,plx -; RUN: ls %t.cache | count 2 +; RUN: ls %t.cache/llvmcache.timestamp +; RUN: ls %t.cache | count 3 ; Same, but without hash, the index will be empty and caching should not happen @@ -28,7 +29,8 @@ ; RUN: rm -Rf %t.cache ; RUN: llvm-lto2 -o %t.o %t2.bc %t.bc -cache-dir %t.cache \ ; RUN: -r=%t2.bc,_main,plx -; RUN: ls %t.cache | count 0 +; RUN: ls %t.cache/llvmcache.timestamp +; RUN: ls %t.cache | count 1 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" Index: llvm/test/tools/gold/X86/cache.ll =================================================================== --- llvm/test/tools/gold/X86/cache.ll +++ llvm/test/tools/gold/X86/cache.ll @@ -8,7 +8,8 @@ ; RUN: --plugin-opt=cache-dir=%t.cache \ ; RUN: -o %t3.o %t2.o %t.o -; RUN: ls %t.cache | count 0 +; RUN: ls %t.cache/llvmcache.timestamp +; RUN: ls %t.cache | count 1 ; Verify that enabling caching is working with module with hash. @@ -22,7 +23,8 @@ ; RUN: --plugin-opt=cache-dir=%t.cache \ ; RUN: -o %t3.o %t2.o %t.o -; RUN: ls %t.cache | count 2 +; RUN: ls %t.cache/llvmcache.timestamp +; RUN: ls %t.cache | count 3 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"