LTO: When creating a local cache, create the cache directory if it does not already exist.

[mehdi_amini]

LGTM.

lld/trunk/ELF/LTO.cpp

std::vector<InputFile *> BitcodeCompiler::compile() {
   Buff.resize(MaxTasks);
   Files.resize(MaxTasks);
 
   // The --thinlto-cache-dir option specifies the path to a directory in which
   // to cache native object files for ThinLTO incremental builds. If a path was
   // specified, configure LTO to use it as the cache directory.
   lto::NativeObjectCache Cache;
   if (!Config->ThinLTOCacheDir.empty())
-    Cache = lto::localCache(Config->ThinLTOCacheDir,
-                            [&](size_t Task, StringRef Path) {
-                              Files[Task] = check(MemoryBuffer::getFile(Path));
-                            });
+    Cache = check(lto::localCache(
+        Config->ThinLTOCacheDir, [&](size_t Task, StringRef Path) {
+        }));
 
   checkError(LTOObj->run(
       [&](size_t Task) {
         return llvm::make_unique<lto::NativeObjectStream>(
             llvm::make_unique<raw_svector_ostream>(Buff[Task]));
       },
       Cache));
 

lld/trunk/test/ELF/lto/cache.ll

 ; REQUIRES: x86
 
 ; RUN: opt -module-hash -module-summary %s -o %t.o
 ; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o
 
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache
 ; RUN: ld.lld --thinlto-cache-dir=%t.cache -o %t3.o %t2.o %t.o
 
 ; RUN: ls %t.cache | count 2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @globalfunc() #0 {
 entry:
   ret void
 }

llvm/trunk/include/llvm/LTO/Caching.h

 
 /// This type defines the callback to add a pre-existing native object file
 /// (e.g. in a cache).
 ///
 /// File callbacks must be thread safe.
 typedef std::function<void(unsigned Task, StringRef Path)> AddFileFn;
 
 /// Create a local file system cache which uses the given cache directory and
-/// file callback.
-NativeObjectCache localCache(StringRef CacheDirectoryPath, AddFileFn AddFile);
+/// file callback. This function also creates the cache directory if it does not
+/// already exist.
+Expected<NativeObjectCache> localCache(StringRef CacheDirectoryPath,
+                                       AddFileFn AddFile);
 
 } // namespace lto
 } // namespace llvm
 
 #endif

llvm/trunk/lib/LTO/Caching.cpp

// cache directory.
       // I'm not sure what are the guarantee if two processes are doing this
       // at the same time.
       OS << (*ReloadedBufferOrErr)->getBuffer();
     }
     sys::fs::remove(TempFilename);
   }
 }
 
-NativeObjectCache lto::localCache(StringRef CacheDirectoryPath,
+Expected<NativeObjectCache> lto::localCache(StringRef CacheDirectoryPath,
                                             AddFileFn AddFile) {
+  if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath))
+    return errorCodeToError(EC);
+
   return [=](unsigned Task, StringRef Key) -> AddStreamFn {
     // First, see if we have a cache hit.
     SmallString<64> EntryPath;
     sys::path::append(EntryPath, CacheDirectoryPath, Key);
     if (sys::fs::exists(EntryPath)) {
       AddFile(Task, EntryPath);
       return AddStreamFn();
     }

llvm/trunk/test/ThinLTO/X86/cache-config.ll

-; RUN: rm -rf %t.cache && mkdir %t.cache
+; RUN: rm -rf %t.cache
 ; RUN: opt -module-hash -module-summary %s -o %t.bc
 
 ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx
 ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -mcpu=yonah
 ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -relax-elf-relocations
 ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -function-sections
 ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -data-sections
 ; RUN: llvm-lto2 -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -debugger-tune=sce

llvm/trunk/test/ThinLTO/X86/cache.ll

 ; Verify first that *without* hash, we don't use the cache.
 
 ; RUN: opt -module-summary %s -o %t.bc
 ; RUN: opt -module-summary %p/Inputs/cache.ll -o %t2.bc
 
 ; Verify that enabling caching is ignoring module without hash
 ; RUN: rm -Rf %t.cache && mkdir %t.cache
 ; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc  %t.bc -thinlto-cache-dir %t.cache
 ; RUN: ls %t.cache/llvmcache.timestamp
 ; RUN: ls %t.cache | count 1
 
 ; Verify that enabling caching is ignoring module without hash with llvm-lto2
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache
 ; RUN: llvm-lto2 -o %t.o %t2.bc  %t.bc -cache-dir %t.cache \
 ; RUN:  -r=%t2.bc,_main,plx \
 ; RUN:  -r=%t2.bc,_globalfunc,lx \
 ; RUN:  -r=%t.bc,_globalfunc,plx
 ; RUN: ls %t.cache | count 0
 
 
 ; Repeat again, *with* hash this time.
 
 ; RUN: opt -module-hash -module-summary %s -o %t.bc
 ; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.bc
 
 ; Verify that enabling caching is working
 ; RUN: rm -Rf %t.cache && mkdir %t.cache
 ; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc  %t.bc -thinlto-cache-dir %t.cache
 ; RUN: ls %t.cache/llvmcache.timestamp
 ; RUN: ls %t.cache | count 3
 
 ; Verify that enabling caching is working with llvm-lto2
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache
 ; RUN: llvm-lto2 -o %t.o %t2.bc  %t.bc -cache-dir %t.cache \
 ; RUN:  -r=%t2.bc,_main,plx \
 ; RUN:  -r=%t2.bc,_globalfunc,lx \
 ; RUN:  -r=%t.bc,_globalfunc,plx
 ; RUN: ls %t.cache | count 2
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
 define void @globalfunc() #0 {
 entry:
   ret void
 }

llvm/trunk/test/ThinLTO/X86/empty_module_with_cache.ll

 ; RUN: opt -module-hash -module-summary %s -o %t.bc
 ; RUN: opt -module-hash -module-summary %p/Inputs/empty_module_with_cache.ll -o %t2.bc
 
 ; Verify that enabling caching is working, even if the module is empty
 ; RUN: rm -Rf %t.cache && mkdir %t.cache
 ; RUN: llvm-lto -thinlto-action=run %t2.bc  %t.bc -exported-symbol=main -thinlto-cache-dir %t.cache
 ; RUN: ls %t.cache/llvmcache.timestamp
 ; RUN: ls %t.cache | count 3
 
 ; Verify that enabling caching is working with llvm-lto2
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache
 ; RUN: llvm-lto2 -o %t.o %t2.bc  %t.bc -cache-dir %t.cache \
 ; RUN:  -r=%t2.bc,_main,plx
 ; RUN: ls %t.cache | count 2
 
 ; Same, but without hash, the index will be empty and caching should not happen
 
 ; RUN: opt -module-summary %s -o %t.bc
 ; RUN: opt -module-summary %p/Inputs/empty_module_with_cache.ll -o %t2.bc
 
 ; Verify that caching is disabled for module without hash
 ; RUN: rm -Rf %t.cache && mkdir %t.cache
 ; RUN: llvm-lto -thinlto-action=run %t2.bc  %t.bc -exported-symbol=main -thinlto-cache-dir %t.cache
 ; RUN: ls %t.cache/llvmcache.timestamp
 ; RUN: ls %t.cache | count 1
 
 ; Verify that caching is disabled for module without hash, with llvm-lto2
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache
 ; RUN: llvm-lto2 -o %t.o %t2.bc  %t.bc -cache-dir %t.cache \
 ; RUN:  -r=%t2.bc,_main,plx
 ; RUN: ls %t.cache | count 0
 
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"

llvm/trunk/test/tools/gold/X86/cache.ll

 ; Verify that enabling caching is ignoring module when we emit them without hash.
 ; RUN: opt -module-summary %s -o %t.o
 ; RUN: opt -module-summary %p/Inputs/cache.ll -o %t2.o
 
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold.so \
 ; RUN:     --plugin-opt=thinlto \
 ; RUN:     --plugin-opt=cache-dir=%t.cache \
 ; RUN:     -o %t3.o %t2.o %t.o
 
 ; RUN: ls %t.cache | count 0
 
 
 ; Verify that enabling caching is working with module with hash.
 
 ; RUN: opt -module-hash -module-summary %s -o %t.o
 ; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o
 
-; RUN: rm -Rf %t.cache && mkdir %t.cache
+; RUN: rm -Rf %t.cache
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold.so \
 ; RUN:     --plugin-opt=thinlto \
 ; RUN:     --plugin-opt=cache-dir=%t.cache \
 ; RUN:     -o %t3.o %t2.o %t.o
 
 ; RUN: ls %t.cache | count 2
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @globalfunc() #0 {
 entry:
   ret void
 }

llvm/trunk/tools/gold/gold-plugin.cpp

auto AddStream =
     return llvm::make_unique<lto::NativeObjectStream>(
         llvm::make_unique<llvm::raw_fd_ostream>(FD, true));
   };
 
   auto AddFile = [&](size_t Task, StringRef Path) { Filenames[Task] = Path; };
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-    Cache = localCache(options::cache_dir, AddFile);
+    Cache = check(localCache(options::cache_dir, AddFile));
 
   check(Lto->run(AddStream, Cache));
 
   if (options::TheOutputType == options::OT_DISABLE ||
       options::TheOutputType == options::OT_BC_ONLY)
     return LDPS_OK;
 
   if (options::thinlto_index_only) {

llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp

if (auto EC = ReloadedBufferOrErr.getError())
       report_fatal_error(Twine("Can't reload cached file '") + Path + "': " +
                          EC.message() + "\n");
 
     *AddStream(Task)->OS << (*ReloadedBufferOrErr)->getBuffer();
   };
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-    Cache = localCache(CacheDir, AddFile);
+    Cache = check(localCache(CacheDir, AddFile), "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
 }