[Driver] Fix rpath for compiler-rt

[yaxunl]

Summary

The compiler-rt library path can be either {resource_dir}/lib/{triple}
or {resource_dir}/lib/{OS}/{arch} depending on whether
LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default is ON or OFF.
Currently, the rpath added by -rtlib-add-rpath only adds
the latter. This patch checks both and adds the one that exists.

[MaskRay]

This change is correct for Linux. llvm/CMakeLists.txt says:

if(CMAKE_SYSTEM_NAME MATCHES "BSD|Linux|OS390")
  set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default ON)

Some rpath using OSes (notably macOS) use LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default=OFF. Is the rpath setting ever usable on macOS?
If not, the change is correct.

[yaxunl]

In D146686#4215577, @MaskRay wrote:

This change is correct for Linux. llvm/CMakeLists.txt says:

if(CMAKE_SYSTEM_NAME MATCHES "BSD|Linux|OS390")
  set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default ON)

Some rpath using OSes (notably macOS) use LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default=OFF. Is the rpath setting ever usable on macOS?
If not, the change is correct.

How about I try to add both? there is check whether the dir exist then adding it, so only the existing one will be added. then it should work in either case.

[MaskRay]

In D146686#4216612, @yaxunl wrote:

In D146686#4215577, @MaskRay wrote:

This change is correct for Linux. llvm/CMakeLists.txt says:

if(CMAKE_SYSTEM_NAME MATCHES "BSD|Linux|OS390")
  set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default ON)

Some rpath using OSes (notably macOS) use LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default=OFF. Is the rpath setting ever usable on macOS?
If not, the change is correct.

How about I try to add both? there is check whether the dir exist then adding it, so only the existing one will be added. then it should work in either case.

Yes. Detecting both locations will be nice.

[yaxunl]

adds check for both {resource_dir}/lib/{triple} and {resource_dir}/lib/{OS}/{arch}

[MaskRay]

The new test can be placed in arch-specific-libdir-rpath.c. One or two additional RUN lines seem sufficient, no need to duplicate too many.

Also, use --target= for new tests and avoid ^//$.

clang/lib/Driver/ToolChains/OHOS.cpp

142–144

delete braces

[yaxunl]

revised by Fangrui's comments

[MaskRay]

clang/lib/Driver/ToolChain.cpp

638

unneeded blank line

[yaxunl]

clang/lib/Driver/ToolChain.cpp

638

will remove when committing

clang/include/clang/Driver/ToolChain.h

std::string getCompilerRTBasename(const llvm::opt::ArgList &Args,
                                     FileType Type = ToolChain::FT_Static) const;
 
   // Returns target specific runtime paths.
   path_list getRuntimePaths() const;
 
   // Returns target specific standard library paths.
   path_list getStdlibPaths() const;
 
-  // Returns <ResourceDir>/lib/<OSName>/<arch>.  This is used by runtimes (such
-  // as OpenMP) to find arch-specific libraries.
-  virtual std::string getArchSpecificLibPath() const;
+  // Returns <ResourceDir>/lib/<OSName>/<arch> or <ResourceDir>/lib/<triple>.
+  // This is used by runtimes (such as OpenMP) to find arch-specific libraries.
+  virtual path_list getArchSpecificLibPaths() const;
 
   // Returns <OSname> part of above.
   virtual StringRef getOSLibName() const;
 
   /// needsProfileRT - returns true if instrumentation profile is on.
   static bool needsProfileRT(const llvm::opt::ArgList &Args);
 
   /// Returns true if gcov instrumentation (-fprofile-arcs or --coverage) is on.

clang/lib/Driver/ToolChain.cpp

auto addIfExists = [this](path_list &List, const std::string &Path) {
     if (getVFS().exists(Path))
       List.push_back(Path);
   };
 
   for (const auto &Path : getRuntimePaths())
     addIfExists(getLibraryPaths(), Path);
   for (const auto &Path : getStdlibPaths())
     addIfExists(getFilePaths(), Path);
-  addIfExists(getFilePaths(), getArchSpecificLibPath());
+  for (const auto &Path : getArchSpecificLibPaths())
+    addIfExists(getFilePaths(), Path);
 }
 
 llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
 ToolChain::executeToolChainProgram(StringRef Executable) const {
   llvm::SmallString<64> OutputFile;
   llvm::sys::fs::createTemporaryFile("toolchain-program", "txt", OutputFile);
   llvm::FileRemover OutputRemover(OutputFile.c_str());
   std::optional<llvm::StringRef> Redirects[] = {
ToolChain::path_list ToolChain::getStdlibPaths() const {
   path_list Paths;
   SmallString<128> P(D.Dir);
   llvm::sys::path::append(P, "..", "lib", getTripleString());
   Paths.push_back(std::string(P.str()));
 
   return Paths;
 }
 
-std::string ToolChain::getArchSpecificLibPath() const {
+ToolChain::path_list ToolChain::getArchSpecificLibPaths() const {
+  path_list Paths;
+
+  auto AddPath = [&](const ArrayRef<StringRef> &SS) {
     SmallString<128> Path(getDriver().ResourceDir);
-  llvm::sys::path::append(Path, "lib", getOSLibName(),
-                          llvm::Triple::getArchTypeName(getArch()));
-  return std::string(Path.str());
+    llvm::sys::path::append(Path, "lib");
+    for (auto &S : SS)
+      llvm::sys::path::append(Path, S);
+    Paths.push_back(std::string(Path.str()));
+  };
+
+  AddPath({getTriple().str()});
+  AddPath({getOSLibName(), llvm::Triple::getArchTypeName(getArch())});
+  return Paths;
 }
 
 bool ToolChain::needsProfileRT(const ArgList &Args) {
   if (Args.hasArg(options::OPT_noprofilelib))
     return false;
 
   return Args.hasArg(options::OPT_fprofile_generate) ||
          Args.hasArg(options::OPT_fprofile_generate_EQ) ||

clang/lib/Driver/ToolChains/CommonArgs.cpp

void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args,
                                  ArgStringList &CmdArgs) {
   // Enable -frtlib-add-rpath by default for the case of VE.
   const bool IsVE = TC.getTriple().isVE();
   bool DefaultValue = IsVE;
   if (!Args.hasFlag(options::OPT_frtlib_add_rpath,
                     options::OPT_fno_rtlib_add_rpath, DefaultValue))
     return;
 
-  std::string CandidateRPath = TC.getArchSpecificLibPath();
+  for (const auto &CandidateRPath : TC.getArchSpecificLibPaths()) {
     if (TC.getVFS().exists(CandidateRPath)) {
       CmdArgs.push_back("-rpath");
       CmdArgs.push_back(Args.MakeArgString(CandidateRPath));
     }
   }
+}
 
 bool tools::addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC,
                              const ArgList &Args, bool ForceStaticHostRuntime,
                              bool IsOffloadingHost, bool GompNeedsRT) {
   if (!Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                     options::OPT_fno_openmp, false))
     return false;
 

clang/lib/Driver/ToolChains/OHOS.h

protected:
   std::string getMultiarchTriple(const llvm::Triple &T) const;
   std::string getMultiarchTriple(const Driver &D,
                                  const llvm::Triple &TargetTriple,
                                  StringRef SysRoot) const override;
   void addExtraOpts(llvm::opt::ArgStringList &CmdArgs) const override;
   SanitizerMask getSupportedSanitizers() const override;
   void addProfileRTLibs(const llvm::opt::ArgList &Args,
                              llvm::opt::ArgStringList &CmdArgs) const override;
-  std::string getArchSpecificLibPath() const override;
+  path_list getArchSpecificLibPaths() const override;
+
 private:
   Multilib SelectedMultilib;
 };
 
 } // end namespace toolchains
 } // end namespace driver
 } // end namespace clang
 
 #endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_OHOS_H

clang/lib/Driver/ToolChains/OHOS.cpp

OHOS::OHOS(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
 
   // Select the correct multilib according to the given arguments.
   DetectedMultilibs Result;
   findOHOSMultilibs(D, *this, Triple, "", Args, Result);
   Multilibs = Result.Multilibs;
   SelectedMultilib = Result.SelectedMultilib;
 
   getFilePaths().clear();
-  std::string CandidateLibPath = getArchSpecificLibPath();
+  for (const auto &CandidateLibPath : getArchSpecificLibPaths())
     if (getVFS().exists(CandidateLibPath))
       getFilePaths().push_back(CandidateLibPath);
 
   getLibraryPaths().clear();
   for (auto &Path : getRuntimePaths())
     if (getVFS().exists(Path))
       getLibraryPaths().push_back(Path);
 
   // OHOS sysroots contain a library directory for each supported OS
   // version as well as some unversioned libraries in the usual multiarch
void OHOS::addProfileRTLibs(const llvm::opt::ArgList &Args,
   // Add linker option -u__llvm_profile_runtime to cause runtime
   // initialization module to be linked in.
   if (needsProfileRT(Args))
     CmdArgs.push_back(Args.MakeArgString(
         Twine("-u", llvm::getInstrProfRuntimeHookVarName())));
   ToolChain::addProfileRTLibs(Args, CmdArgs);
 }
 
-std::string OHOS::getArchSpecificLibPath() const {
+ToolChain::path_list OHOS::getArchSpecificLibPaths() const {
+  ToolChain::path_list Paths;
   llvm::Triple Triple = getTriple();
-  return makePath({getDriver().ResourceDir, "lib", getMultiarchTriple(Triple)});
+  Paths.push_back(
+      makePath({getDriver().ResourceDir, "lib", getMultiarchTriple(Triple)}));
+  return Paths;
 }
 
 ToolChain::UnwindLibType OHOS::GetUnwindLibType(const llvm::opt::ArgList &Args) const {
   if (Args.getLastArg(options::OPT_unwindlib_EQ))
     return Generic_ELF::GetUnwindLibType(Args);
   return GetDefaultUnwindLibType();
 }

clang/lib/Driver/ToolChains/VEToolchain.cpp

VEToolChain::VEToolChain(const Driver &D, const llvm::Triple &Triple,
   getProgramPaths().push_back("/opt/nec/ve/bin");
   // ProgramPaths are found via 'PATH' environment variable.
 
   // Default library paths are following:
   //   ${RESOURCEDIR}/lib/ve-unknown-linux-gnu,
   // These are OK.
 
   // Default file paths are following:
-  //   ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPath)
+  //   ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPaths)
   //   /lib/../lib64,
   //   /usr/lib/../lib64,
   //   ${BINPATH}/../lib,
   //   /lib,
   //   /usr/lib,
   // These are OK for host, but no go for VE.
 
   // Define file paths from scratch here.
   getFilePaths().clear();
 
   // Add library directories:
-  //   ${BINPATH}/../lib/ve-unknown-linux-gnu, (== getStdlibPath)
-  //   ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPath)
+  //   ${BINPATH}/../lib/ve-unknown-linux-gnu, (== getStdlibPaths)
+  //   ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPaths)
   //   ${SYSROOT}/opt/nec/ve/lib,
   for (auto &Path : getStdlibPaths())
     getFilePaths().push_back(std::move(Path));
-  getFilePaths().push_back(getArchSpecificLibPath());
+  for (const auto &Path : getArchSpecificLibPaths())
+    getFilePaths().push_back(Path);
   getFilePaths().push_back(computeSysRoot() + "/opt/nec/ve/lib");
 }
 
 Tool *VEToolChain::buildAssembler() const {
   return new tools::gnutools::Assembler(*this);
 }
 
 Tool *VEToolChain::buildLinker() const {

clang/test/Driver/arch-specific-libdir-rpath.c

 // RUN:   | FileCheck --check-prefixes=RESDIR,LIBPATH-X86_64,NO-RPATH-X86_64 %s
 //
 // Do not add LIBPATH or RPATH if arch-specific subdir doesn't exist
 // RUN: %clang %s -### --target=x86_64-linux \
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     -frtlib-add-rpath 2>&1 \
 // RUN:   | FileCheck --check-prefixes=RESDIR,NO-LIBPATH,NO-RPATH %s
 
+// Test that the driver adds an per-target arch-specific subdirectory in
+// {RESOURCE_DIR}/lib/{triple} to the linker search path and to '-rpath'
+//
+// RUN: %clang %s -### 2>&1 --target=x86_64-linux-gnu \
+// RUN:     -fsanitize=address -shared-libasan \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     -frtlib-add-rpath \
+// RUN:   | FileCheck --check-prefixes=PERTARGET %s
+
 // RESDIR: "-resource-dir" "[[RESDIR:[^"]*]]"
 //
 // LIBPATH-X86_64: -L[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)x86_64}}
 // RPATH-X86_64:   "-rpath" "[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)x86_64}}"
 //
 // NO-LIBPATH-X86_64-NOT: -L[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)x86_64}}
 // NO-RPATH-X86_64-NOT:   "-rpath" "[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)x86_64}}"
 //
 // LIBPATH-AARCH64: -L[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)aarch64}}
 // RPATH-AARCH64:   "-rpath" "[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)aarch64}}"
 //
 // NO-LIBPATH-NOT: "-L{{[^"]*Inputs(/|\\\\)resource_dir}}"
 // NO-RPATH-NOT:   "-rpath" {{.*(/|\\\\)Inputs(/|\\\\)resource_dir}}
+
+// PERTARGET: "-resource-dir" "[[PTRESDIR:[^"]*]]"
+// PERTARGET: -L[[PTRESDIR]]{{(/|\\\\)lib(/|\\\\)x86_64-unknown-linux-gnu}}
+// PERTARGET:   "-rpath" "[[PTRESDIR]]{{(/|\\\\)lib(/|\\\\)x86_64-unknown-linux-gnu}}"