[CUDA][HIP] Add support for `--offload-arch=native` to CUDA and refactor

[jhuber6]

Summary

This patch adds basic support for --offload-arch=native to CUDA. This
is done using the nvptx-arch tool that was introduced previously. Some
of the logic for handling executing these tools was factored into a
common helper as well. This patch does not add support for OpenMP or the
"new" driver. That will be done later.

[jhuber6]

Change error to print canonical arch name from Triple.

[yaxunl]

clang/test/Driver/amdgpu-hip-system-arch.c

25

comment incorrect?

[jhuber6]

clang/test/Driver/amdgpu-hip-system-arch.c

25

Yes, thanks for catching that. I'll fix it.

[jhuber6]

Fix typo.

[yaxunl]

LGTM. Thanks.

[srj]

For reasons that aren't yet clear to me, this change is failing to compile when using gcc-7 and targeting 32-bit targets; the error is of the form

AMDGPU.cpp:773:10: error: could not convert ‘GPUArchs’ from ‘llvm::SmallVector<std::__cxx11::basic_string<char>, 1>’ to ‘llvm::Expected<llvm::SmallVector<std::__cxx11::basic_string<char> > >’
   return GPUArchs;

it's not (yet) clear to me whether this is specific to gcc-7 (which I realize is fairly old -- is it still supported?) or what -- investigating further.

EDIT: I am (so far) unable to repro the failure using the gcc-12 installed on my local Linux machine, so it may be specific to that compiler version -- I am going to try to install gcc-7 on my local machine to see if there's a reasonable workaround.

[jhuber6]

In D141051#4048456, @srj wrote:

For reasons that aren't yet clear to me, this change is failing to compile when using gcc-7 and targeting 32-bit targets; the error is of the form

AMDGPU.cpp:773:10: error: could not convert ‘GPUArchs’ from ‘llvm::SmallVector<std::__cxx11::basic_string<char>, 1>’ to ‘llvm::Expected<llvm::SmallVector<std::__cxx11::basic_string<char> > >’
   return GPUArchs;

Probably the older GCC doesn't handle the implicit copy elision to the expected type well and thinks that it's copied. I'll put an explicit move on it.

[jhuber6]

In D141051#4048456, @srj wrote:

For reasons that aren't yet clear to me, this change is failing to compile when using gcc-7 and targeting 32-bit targets; the error is of the form

AMDGPU.cpp:773:10: error: could not convert ‘GPUArchs’ from ‘llvm::SmallVector<std::__cxx11::basic_string<char>, 1>’ to ‘llvm::Expected<llvm::SmallVector<std::__cxx11::basic_string<char> > >’
   return GPUArchs;

it's not (yet) clear to me whether this is specific to gcc-7 (which I realize is fairly old -- is it still supported?) or what -- investigating further.

Let me know if rG26d62674cf50 solved it. I can't reproduce it on my system.

clang/include/clang/Basic/DiagnosticDriverKinds.td

 def err_drv_no_hipspv_device_lib : Error<
   "cannot find HIP device library%select{| for %1}0; provide its path via "
   "'--hip-path' or '--hip-device-lib-path', or pass '-nogpulib' to build "
   "without HIP device library">;
 def err_drv_hipspv_no_hip_path : Error<
   "'--hip-path' must be specified when offloading to "
   "SPIR-V%select{| unless %1 is given}0.">;
 
-def err_drv_undetermined_amdgpu_arch : Error<
-  "cannot determine AMDGPU architecture: %0; consider passing it via "
-  "'--march'">;
+def err_drv_undetermined_gpu_arch : Error<
+  "cannot determine %0 architecture: %1; consider passing it via "
+  "'%2'">;
 def err_drv_cuda_version_unsupported : Error<
   "GPU arch %0 is supported by CUDA versions between %1 and %2 (inclusive), "
   "but installation at %3 is %4; use '--cuda-path' to specify a different CUDA "
   "install, pass a different GPU arch with '--cuda-gpu-arch', or pass "
   "'--no-cuda-version-check'">;
 def warn_drv_new_cuda_version: Warning<
   "CUDA version%0 is newer than the latest%select{| partially}1 supported version %2">,
   InGroup<CudaUnknownVersion>;

clang/include/clang/Driver/Options.td

def fgpu_default_stream_EQ : Joined<["-"], "fgpu-default-stream=">,
   NormalizedValues<["Legacy", "PerThread"]>,
   MarshallingInfoEnum<LangOpts<"GPUDefaultStream">, "Legacy">;
 def rocm_path_EQ : Joined<["--"], "rocm-path=">, Group<i_Group>,
   HelpText<"ROCm installation path, used for finding and automatically linking required bitcode libraries.">;
 def hip_path_EQ : Joined<["--"], "hip-path=">, Group<i_Group>,
   HelpText<"HIP runtime installation path, used for finding HIP version and adding HIP include path.">;
 def amdgpu_arch_tool_EQ : Joined<["--"], "amdgpu-arch-tool=">, Group<i_Group>,
   HelpText<"Tool used for detecting AMD GPU arch in the system.">;
+def nvptx_arch_tool_EQ : Joined<["--"], "nvptx-arch-tool=">, Group<i_Group>,
+  HelpText<"Tool used for detecting NVIDIA GPU arch in the system.">;
 def rocm_device_lib_path_EQ : Joined<["--"], "rocm-device-lib-path=">, Group<Link_Group>,
   HelpText<"ROCm device library path. Alternative to rocm-path.">;
 def : Joined<["--"], "hip-device-lib-path=">, Alias<rocm_device_lib_path_EQ>;
 def hip_device_lib_EQ : Joined<["--"], "hip-device-lib=">, Group<Link_Group>,
   HelpText<"HIP device library">;
 def hip_version_EQ : Joined<["--"], "hip-version=">,
   HelpText<"HIP version in the format of major.minor.patch">;
 def fhip_dump_offload_linker_script : Flag<["-"], "fhip-dump-offload-linker-script">,

clang/include/clang/Driver/ToolChain.h

 
 protected:
   MultilibSet Multilibs;
   Multilib SelectedMultilib;
 
   ToolChain(const Driver &D, const llvm::Triple &T,
             const llvm::opt::ArgList &Args);
 
+  /// Executes the given \p Executable and returns the stdout.
+  llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
+  executeToolChainProgram(StringRef Executable) const;
+
   void setTripleEnvironment(llvm::Triple::EnvironmentType Env);
 
   virtual Tool *buildAssembler() const;
   virtual Tool *buildLinker() const;
   virtual Tool *buildStaticLibTool() const;
   virtual Tool *getTool(Action::ActionClass AC) const;
 
   virtual std::string buildCompilerRTBasename(const llvm::opt::ArgList &Args,
public:
 
   /// AddFastMathRuntimeIfAvailable - If a runtime library exists that sets
   /// global flags for unsafe floating point math, add it and return true.
   ///
   /// This checks for presence of the -Ofast, -ffast-math or -funsafe-math flags.
   bool addFastMathRuntimeIfAvailable(
     const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs) const;
 
+  /// getSystemGPUArchs - Use a tool to detect the user's availible GPUs.
+  virtual Expected<SmallVector<std::string>>
+  getSystemGPUArchs(const llvm::opt::ArgList &Args) const;
+
   /// addProfileRTLibs - When -fprofile-instr-profile is specified, try to pass
   /// a suitable profile runtime library to the linker.
   virtual void addProfileRTLibs(const llvm::opt::ArgList &Args,
                                 llvm::opt::ArgStringList &CmdArgs) const;
 
   /// Add arguments to use system-specific CUDA includes.
   virtual void AddCudaIncludeArgs(const llvm::opt::ArgList &DriverArgs,
                                   llvm::opt::ArgStringList &CC1Args) const;

clang/lib/Driver/Driver.cpp

bool initialize() override {
               A->getOption().matches(options::OPT_no_offload_arch_EQ)))
           continue;
         A->claim();
 
         for (StringRef ArchStr : llvm::split(A->getValue(), ",")) {
           if (A->getOption().matches(options::OPT_no_offload_arch_EQ) &&
               ArchStr == "all") {
             GpuArchs.clear();
-          } else if (ArchStr == "native" &&
-                     ToolChains.front()->getTriple().isAMDGPU()) {
-            auto *TC = static_cast<const toolchains::HIPAMDToolChain *>(
-                ToolChains.front());
-            SmallVector<std::string, 1> GPUs;
-            auto Err = TC->detectSystemGPUs(Args, GPUs);
-            if (!Err) {
-              for (auto GPU : GPUs)
-                GpuArchs.insert(Args.MakeArgString(GPU));
-            } else
-              llvm::consumeError(std::move(Err));
+          } else if (ArchStr == "native") {
+            const ToolChain &TC = *ToolChains.front();
+            auto GPUsOrErr = ToolChains.front()->getSystemGPUArchs(Args);
+            if (!GPUsOrErr) {
+              TC.getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
+                  << llvm::Triple::getArchTypeName(TC.getArch())
+                  << llvm::toString(GPUsOrErr.takeError()) << "--offload-arch";
+              continue;
+            }
+
+            for (auto GPU : *GPUsOrErr) {
+            }
           } else {
             ArchStr = getCanonicalOffloadArch(ArchStr);
             if (ArchStr.empty()) {
               Error = true;
             } else if (A->getOption().matches(options::OPT_offload_arch_EQ))
               GpuArchs.insert(ArchStr);
             else if (A->getOption().matches(options::OPT_no_offload_arch_EQ))
               GpuArchs.erase(ArchStr);

clang/lib/Driver/ToolChain.cpp

 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TargetParser.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <cassert>
 #include <cstddef>
 #include <cstring>
 #include <string>
ToolChain::ToolChain(const Driver &D, const llvm::Triple &T,
 
   for (const auto &Path : getRuntimePaths())
     addIfExists(getLibraryPaths(), Path);
   for (const auto &Path : getStdlibPaths())
     addIfExists(getFilePaths(), Path);
   addIfExists(getFilePaths(), getArchSpecificLibPath());
 }
 
+llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
+ToolChain::executeToolChainProgram(StringRef Executable) const {
+  llvm::SmallString<64> OutputFile;
+  llvm::sys::fs::createTemporaryFile("toolchain-program", "txt", OutputFile);
+  llvm::FileRemover OutputRemover(OutputFile.c_str());
+  std::optional<llvm::StringRef> Redirects[] = {
+      {""},
+      OutputFile.str(),
+      {""},
+  };
+
+  std::string ErrorMessage;
+  if (llvm::sys::ExecuteAndWait(Executable, {}, {}, Redirects,
+                                /* SecondsToWait */ 0,
+                                /*MemoryLimit*/ 0, &ErrorMessage))
+    return llvm::createStringError(std::error_code(),
+                                   Executable + ": " + ErrorMessage);
+
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> OutputBuf =
+      llvm::MemoryBuffer::getFile(OutputFile.c_str());
+  if (!OutputBuf)
+    return llvm::createStringError(OutputBuf.getError(),
+                                   "Failed to read stdout of " + Executable +
+                                       ": " + OutputBuf.getError().message());
+  return std::move(*OutputBuf);
+}
+
 void ToolChain::setTripleEnvironment(llvm::Triple::EnvironmentType Env) {
   Triple.setEnvironment(Env);
   if (EffectiveTriple != llvm::Triple())
     EffectiveTriple.setEnvironment(Env);
 }
 
 ToolChain::~ToolChain() = default;
 
bool ToolChain::addFastMathRuntimeIfAvailable(const ArgList &Args,
   if (isFastMathRuntimeAvailable(Args, Path)) {
     CmdArgs.push_back(Args.MakeArgString(Path));
     return true;
   }
 
   return false;
 }
 
+Expected<SmallVector<std::string>>
+ToolChain::getSystemGPUArchs(const llvm::opt::ArgList &Args) const {
+  return SmallVector<std::string>();
+}
+
 SanitizerMask ToolChain::getSupportedSanitizers() const {
   // Return sanitizers which don't require runtime support and are not
   // platform dependent.
 
   SanitizerMask Res =
       (SanitizerKind::Undefined & ~SanitizerKind::Vptr &
        ~SanitizerKind::Function) |
       (SanitizerKind::CFI & ~SanitizerKind::CFIICall) |

clang/lib/Driver/ToolChains/AMDGPU.h

public:
   }
 
   /// Needed for translating LTO options.
   const char *getDefaultLinker() const override { return "ld.lld"; }
 
   /// Should skip argument.
   bool shouldSkipArgument(const llvm::opt::Arg *Arg) const;
 
-  /// Uses amdgpu_arch tool to get arch of the system GPU. Will return error
+  /// Uses amdgpu-arch tool to get arch of the system GPU. Will return error
   /// if unable to find one.
-  llvm::Error getSystemGPUArch(const llvm::opt::ArgList &Args,
-                               std::string &GPUArch) const;
-
-  llvm::Error detectSystemGPUs(const llvm::opt::ArgList &Args,
-                               SmallVector<std::string, 1> &GPUArchs) const;
+  virtual Expected<SmallVector<std::string>>
+  getSystemGPUArchs(const llvm::opt::ArgList &Args) const override;
 
 protected:
   /// Check and diagnose invalid target ID specified by -mcpu.
   virtual void checkTargetID(const llvm::opt::ArgList &DriverArgs) const;
 
   /// The struct type returned by getParsedTargetID.
   struct ParsedTargetIDType {
     std::optional<std::string> OptionalTargetID;

clang/lib/Driver/ToolChains/AMDGPU.cpp

 #include "clang/Basic/TargetID.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Distro.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/LineIterator.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <optional>
 #include <system_error>
 
-#define AMDGPU_ARCH_PROGRAM_NAME "amdgpu-arch"
-
 using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 using namespace clang;
 using namespace llvm::opt;
 
 // Look for sub-directory starts with PackageName under ROCm candidate path.
 // If there is one and only one matching sub-directory found, append the
void AMDGPUToolChain::checkTargetID(
     const llvm::opt::ArgList &DriverArgs) const {
   auto PTID = getParsedTargetID(DriverArgs);
   if (PTID.OptionalTargetID && !PTID.OptionalGPUArch) {
     getDriver().Diag(clang::diag::err_drv_bad_target_id)
         << *PTID.OptionalTargetID;
   }
 }
 
-llvm::Error
-AMDGPUToolChain::detectSystemGPUs(const ArgList &Args,
-                                  SmallVector<std::string, 1> &GPUArchs) const {
+Expected<SmallVector<std::string>>
+AMDGPUToolChain::getSystemGPUArchs(const ArgList &Args) const {
+  // Detect AMD GPUs availible on the system.
   std::string Program;
   if (Arg *A = Args.getLastArg(options::OPT_amdgpu_arch_tool_EQ))
     Program = A->getValue();
   else
-    Program = GetProgramPath(AMDGPU_ARCH_PROGRAM_NAME);
-  llvm::SmallString<64> OutputFile;
-  llvm::sys::fs::createTemporaryFile("print-system-gpus", "" /* No Suffix */,
-                                     OutputFile);
-  llvm::FileRemover OutputRemover(OutputFile.c_str());
-  std::optional<llvm::StringRef> Redirects[] = {
-      {""},
-      OutputFile.str(),
-      {""},
-  };
-
-  std::string ErrorMessage;
-  if (int Result = llvm::sys::ExecuteAndWait(
-          Program, {}, {}, Redirects, /* SecondsToWait */ 0,
-          /*MemoryLimit*/ 0, &ErrorMessage)) {
-    if (Result > 0) {
-      ErrorMessage = "Exited with error code " + std::to_string(Result);
-    } else if (Result == -1) {
-      ErrorMessage = "Execute failed: " + ErrorMessage;
-    } else {
-      ErrorMessage = "Crashed: " + ErrorMessage;
-    }
+    Program = GetProgramPath("amdgpu-arch");
 
-    return llvm::createStringError(std::error_code(),
-                                   Program + ": " + ErrorMessage);
-  }
-
-  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> OutputBuf =
-      llvm::MemoryBuffer::getFile(OutputFile.c_str());
-  if (!OutputBuf) {
-    return llvm::createStringError(OutputBuf.getError(),
-                                   "Failed to read stdout of " + Program +
-                                       ": " + OutputBuf.getError().message());
-  }
+  auto StdoutOrErr = executeToolChainProgram(Program);
+  if (!StdoutOrErr)
+    return StdoutOrErr.takeError();
 
-  for (llvm::line_iterator LineIt(**OutputBuf); !LineIt.is_at_end(); ++LineIt) {
-    GPUArchs.push_back(LineIt->str());
-  }
-  return llvm::Error::success();
-}
-
-llvm::Error AMDGPUToolChain::getSystemGPUArch(const ArgList &Args,
-                                              std::string &GPUArch) const {
-  // detect the AMDGPU installed in system
   SmallVector<std::string, 1> GPUArchs;
-  auto Err = detectSystemGPUs(Args, GPUArchs);
-  if (Err) {
-    return Err;
-  }
-  if (GPUArchs.empty()) {
+  for (StringRef Arch : llvm::split((*StdoutOrErr)->getBuffer(), "\n"))
+    if (!Arch.empty())
+      GPUArchs.push_back(Arch.str());
+
+  if (GPUArchs.empty())
     return llvm::createStringError(std::error_code(),
                                    "No AMD GPU detected in the system");
-  }
-  GPUArch = GPUArchs[0];
-  if (GPUArchs.size() > 1) {
-    if (!llvm::all_equal(GPUArchs))
-      return llvm::createStringError(
-          std::error_code(), "Multiple AMD GPUs found with different archs");
-  }
-  return llvm::Error::success();
+
+  return GPUArchs;
 }
 
 void ROCMToolChain::addClangTargetOptions(
     const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
     Action::OffloadKind DeviceOffloadingKind) const {
   AMDGPUToolChain::addClangTargetOptions(DriverArgs, CC1Args,
                                          DeviceOffloadingKind);
 

clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp

 using namespace clang::driver::tools;
 using namespace clang;
 using namespace llvm::opt;
 
 namespace {
 
 static bool checkSystemForAMDGPU(const ArgList &Args, const AMDGPUToolChain &TC,
                                  std::string &GPUArch) {
-  if (auto Err = TC.getSystemGPUArch(Args, GPUArch)) {
+  auto CheckError = [&](llvm::Error Err) -> bool {
     std::string ErrMsg =
         llvm::formatv("{0}", llvm::fmt_consume(std::move(Err)));
-    TC.getDriver().Diag(diag::err_drv_undetermined_amdgpu_arch) << ErrMsg;
+    TC.getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
+        << llvm::Triple::getArchTypeName(TC.getArch()) << ErrMsg << "-march";
     return false;
-  }
+  };
+
+  auto ArchsOrErr = TC.getSystemGPUArchs(Args);
+  if (!ArchsOrErr)
+    return CheckError(ArchsOrErr.takeError());
+
+  if (ArchsOrErr->size() > 1)
+    if (!llvm::all_equal(*ArchsOrErr))
+      return CheckError(llvm::createStringError(
+          std::error_code(), "Multiple AMD GPUs found with different archs"));
 
+  GPUArch = ArchsOrErr->front();
   return true;
 }
 } // namespace
 
 AMDGPUOpenMPToolChain::AMDGPUOpenMPToolChain(const Driver &D,
                                              const llvm::Triple &Triple,
                                              const ToolChain &HostTC,
                                              const ArgList &Args)

clang/lib/Driver/ToolChains/Cuda.h

public:
 
   unsigned GetDefaultDwarfVersion() const override { return 2; }
   // NVPTX supports only DWARF2.
   unsigned getMaxDwarfVersion() const override { return 2; }
 
   const ToolChain &HostTC;
   CudaInstallationDetector CudaInstallation;
 
+  /// Uses nvptx-arch tool to get arch of the system GPU. Will return error
+  /// if unable to find one.
+  virtual Expected<SmallVector<std::string>>
+  getSystemGPUArchs(const llvm::opt::ArgList &Args) const override;
+
 protected:
   Tool *buildAssembler() const override;  // ptxas
   Tool *buildLinker() const override;     // fatbinary (ok, not really a linker)
 };
 
 } // end namespace toolchains
 } // end namespace driver
 } // end namespace clang
 
 #endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_CUDA_H

clang/lib/Driver/ToolChains/Cuda.cpp

CudaToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
 
   if (!BoundArch.empty()) {
     DAL->eraseArg(options::OPT_march_EQ);
     DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ), BoundArch);
   }
   return DAL;
 }
 
+Expected<SmallVector<std::string>>
+CudaToolChain::getSystemGPUArchs(const ArgList &Args) const {
+  // Detect NVIDIA GPUs availible on the system.
+  std::string Program;
+  if (Arg *A = Args.getLastArg(options::OPT_nvptx_arch_tool_EQ))
+    Program = A->getValue();
+  else
+    Program = GetProgramPath("nvptx-arch");
+
+  auto StdoutOrErr = executeToolChainProgram(Program);
+  if (!StdoutOrErr)
+    return StdoutOrErr.takeError();
+
+  SmallVector<std::string, 1> GPUArchs;
+  for (StringRef Arch : llvm::split((*StdoutOrErr)->getBuffer(), "\n"))
+    if (!Arch.empty())
+      GPUArchs.push_back(Arch.str());
+
+  if (GPUArchs.empty())
+    return llvm::createStringError(std::error_code(),
+                                   "No NVIDIA GPU detected in the system");
+
+  return GPUArchs;
+}
+
 Tool *CudaToolChain::buildAssembler() const {
   return new tools::NVPTX::Assembler(*this);
 }
 
 Tool *CudaToolChain::buildLinker() const {
   return new tools::NVPTX::Linker(*this);
 }
 

clang/test/Driver/Inputs/nvptx-arch/nvptx_arch_fail

+#!/bin/sh
+exit 1

clang/test/Driver/Inputs/nvptx-arch/nvptx_arch_sm_70

+#!/bin/sh
+echo sm_70
+exit 0

clang/test/Driver/amdgpu-hip-system-arch.c

+// REQUIRES: system-linux
+// REQUIRES: x86-registered-target
+// REQUIRES: amdgpu-registered-target
+// REQUIRES: shell
+
+// RUN: mkdir -p %t
+// RUN: cp %S/Inputs/amdgpu-arch/amdgpu_arch_fail %t/
+// RUN: cp %S/Inputs/amdgpu-arch/amdgpu_arch_gfx906 %t/
+// RUN: echo '#!/bin/sh' > %t/amdgpu_arch_empty
+// RUN: chmod +x %t/amdgpu_arch_fail
+// RUN: chmod +x %t/amdgpu_arch_gfx906
+// RUN: chmod +x %t/amdgpu_arch_empty
+
+// case when amdgpu-arch returns nothing or fails
+// RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib --offload-arch=native --amdgpu-arch-tool=%t/amdgpu_arch_fail -x hip %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=NO-OUTPUT-ERROR
+// NO-OUTPUT-ERROR: error: cannot determine amdgcn architecture{{.*}}; consider passing it via '--offload-arch'
+
+// case when amdgpu-arch does not return anything with successful execution
+// RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib --offload-arch=native --amdgpu-arch-tool=%t/amdgpu_arch_empty -x hip %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=EMPTY-OUTPUT
+// EMPTY-OUTPUT: error: cannot determine amdgcn architecture: No AMD GPU detected in the system; consider passing it via '--offload-arch'
+
+// case when amdgpu-arch returns a gfx906 GPU.
+// RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib --offload-arch=native --amdgpu-arch-tool=%t/amdgpu_arch_gfx906 -x hip %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=ARCH-GFX906
+// ARCH-GFX906: "-cc1" "-triple" "amdgcn-amd-amdhsa"{{.*}}"-target-cpu" "gfx906"

clang/test/Driver/amdgpu-openmp-system-arch-fail.c

 // RUN: echo '#!/bin/sh' > %t/amdgpu_arch_empty
 // RUN: chmod +x %t/amdgpu_arch_fail
 // RUN: chmod +x %t/amdgpu_arch_different
 // RUN: chmod +x %t/amdgpu_arch_empty
 
 // case when amdgpu_arch returns nothing or fails
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -nogpulib --amdgpu-arch-tool=%t/amdgpu_arch_fail %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=NO-OUTPUT-ERROR
-// NO-OUTPUT-ERROR: error: cannot determine AMDGPU architecture{{.*}}Exited with error code 1; consider passing it via '--march'
+// NO-OUTPUT-ERROR: error: cannot determine amdgcn architecture{{.*}}; consider passing it via '-march'
 
 // case when amdgpu_arch returns multiple gpus but all are different
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -nogpulib --amdgpu-arch-tool=%t/amdgpu_arch_different %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=MULTIPLE-OUTPUT-ERROR
-// MULTIPLE-OUTPUT-ERROR: error: cannot determine AMDGPU architecture: Multiple AMD GPUs found with different archs; consider passing it via '--march'
+// MULTIPLE-OUTPUT-ERROR: error: cannot determine amdgcn architecture: Multiple AMD GPUs found with different archs; consider passing it via '-march'
 
 // case when amdgpu_arch does not return anything with successful execution
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -nogpulib --amdgpu-arch-tool=%t/amdgpu_arch_empty %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=EMPTY-OUTPUT
-// EMPTY-OUTPUT: error: cannot determine AMDGPU architecture: No AMD GPU detected in the system; consider passing it via '--march'
+// EMPTY-OUTPUT: error: cannot determine amdgcn architecture: No AMD GPU detected in the system; consider passing it via '-march'

clang/test/Driver/nvptx-cuda-system-arch.c

+// REQUIRES: system-linux
+// REQUIRES: x86-registered-target
+// REQUIRES: nvptx-registered-target
+// REQUIRES: shell
+
+// RUN: mkdir -p %t
+// RUN: cp %S/Inputs/nvptx-arch/nvptx_arch_fail %t/
+// RUN: cp %S/Inputs/nvptx-arch/nvptx_arch_sm_70 %t/
+// RUN: echo '#!/bin/sh' > %t/nvptx_arch_empty
+// RUN: chmod +x %t/nvptx_arch_fail
+// RUN: chmod +x %t/nvptx_arch_sm_70
+// RUN: chmod +x %t/nvptx_arch_empty
+
+// case when nvptx-arch returns nothing or fails
+// RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib --offload-arch=native --nvptx-arch-tool=%t/nvptx_arch_fail -x cuda %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=NO-OUTPUT-ERROR
+// NO-OUTPUT-ERROR: error: cannot determine nvptx64 architecture{{.*}}; consider passing it via '--offload-arch'
+
+// case when nvptx-arch does not return anything with successful execution
+// RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib --offload-arch=native --nvptx-arch-tool=%t/nvptx_arch_empty -x cuda %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=EMPTY-OUTPUT
+// EMPTY-OUTPUT: error: cannot determine nvptx64 architecture: No NVIDIA GPU detected in the system; consider passing it via '--offload-arch'
+
+// case when nvptx-arch does not return anything with successful execution
+// RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib --offload-arch=native --nvptx-arch-tool=%t/nvptx_arch_sm_70 -x cuda %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=ARCH-sm_70
+// ARCH-sm_70: "-cc1" "-triple" "nvptx64-nvidia-cuda"{{.*}}"-target-cpu" "sm_70"