[OpenMP] Allow CUDA to be linked with OpenMP using the new driver

[jhuber6]

Summary

After basic support for embedding and handling CUDA files was added to
the new driver, we should be able to call CUDA functions from OpenMP
code. This patch makes the necessary changes to successfuly link in CUDA
programs that were compiled using the new driver. With this patch it
should be possible to compile device-only CUDA code (no kernels) and
call it from OpenMP as follows:

$ clang++ cuda.cu -fopenmp-new-driver -offload-arch=sm_70 -c
$ clang++ openmp.cpp cuda.o -fopenmp-new-driver -fopenmp -fopenmp-targets=nvptx64 -Xopenmp-target=nvptx64 -march=sm_70

Currently this requires using a host variant to suppress the generation
of a CPU-side fallback call.

Depends on D120272

[jhuber6]

Make sure we don't pass PTX to nvlink.

[jhuber6]

Changing after rebasing previous commits.

[jhuber6]

Updating to use fatbinaries and fatbinary magic.

[jhuber6]

Update handling for fatbinaries.

[jhuber6]

rebase.

[jhuber6]

Ping and rebase

[jhuber6]

Adding test

[jdoerfert]

LG

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

struct DeviceFile {
 
   std::string Kind;
   std::string TheTriple;
   std::string Arch;
   std::string Filename;
 };
 
 namespace llvm {
-/// Helper that allows DeviceFile to be used as a key in a DenseMap.
+/// Helper that allows DeviceFile to be used as a key in a DenseMap. For now we
+/// assume device files with matching architectures and triples but different
+/// offloading kinds should be handlded together, this may not be true in the
+/// future.
 template <> struct DenseMapInfo<DeviceFile> {
   static DeviceFile getEmptyKey() {
     return {DenseMapInfo<StringRef>::getEmptyKey(),
             DenseMapInfo<StringRef>::getEmptyKey(),
             DenseMapInfo<StringRef>::getEmptyKey(),
             DenseMapInfo<StringRef>::getEmptyKey()};
   }
   static DeviceFile getTombstoneKey() {

 
 namespace {
 
 Expected<Optional<std::string>>
 extractFromBuffer(std::unique_ptr<MemoryBuffer> Buffer,
                   SmallVectorImpl<DeviceFile> &DeviceFiles);
 
 static StringRef getDeviceFileExtension(StringRef DeviceTriple,
-                                        bool IsBitcode = false) {
+                                        file_magic magic) {
   Triple TheTriple(DeviceTriple);
-  if (TheTriple.isAMDGPU() || IsBitcode)
+  if (magic == file_magic::bitcode)
     return "bc";
+  if (TheTriple.isNVPTX() && magic == file_magic::unknown)
+    return "s";
   if (TheTriple.isNVPTX())
     return "cubin";
   return "o";
 }
 
 std::string getMainExecutable(const char *Name) {
   void *Ptr = (void *)(intptr_t)&getMainExecutable;
   auto COWPath = sys::fs::getMainExecutable(Name, Ptr);
for (const SectionRef &Sec : Obj.sections()) {
     SmallVector<StringRef, 4> SectionFields;
     Name->split(SectionFields, '.');
     StringRef Kind = SectionFields[3];
     StringRef DeviceTriple = SectionFields[4];
     StringRef Arch = SectionFields[5];
 
     if (Expected<StringRef> Contents = Sec.getContents()) {
       SmallString<128> TempFile;
-      StringRef DeviceExtension = getDeviceFileExtension(
-          DeviceTriple, identify_magic(*Contents) == file_magic::bitcode);
+      StringRef DeviceExtension =
+          getDeviceFileExtension(DeviceTriple, identify_magic(*Contents));
       if (Error Err = createOutputFile(Prefix + "-" + Kind + "-" +
                                            DeviceTriple + "-" + Arch,
                                        DeviceExtension, TempFile))
         return std::move(Err);
 
       Expected<std::unique_ptr<FileOutputBuffer>> OutputOrErr =
           FileOutputBuffer::create(TempFile, Sec.getSize());
       if (!OutputOrErr)
for (GlobalVariable &GV : M->globals()) {
     SmallVector<StringRef, 4> SectionFields;
     GV.getSection().split(SectionFields, '.');
     StringRef Kind = SectionFields[3];
     StringRef DeviceTriple = SectionFields[4];
     StringRef Arch = SectionFields[5];
 
     StringRef Contents = CDS->getAsString();
     SmallString<128> TempFile;
-    StringRef DeviceExtension = getDeviceFileExtension(
-        DeviceTriple, identify_magic(Contents) == file_magic::bitcode);
+    StringRef DeviceExtension =
+        getDeviceFileExtension(DeviceTriple, identify_magic(Contents));
     if (Error Err = createOutputFile(Prefix + "-" + Kind + "-" + DeviceTriple +
                                          "-" + Arch,
                                      DeviceExtension, TempFile))
       return std::move(Err);
 
     Expected<std::unique_ptr<FileOutputBuffer>> OutputOrErr =
         FileOutputBuffer::create(TempFile, Contents.size());
     if (!OutputOrErr)
if (DebugInfo != NoDebugInfo)
     CmdArgs.push_back("-g");
   CmdArgs.push_back("-o");
   CmdArgs.push_back(TempFile);
   CmdArgs.push_back("-arch");
   CmdArgs.push_back(Arch);
 
   // Add extracted input files.
   for (StringRef Input : InputFiles)
+    if (Input.endswith(".cubin"))
       CmdArgs.push_back(Input);
 
   if (sys::ExecuteAndWait(*NvlinkPath, CmdArgs))
     return createStringError(inconvertibleErrorCode(), "'nvlink' failed");
 
   return static_cast<std::string>(TempFile);
 }
 } // namespace nvptx
 namespace amdgcn {
Error linkBitcodeFiles(SmallVectorImpl<std::string> &InputFiles,
   // save.
   for (StringRef File : InputFiles) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
         MemoryBuffer::getFileOrSTDIN(File);
     if (std::error_code EC = BufferOrErr.getError())
       return createFileError(File, EC);
 
     file_magic Type = identify_magic((*BufferOrErr)->getBuffer());
-    if (Type != file_magic::bitcode) {
+    switch (Type) {
+    case file_magic::bitcode: {
+      Expected<std::unique_ptr<lto::InputFile>> InputFileOrErr =
+          llvm::lto::InputFile::create(**BufferOrErr);
+      if (!InputFileOrErr)
+        return InputFileOrErr.takeError();
+
+      // Save the input file and the buffer associated with its memory.
+      BitcodeFiles.push_back(std::move(*InputFileOrErr));
+      SavedBuffers.push_back(std::move(*BufferOrErr));
+      continue;
+    }
+    case file_magic::elf_relocatable:
+    case file_magic::elf_shared_object:
+    case file_magic::macho_object:
+    case file_magic::coff_object: {
       Expected<std::unique_ptr<ObjectFile>> ObjFile =
           ObjectFile::createObjectFile(**BufferOrErr, Type);
       if (!ObjFile)
         return ObjFile.takeError();
 
       NewInputFiles.push_back(File.str());
       for (auto &Sym : (*ObjFile)->symbols()) {
         Expected<StringRef> Name = Sym.getName();
         if (!Name)
           return Name.takeError();
 
         // Record if we've seen these symbols in any object or shared libraries.
         if ((*ObjFile)->isRelocatableObject())
           UsedInRegularObj.insert(Saver.save(*Name));
         else
           UsedInSharedLib.insert(Saver.save(*Name));
       }
-    } else {
-      Expected<std::unique_ptr<lto::InputFile>> InputFileOrErr =
-          llvm::lto::InputFile::create(**BufferOrErr);
-      if (!InputFileOrErr)
-        return InputFileOrErr.takeError();
-
-      // Save the input file and the buffer associated with its memory.
-      BitcodeFiles.push_back(std::move(*InputFileOrErr));
-      SavedBuffers.push_back(std::move(*BufferOrErr));
+      continue;
+    }
+    default:
+      continue;
     }
   }
 
   if (BitcodeFiles.empty())
     return Error::success();
 
   auto HandleError = [&](Error Err) {
     logAllUnhandledErrors(std::move(Err),