[mlir][mlir-rocm-runner] Detect HIP version and AMD ISA version.

[whchung]

Summary

Prior to ROCm / HIP 3.5, HSA code object V2 is the default binary format.
Starting from ROCm 3.5, HSA code object V3 becomes default.

Also invoke rocm_agent_enumerator to detect proper AMD ISA version on the
system. Use gfx900 as the fallback value.

[whchung]

Fix build error due to incorrect lint.

[whchung]

Add logic to detech AMD ISA version.

[whchung]

Revise HIP version detection logic to be more descriptive.

[whchung]

Fix typos in the commit log.

[yaxunl]

LGTM

mlir/tools/mlir-rocm-runner/CMakeLists.txt

if(MLIR_ROCM_RUNNER_ENABLED)
   set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
   find_package(HIP)
   if (NOT HIP_FOUND)
     message(SEND_ERROR "Build the mlir rocm runner requires a working ROCm and HIP install")
   else()
     message(STATUS "ROCm HIP version: ${HIP_VERSION}")
   endif()
 
+  # Set compile-time flags for ROCm path.
+  add_definitions(-D__ROCM_PATH__="${ROCM_PATH}")
+
   # Locate HIP runtime library.
   find_library(ROCM_RUNTIME_LIBRARY hip_hcc
                PATHS "${HIP_PATH}/lib")
   if (NOT ROCM_RUNTIME_LIBRARY)
     message(SEND_ERROR "Could not locate ROCm HIP runtime library")
   else()
     message(STATUS "ROCm HIP runtime lib: ${ROCM_RUNTIME_LIBRARY}")
   endif()
if(MLIR_ROCM_RUNNER_ENABLED)
 
   add_llvm_tool(mlir-rocm-runner
     mlir-rocm-runner.cpp
 
     DEPENDS
     rocm-runtime-wrappers
     )
   llvm_update_compile_flags(mlir-rocm-runner)
+  target_include_directories(mlir-rocm-runner
+    PRIVATE
+    "${HIP_PATH}/../include"
+    "${HIP_PATH}/include"
+  )
   target_link_libraries(mlir-rocm-runner PRIVATE ${LIBS} ${targets_to_link})
 
 endif()

mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp

 #include "mlir/IR/Module.h"
 #include "mlir/InitAllDialects.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Support/FileUtilities.h"
 #include "mlir/Target/ROCDLIR.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
+#include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/Program.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
 
 // MC headers.
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
 
 // lld headers.
 #include "lld/Common/Driver.h"
 
+// HIP headers.
+#include "hip/hip_version.h"
+
 using namespace mlir;
 using namespace llvm;
 
 using Blob = SmallVector<char, 0>;
 
 static cl::opt<std::string> tripleName("triple", cl::desc("target triple"),
                                        cl::value_desc("triple string"),
                                        cl::init("amdgcn-amd-amdhsa"));
 
-// TODO(whchung): Add feature to automatically detect available AMD GCN ISA
-// version via `rocm-agent-enumerator` utility.
 static cl::opt<std::string> targetChip("target", cl::desc("target chip"),
                                        cl::value_desc("AMDGPU ISA version"),
-                                       cl::init("gfx900"));
+                                       cl::init(""));
 
 static cl::opt<std::string> features("feature", cl::desc("target features"),
                                      cl::value_desc("AMDGPU target features"),
-                                     cl::init("-code-object-v3"));
+                                     cl::init(""));
+
+static constexpr const char kRunnerProgram[] = "mlir-rocm-runner";
+static constexpr const char kRocmAgentEnumerator[] = "rocm_agent_enumerator";
+static constexpr const char kDefaultTargetChip[] = "gfx900";
 
 static LogicalResult assembleIsa(const std::string isa, StringRef name,
                                  Blob &result) {
   raw_svector_ostream os(result);
 
   std::string error;
   Triple theTriple(Triple::normalize(tripleName));
   const Target *theTarget =
if (succeeded(assembleIsa(isa, name, isaBlob)) &&
       succeeded(createHsaco(isaBlob, name, hsacoBlob)))
     return std::make_unique<std::vector<char>>(hsacoBlob.begin(),
                                                hsacoBlob.end());
 
   WithColor::error(errs(), name) << "producing HSA code object error.\n";
   return {};
 }
 
+static void configTargetChip() {
+  // Set targetChip to default value first.
+  targetChip = kDefaultTargetChip;
+
+  // Locate rocm_agent_enumerator.
+  llvm::ErrorOr<std::string> rocmAgentEnumerator = llvm::sys::findProgramByName(
+      kRocmAgentEnumerator, {__ROCM_PATH__ "/bin"});
+  std::error_code ec;
+  if (ec = rocmAgentEnumerator.getError()) {
+    WithColor::warning(errs(), kRunnerProgram)
+        << kRocmAgentEnumerator << " couldn't be located under "
+        << __ROCM_PATH__ << ", set target as " << kDefaultTargetChip << "\n";
+    return;
+  }
+
+  // Prepare temp file to hold the outputs.
+  int tempFd = -1;
+  SmallString<128> tempFilename;
+  ec = sys::fs::createTemporaryFile("rocm_agent", "txt", tempFd, tempFilename);
+  if (ec) {
+    WithColor::warning(errs(), kRunnerProgram)
+        << "temporary file for " << kRocmAgentEnumerator
+        << " creation error, set target as " << kDefaultTargetChip << "\n";
+    return;
+  }
+  FileRemover cleanup(tempFilename);
+
+  // Invoke rocm_agent_enumerator.
+  std::string errorMessage;
+  SmallVector<StringRef, 2> args{"-t", "GPU"};
+  Optional<StringRef> redirects[3] = {{""}, tempFilename.str(), {""}};
+  int result =
+      llvm::sys::ExecuteAndWait(rocmAgentEnumerator.get(), args, llvm::None,
+                                redirects, 0, 0, &errorMessage);
+  if (result) {
+    WithColor::warning(errs(), kRunnerProgram)
+        << kRocmAgentEnumerator << " invocation error: " << errorMessage
+        << ", set target as " << kDefaultTargetChip << "\n";
+    return;
+  }
+
+  // Load and parse the result.
+  auto gfxIsaList = mlir::openInputFile(tempFilename);
+  if (!gfxIsaList) {
+    WithColor::error(errs(), kRunnerProgram)
+        << "read ROCm agent list temp file error, set target as "
+        << kDefaultTargetChip << "\n";
+    return;
+  }
+  for (line_iterator lines(*gfxIsaList); !lines.is_at_end(); ++lines) {
+    // Skip the line with content "gfx000".
+    if (*lines == "gfx000")
+      continue;
+    // Use the first ISA version found.
+    targetChip = lines->str();
+    break;
+  }
+}
+
+static void configTargetFeatures() {
+  if (features.size() > 0)
+    features += ",";
+  // After ROCm 3.5, adopt HSA code object V3.
+  if (HIP_VERSION_MAJOR >= 3 && HIP_VERSION_MINOR >= 5)
+    features += "+code-object-v3";
+  else
+    features += "-code-object-v3";
+}
+
 static LogicalResult runMLIRPasses(ModuleOp m) {
   PassManager pm(m.getContext());
   applyPassManagerCLOptions(pm);
 
+  // Configure target chip ISA version if it has not been specified.
+  if (!targetChip.size())
+    configTargetChip();
+
+  // Configure target features per ROCm / HIP version.
+  configTargetFeatures();
+
   pm.addPass(createGpuKernelOutliningPass());
   auto &kernelPm = pm.nest<gpu::GPUModuleOp>();
   kernelPm.addPass(createStripDebugInfoPass());
   kernelPm.addPass(createLowerGpuOpsToROCDLOpsPass());
   kernelPm.addPass(createConvertGPUKernelToBlobPass(
       compileModuleToROCDLIR, compileISAToHsaco, tripleName, targetChip,
       features, /*gpuBinaryAnnotation=*/"rocdl.hsaco"));
   pm.addPass(createLowerToLLVMPass());