[CUDA, MemCpyOpt] Add a flag to force-enable memcpyopt and use it for CUDA.

[tra]

Summary

Attempt to enable MemCpyOpt unconditionally in D104801 uncovered the fact that
there are users that do not expect LLVM to materialize memset intrinsic.

While other passes can do that, too, MemCpyOpt triggers it more frequently and breaks sanitizers and
some downstream users.

For now introduce a flag to force-enable the flag and opt-in only CUDA
compilation with NVPTX back-end.

[tra]

Fixed the option name.

[nikic]

Would the variant of the original patch at D106769 be sufficient for your purposes? Or are you also interested in the optimizations that introduce new memset/memcpy?

[tra]

In D106401#2903114, @nikic wrote:

Would the variant of the original patch at D106769 be sufficient for your purposes? Or are you also interested in the optimizations that introduce new memset/memcpy?

The specific issue I was looking to fix does need memcpy->memset transformation, so I would still need a way to bypass libcall availability checks.

[tra]

Updated post D106769

[tra]

I've updated the patch and added a test to verify that the knob does work as expected. Please take a look.

[asbirlea]

lgtm.

[tra]

rebase to HEAD.

clang/lib/Driver/ToolChains/Cuda.cpp

void CudaToolChain::addClangTargetOptions(
 
   StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);
   assert(!GpuArch.empty() && "Must have an explicit GPU arch.");
   assert((DeviceOffloadingKind == Action::OFK_OpenMP ||
           DeviceOffloadingKind == Action::OFK_Cuda) &&
          "Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs.");
 
   if (DeviceOffloadingKind == Action::OFK_Cuda) {
-    CC1Args.push_back("-fcuda-is-device");
+    CC1Args.append(
+        {"-fcuda-is-device", "-mllvm", "-enable-memcpyopt-without-libcalls"});
 
     if (DriverArgs.hasFlag(options::OPT_fcuda_approx_transcendentals,
                            options::OPT_fno_cuda_approx_transcendentals, false))
       CC1Args.push_back("-fcuda-approx-transcendentals");
   }
 
   if (DriverArgs.hasArg(options::OPT_nogpulib))
     return;

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

 #include <cassert>
 #include <cstdint>
 #include <utility>
 
 using namespace llvm;
 
 #define DEBUG_TYPE "memcpyopt"
 
+static cl::opt<bool> EnableMemCpyOptWithoutLibcalls(
+    "enable-memcpyopt-without-libcalls", cl::init(false), cl::Hidden,
+    cl::desc("Enable memcpyopt even when libcalls are disabled"));
+
 static cl::opt<bool>
     EnableMemorySSA("enable-memcpyopt-memoryssa", cl::init(true), cl::Hidden,
                     cl::desc("Use MemorySSA-backed MemCpyOpt."));
 
 STATISTIC(NumMemCpyInstr, "Number of memcpy instructions deleted");
 STATISTIC(NumMemSetInfer, "Number of memsets inferred");
 STATISTIC(NumMoveToCpy,   "Number of memmoves converted to memcpy");
 STATISTIC(NumCpyToSet,    "Number of memcpys converted to memset");
if (LoadInst *LI = dyn_cast<LoadInst>(StoredVal)) {
     if (LI->isSimple() && LI->hasOneUse() &&
         LI->getParent() == SI->getParent()) {
 
       auto *T = LI->getType();
       // Don't introduce calls to memcpy/memmove intrinsics out of thin air if
       // the corresponding libcalls are not available.
       // TODO: We should really distinguish between libcall availability and
       // our ability to introduce intrinsics.
-      if (T->isAggregateType() && TLI->has(LibFunc_memcpy) &&
-          TLI->has(LibFunc_memmove)) {
+      if (T->isAggregateType() &&
+          (EnableMemCpyOptWithoutLibcalls ||
+           (TLI->has(LibFunc_memcpy) && TLI->has(LibFunc_memmove)))) {
         MemoryLocation LoadLoc = MemoryLocation::get(LI);
 
         // We use alias analysis to check if an instruction may store to
         // the memory we load from in between the load and the store. If
         // such an instruction is found, we try to promote there instead
         // of at the store position.
         // TODO: Can use MSSA for this.
         Instruction *P = SI;
if (LI->isSimple() && LI->hasOneUse() &&
       }
     }
   }
 
   // The following code creates memset intrinsics out of thin air. Don't do
   // this if the corresponding libfunc is not available.
   // TODO: We should really distinguish between libcall availability and
   // our ability to introduce intrinsics.
-  if (!TLI->has(LibFunc_memset))
+  if (!(TLI->has(LibFunc_memset) || EnableMemCpyOptWithoutLibcalls))
     return false;
 
   // There are two cases that are interesting for this code to handle: memcpy
   // and memset.  Right now we only handle memset.
 
   // Ensure that the value being stored is something that can be memset'able a
   // byte at a time like "0" or "-1" or any width, as well as things like
   // 0xA0A0A0A0 and 0.0.

llvm/test/Transforms/MemCpyOpt/no-libcalls.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -memcpyopt < %s | FileCheck %s --check-prefixes=CHECK,LIBCALLS
 ; RUN: opt -S -memcpyopt -mtriple=amdgcn-- < %s | FileCheck %s --check-prefixes=CHECK,NO-LIBCALLS
+; RUN: opt -S -memcpyopt -mtriple=amdgcn-- -enable-memcpyopt-without-libcalls < %s \
+; RUN:     | FileCheck %s --check-prefixes=CHECK,LIBCALLS
 
 ; REQUIRES: amdgpu-registered-target
 
 define void @dont_create_memset(ptr %p) {
 ; LIBCALLS-LABEL: @dont_create_memset(
 ; LIBCALLS-NEXT:    [[P1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 1
 ; LIBCALLS-NEXT:    [[P2:%.*]] = getelementptr i32, ptr [[P]], i64 2
 ; LIBCALLS-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P]], i64 3