PreISelIntrinsicLowering: Check RuntimeLibcalls instead of TLI for memory functions

[arsenm]

Summary

We need a better mechanism for expressing which calls you are allowed
to emit and which calls are recognized. This should be applied to the 17 branch.

[arsenm]

Fix diff

[paulkirth]

Thanks for getting a patch out so quickly. This looks like it avoids the issue we were seeing in our build.

llvm/test/Transforms/PreISelIntrinsicLowering/X86/load-relative.ll

1

Why were load-relative.ll and obj-arc.ll made x86 specific tests? They seem like they should work on any platform.

I'm not opposed to the change, but it's unclear to me what this accomplishes.

[arsenm]

llvm/test/Transforms/PreISelIntrinsicLowering/X86/load-relative.ll

1

The test needs a TargetMachine to query the subtarget. It was weird this didn't have one before

[paulkirth]

llvm/test/Transforms/PreISelIntrinsicLowering/X86/load-relative.ll

1

Right, I missed that. Moving this makes total sense. Thanks for the explanation.

[arsenm]

c8cac156135de8e211b800ea5abe9666803b2875

[nikic]

llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp

208

This is using the TLI for the wrong function: It's the one for the intrinsic, rather than the one for the intrinsic caller.

[arsenm]

llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp

208

I think this is harmless because the runtime libcall system isn't really refined per function. It's effectively a hardcoded table driven off the triple

llvm/include/llvm/CodeGen/CodeGenPassBuilder.h

#include "llvm/CodeGen/MachinePassRegistry.def"
   return Ret;
 }
 
 template <typename Derived>
 void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
   if (TM.useEmulatedTLS())
     addPass(LowerEmuTLSPass());
 
-  addPass(PreISelIntrinsicLoweringPass());
+  addPass(PreISelIntrinsicLoweringPass(TM));
 
   derived().addIRPasses(addPass);
   derived().addCodeGenPrepare(addPass);
   addPassesToHandleExceptions(addPass);
   derived().addISelPrepare(addPass);
 }
 
 /// Add common target configurable passes that perform LLVM IR to IR transforms

llvm/include/llvm/CodeGen/PreISelIntrinsicLowering.h

 #ifndef LLVM_CODEGEN_PREISELINTRINSICLOWERING_H
 #define LLVM_CODEGEN_PREISELINTRINSICLOWERING_H
 
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
 
 class Module;
+class TargetMachine;
 
 struct PreISelIntrinsicLoweringPass
     : PassInfoMixin<PreISelIntrinsicLoweringPass> {
+  const TargetMachine &TM;
+
+  PreISelIntrinsicLoweringPass(const TargetMachine &TM) : TM(TM) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_PREISELINTRINSICLOWERING_H

llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp

 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
 #include "llvm/Analysis/ObjCARCInstKind.h"
 #include "llvm/Analysis/ObjCARCUtil.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
 
 using namespace llvm;
 
 /// Threshold to leave statically sized memory intrinsic calls. Calls of known
 /// size larger than this will be expanded by the pass. Calls of unknown or
 /// lower size will be left for expansion in codegen.
 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
     "mem-intrinsic-expand-size",
     cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
     cl::Hidden);
 
 namespace {
 
 struct PreISelIntrinsicLowering {
+  const TargetMachine &TM;
   const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
-  const function_ref<TargetLibraryInfo &(Function &)> LookupLibInfo;
 
   /// If this is true, assume it's preferably to leave memory intrinsic calls
   /// for replacement with a library call later. Otherwise this depends on
-  /// TargetLibraryInfo availability of the corresponding function.
+  /// TargetLoweringInfo availability of the corresponding function.
   const bool UseMemIntrinsicLibFunc;
 
   explicit PreISelIntrinsicLowering(
+      const TargetMachine &TM_,
       function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
-      function_ref<TargetLibraryInfo &(Function &)> LookupLibInfo_,
       bool UseMemIntrinsicLibFunc_ = true)
-      : LookupTTI(LookupTTI_), LookupLibInfo(LookupLibInfo_),
+      : TM(TM_), LookupTTI(LookupTTI_),
         UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
 
   static bool shouldExpandMemIntrinsicWithSize(Value *Size,
                                                const TargetTransformInfo &TTI);
   bool expandMemIntrinsicUses(Function &F) const;
   bool lowerIntrinsics(Module &M) const;
 };
 
uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
                            : TTI.getMaxMemIntrinsicInlineSizeThreshold();
   uint64_t SizeVal = CI->getZExtValue();
 
   // Treat a threshold of 0 as a special case to force expansion of all
   // intrinsics, including size 0.
   return SizeVal > Threshold || Threshold == 0;
 }
 
+static bool canEmitLibcall(const TargetLowering &TLI, RTLIB::Libcall LC) {
+  // TODO: Should this consider the address space of the memcpy?
+  return TLI.getLibcallName(LC) != nullptr;
+}
+
 // TODO: Handle atomic memcpy and memcpy.inline
 // TODO: Pass ScalarEvolution
 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
+  const TargetLowering *TLI = TM.getSubtargetImpl(F)->getTargetLowering();
   Intrinsic::ID ID = F.getIntrinsicID();
   bool Changed = false;
 
   for (User *U : llvm::make_early_inc_range(F.users())) {
     Instruction *Inst = cast<Instruction>(U);
 
     switch (ID) {
     case Intrinsic::memcpy: {
       auto *Memcpy = cast<MemCpyInst>(Inst);
       Function *ParentFunc = Memcpy->getFunction();
       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
       if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
-        if (UseMemIntrinsicLibFunc &&
-            LookupLibInfo(*ParentFunc).has(LibFunc_memcpy))
+        if (UseMemIntrinsicLibFunc && canEmitLibcall(*TLI, RTLIB::MEMCPY))
           break;
 
+        // TODO: For optsize, emit the loop into a separate function
         expandMemCpyAsLoop(Memcpy, TTI);
         Changed = true;
         Memcpy->eraseFromParent();
       }
 
       break;
     }
     case Intrinsic::memmove: {
       auto *Memmove = cast<MemMoveInst>(Inst);
       Function *ParentFunc = Memmove->getFunction();
       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
       if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
-        if (UseMemIntrinsicLibFunc &&
-            LookupLibInfo(*ParentFunc).has(LibFunc_memmove))
+        if (UseMemIntrinsicLibFunc && canEmitLibcall(*TLI, RTLIB::MEMMOVE))
           break;
 
         if (expandMemMoveAsLoop(Memmove, TTI)) {
           Changed = true;
           Memmove->eraseFromParent();
         }
       }
 
       break;
     }
     case Intrinsic::memset: {
       auto *Memset = cast<MemSetInst>(Inst);
       Function *ParentFunc = Memset->getFunction();
       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
       if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
-        if (UseMemIntrinsicLibFunc &&
-            LookupLibInfo(*Memset->getFunction()).has(LibFunc_memset))
+        if (UseMemIntrinsicLibFunc && canEmitLibcall(*TLI, RTLIB::MEMSET))
           break;
 
         expandMemSetAsLoop(Memset);
         Changed = true;
         Memset->eraseFromParent();
       }
 
       break;

 
 class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
 public:
   static char ID;
 
   PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<TargetLibraryInfoWrapperPass>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
+    AU.addRequired<TargetPassConfig>();
   }
 
   bool runOnModule(Module &M) override {
     auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
       return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
     };
 
-    auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
-      return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
-    };
-
-    PreISelIntrinsicLowering Lowering(LookupTTI, LookupTLI);
+    const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+    PreISelIntrinsicLowering Lowering(TM, LookupTTI);
     return Lowering.lowerIntrinsics(M);
   }
 };
 
 } // end anonymous namespace
 
 char PreISelIntrinsicLoweringLegacyPass::ID;
 
-INITIALIZE_PASS(PreISelIntrinsicLoweringLegacyPass,
-                "pre-isel-intrinsic-lowering", "Pre-ISel Intrinsic Lowering",
-                false, false)
+INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
+                      "pre-isel-intrinsic-lowering",
+                      "Pre-ISel Intrinsic Lowering", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
+                    "pre-isel-intrinsic-lowering",
+                    "Pre-ISel Intrinsic Lowering", false, false)
 
 ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
-  return new PreISelIntrinsicLoweringLegacyPass;
+  return new PreISelIntrinsicLoweringLegacyPass();
 }
 
 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
                                                     ModuleAnalysisManager &AM) {
   auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 
-  auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
-    return FAM.getResult<TargetLibraryAnalysis>(F);
-  };
-
   auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
     return FAM.getResult<TargetIRAnalysis>(F);
   };
 
-  PreISelIntrinsicLowering Lowering(LookupTTI, LookupTLI);
+  PreISelIntrinsicLowering Lowering(TM, LookupTTI);
   if (!Lowering.lowerIntrinsics(M))
     return PreservedAnalyses::all();
   else
     return PreservedAnalyses::none();
 }

llvm/test/CodeGen/ARM/no-expand-memcpy-no-builtins.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=thumbv7em-apple-unknown-macho < %s | FileCheck %s
+
+target datalayout = "e-m:o-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) #0
+declare void @llvm.memmove.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) #0
+declare void @llvm.memset.p0.i32(ptr noalias nocapture writeonly, i8, i32, i1 immarg) #0
+
+; Check we don't expand memcpy to a loop when the caller
+; even if we have no-builtins attached.
+
+; CHECK: bl _memcpy
+define arm_aapcs_vfpcc void @test_memcpy(ptr %p1, ptr %p2) #1 {
+  call void @llvm.memcpy.p0.p0.i32(ptr %p1, ptr %p2, i32 128, i1 false)
+  ret void
+}
+
+; CHECK: bl _memmove
+define arm_aapcs_vfpcc void @test_memmove(ptr %p1, ptr %p2) #1 {
+  call void @llvm.memmove.p0.p0.i32(ptr %p1, ptr %p2, i32 128, i1 false)
+  ret void
+}
+
+; CHECK: bl _memset
+define arm_aapcs_vfpcc void @test_memset(ptr %p1) #1 {
+  call void @llvm.memset.p0.i32(ptr %p1, i8 0, i32 128, i1 false)
+  ret void
+}
+
+attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
+attributes #1 = { "no-builtins" }

llvm/test/Transforms/PreISelIntrinsicLowering/X86/lit.local.cfg

+if not "X86" in config.root.targets:
+    config.unsupported = True

llvm/test/Transforms/PreISelIntrinsicLowering/X86/load-relative.ll

-; RUN: opt -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-pc-linux-gnu -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
 
 ; CHECK: define ptr @foo32(ptr [[P:%.*]], i32 [[O:%.*]])
 define ptr @foo32(ptr %p, i32 %o) {
   ; CHECK: [[OP:%.*]] = getelementptr i8, ptr [[P]], i32 [[O]]
   ; CHECK: [[OI32:%.*]] = load i32, ptr [[OP]], align 4
   ; CHECK: [[R:%.*]] = getelementptr i8, ptr [[P]], i32 [[OI32]]
   ; CHECK: ret ptr [[R]]
   %l = call ptr @llvm.load.relative.i32(ptr %p, i32 %o)

llvm/test/Transforms/PreISelIntrinsicLowering/X86/objc-arc.ll

-; RUN: opt -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-pc-linux-gnu -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
 
 ; Make sure calls to the objc intrinsics are translated to calls in to the
 ; runtime
 
 declare ptr @foo()
 declare i32 @__gxx_personality_v0(...)
 
 define ptr @test_objc_autorelease(ptr %arg0) {