Fix missing memcpy, memmove and memset tail calls

[sanwou01]

Summary

If a wrapper around one of the mem* stdlib functions bitcasts the returned
pointer value before returning it (e.g. to a wchar_t*), LLVM does not emit a
tail call.

Add a check for this scenario so that we emit a tail call.

[lebedev.ri]

Test?

[ramred01]

Updated the summary to properly reflect the issue and the solution.

Added a test case.

[wmi]

LGTM.

[sanwou01]

Hi, I'd like to take over this change to get it committed. There is a crash lurking when llvm::returnTypeIsEligibleForTailCall is called with a function that takes no arguments, so I'd like to make some changes before committing. Thanks.

[sanwou01]

Fix crash and refactor for readability

[dmgreen]

Yeah. LGTM. Thanks for carrying this forward!

llvm/lib/CodeGen/Analysis.cpp

617

I don't think these two asserts are required if you wanted to leave them out. It's pretty much implied from the signature that the two values will not be null.

If you do want to keep them in, run clang format and add a message to the asserts. Something like:
assert(a && b && "Expected non-null inputs!");

llvm/lib/CodeGen/Analysis.cpp

bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
   }
 
   // If they're still different, there's some facet we don't understand
   // (currently only "inreg", but in future who knows). It may be OK but the
   // only safe option is to reject the tail call.
   return CallerAttrs == CalleeAttrs;
 }
 
+/// Check whether B is a bitcast of a pointer type to another pointer type,
+/// which is equal to A.
+static bool isPointerBitcastEqualTo(const Value *A, const Value *B) {
+  assert(A && B && "Expected non-null inputs!");
+
+  auto *BitCastIn = dyn_cast<BitCastInst>(B);
+
+  if (!BitCastIn)
+    return false;
+
+  if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
+    return false;
+
+  return A == BitCastIn->getOperand(0);
+}
+
 bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
                                            const Instruction *I,
                                            const ReturnInst *Ret,
                                            const TargetLoweringBase &TLI) {
   // If the block ends with a void return or unreachable, it doesn't matter
   // what the call's return type is.
   if (!Ret || Ret->getNumOperands() == 0) return true;
 
bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
   if (Function *F = Call->getCalledFunction()) {
     Intrinsic::ID IID = F->getIntrinsicID();
     if (((IID == Intrinsic::memcpy &&
           TLI.getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy")) ||
          (IID == Intrinsic::memmove &&
           TLI.getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove")) ||
          (IID == Intrinsic::memset &&
           TLI.getLibcallName(RTLIB::MEMSET) == StringRef("memset"))) &&
-        RetVal == Call->getArgOperand(0))
+        (RetVal == Call->getArgOperand(0) ||
+         isPointerBitcastEqualTo(RetVal, Call->getArgOperand(0))))
       return true;
   }
 
   SmallVector<unsigned, 4> RetPath, CallPath;
   SmallVector<CompositeType *, 4> RetSubTypes, CallSubTypes;
 
   bool RetEmpty = !firstRealType(RetVal->getType(), RetSubTypes, RetPath);
   bool CallEmpty = !firstRealType(CallVal->getType(), CallSubTypes, CallPath);

llvm/test/CodeGen/AArch64/tailcall-bitcast-memcpy.ll

+;RUN: llc %s -o - -verify-machineinstrs | FileCheck %s
+target triple = "aarch64-arm-none-eabi"
+
+;CHECK-LABEL: @wmemcpy
+;CHECK: lsl
+;CHECK-NOT: bl
+;CHECK-NOT: mov
+;CHECK-NOT: ldp
+;CHECK-NEXT: b memcpy
+define dso_local i32* @wmemcpy(i32* returned, i32* nocapture readonly, i64) local_unnamed_addr {
+  %4 = bitcast i32* %0 to i8*
+  %5 = bitcast i32* %1 to i8*
+  %6 = shl i64 %2, 2
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %4, i8* align 4 %5, i64 %6, i1 false)
+  ret i32* %0
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)