[AArch64][GlobalISel] Support lowering musttail calls

[paquette]

Summary

Since we now lower most tail calls, it makes sense to support musttail.

Instead of always falling back to SelectionDAG, only fall back when a musttail call was not able to be emitted as a tail call. Once we can handle most incoming and outgoing arguments, we can change this to a report_fatal_error like in ISelLowering.

Remove the assert that we don't have varargs and a musttail, and replace it with a return false. Implementing this requires that we implement saveVarArgRegisters from AArch64ISelLowering, which is an entirely different patch.

Delete call-translator-musttail.ll, since it was just verifying that we don't tail call. Move the now-passing test to call-translator-tail-call.ll.

Add GlobalISel lines to vararg-tallcall.ll to make sure that we produce correct code. Right now we only fall back, but eventually this will be relevant.

[aemerson]

llvm/lib/Target/AArch64/AArch64CallLowering.cpp

872

Do we have a test case for this?

[paquette]

Add test verifying that we fall back to SelectionDAG and die when a musttail call is not lowered as a tail call.

[aemerson]

LGTM.

llvm/lib/Target/AArch64/AArch64CallLowering.cpp

bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
 
   return true;
 }
 
 bool AArch64CallLowering::isEligibleForTailCallOptimization(
     MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
     SmallVectorImpl<ArgInfo> &InArgs,
     SmallVectorImpl<ArgInfo> &OutArgs) const {
+
+  // Must pass all target-independent checks in order to tail call optimize.
+  if (!Info.IsTailCall)
+    return false;
+
   CallingConv::ID CalleeCC = Info.CallConv;
   MachineFunction &MF = MIRBuilder.getMF();
   const Function &CallerF = MF.getFunction();
 
   LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
 
   if (Info.SwiftErrorVReg) {
     // TODO: We should handle this.
bool AArch64CallLowering::isEligibleForTailCallOptimization(
   // I want anyone implementing a new calling convention to think long and hard
   // about this assert.
   assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
          "Unexpected variadic calling convention");
 
   // Before we can musttail varargs, we need to forward parameters like in
   // r345641. Make sure that we don't enable musttail with varargs without
   // addressing that!
-  assert(!(Info.IsVarArg && Info.IsMustTailCall) &&
-         "musttail support for varargs not implemented yet!");
+  if (Info.IsVarArg && Info.IsMustTailCall) {
+    LLVM_DEBUG(
+        dbgs()
+        << "... Cannot handle vararg musttail functions yet.\n");
+    return false;
+  }
 
   // Verify that the incoming and outgoing arguments from the callee are
   // safe to tail call.
   if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
     LLVM_DEBUG(
         dbgs()
         << "... Caller and callee have incompatible calling conventions.\n");
     return false;

 bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
                                     CallLoweringInfo &Info) const {
   MachineFunction &MF = MIRBuilder.getMF();
   const Function &F = MF.getFunction();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   auto &DL = F.getParent()->getDataLayout();
   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
 
-  if (Info.IsMustTailCall) {
-    // TODO: Until we lower all tail calls, we should fall back on this.
-    LLVM_DEBUG(dbgs() << "Cannot lower musttail calls yet.\n");
-    return false;
-  }
-
   SmallVector<ArgInfo, 8> OutArgs;
   for (auto &OrigArg : Info.OrigArgs) {
     splitToValueTypes(OrigArg, OutArgs, DL, MRI, Info.CallConv);
     // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
     if (OrigArg.Ty->isIntegerTy(1))
       OutArgs.back().Flags[0].setZExt();
   }
 
   SmallVector<ArgInfo, 8> InArgs;
   if (!Info.OrigRet.Ty->isVoidTy())
     splitToValueTypes(Info.OrigRet, InArgs, DL, MRI, F.getCallingConv());
 
   // If we can lower as a tail call, do that instead.
-  if (Info.IsTailCall &&
-      isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs))
+  bool CanTailCallOpt =
+      isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
+
+  // We must emit a tail call if we have musttail.
+  if (Info.IsMustTailCall && !CanTailCallOpt) {
+    // There are types of incoming/outgoing arguments we can't handle yet, so
+    // it doesn't make sense to actually die here like in ISelLowering. Instead,
+    // fall back to SelectionDAG and let it try to handle this.
+    LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
+    return false;
+  }
+
+  if (CanTailCallOpt)
     return lowerTailCall(MIRBuilder, Info, OutArgs);
 
   // Find out which ABI gets to decide where things go.
   CCAssignFn *AssignFnFixed =
       TLI.CCAssignFnForCall(Info.CallConv, /*IsVarArg=*/false);
   CCAssignFn *AssignFnVarArg =
       TLI.CCAssignFnForCall(Info.CallConv, /*IsVarArg=*/true);
 

llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll

-; RUN: not llc %s -mtriple aarch64-unknown-unknown -debug-only=aarch64-call-lowering -global-isel -o - 2>&1 | FileCheck %s
+; RUN: not llc %s -mtriple aarch64-apple-darwin -debug-only=aarch64-call-lowering -global-isel -global-isel-abort=2 -o - 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
-; CHECK: Cannot lower musttail calls yet.
-; CHECK-NEXT: LLVM ERROR: unable to translate instruction: call (in function: foo)
-declare void @must_callee(i8*)
-define void @foo(i32*) {
-  musttail call void @must_callee(i8* null)
+; Verify that we fall back to SelectionDAG, and error out when we can't tail call musttail functions
+; CHECK: ... Cannot tail call externally-defined function with weak linkage for this OS.
+; CHECK-NEXT: Failed to lower musttail call as tail call
+; CHECK-NEXT: warning: Instruction selection used fallback path for caller_weak
+; CHECK-NEXT: LLVM ERROR: failed to perform tail call elimination on a call site marked musttail
+declare extern_weak void @callee_weak()
+define void @caller_weak() {
+  musttail call void @callee_weak()
   ret void
 }

llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll

define hidden swiftcc i64 @swiftself_indirect_tail(i64* swiftself %arg) {
   ; COMMON:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
   ; COMMON:   $x20 = COPY [[COPY]](p0)
   ; COMMON:   TCRETURNri [[COPY1]](p0), 0, csr_aarch64_aapcs, implicit $sp, implicit $x20
   %tmp = call i8* @pluto()
   %tmp1 = bitcast i8* %tmp to i64 (i64*)*
   %tmp2 = tail call swiftcc i64 %tmp1(i64* swiftself %arg)
   ret i64 %tmp2
 }
+
+; Verify that we can tail call musttail callees.
+declare void @must_callee(i8*)
+define void @foo(i32*) {
+  ; COMMON-LABEL: name: foo
+  ; COMMON: bb.1 (%ir-block.1):
+  ; COMMON:   liveins: $x0
+  ; COMMON:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; COMMON:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; COMMON:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C]](s64)
+  ; COMMON:   $x0 = COPY [[INTTOPTR]](p0)
+  ; COMMON:   TCRETURNdi @must_callee, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0
+  musttail call void @must_callee(i8* null)
+  ret void
+}

llvm/test/CodeGen/AArch64/vararg-tallcall.ll

 ; RUN: llc -mtriple=aarch64-windows-msvc %s -o - | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -verify-machineinstrs -mtriple=aarch64-windows-msvc %s -o - | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -verify-machineinstrs -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s
 
 target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
 
 %class.X = type { i8 }
 %struct.B = type { i32 (...)** }
 
 $"??_9B@@$BA@AA" = comdat any