[AArch64][FastISel] Handle call with multiple return regs

[aengelke]

Summary

The code closely follows the X86 back-end. Applications that make heavy
use of {i64, i64} returns to use two registers strongly benefit from the
reduced number of SelectionDAG fallbacks.

[aengelke]

Gentle ping

[efriedma]

Can you verify this bails out correctly for a function that returns {i64, i64, i64, i64, i64, i64, i64, i64, i64, i64}?

llvm/test/CodeGen/AArch64/fast-isel-call-struct-return.ll

8

I don't usually like CHECK-NOT checks... is there some other way you could verify this? (Maybe use "-stats" output?)

[aengelke]

Add test case for sret fallback and replace CHECK-NOT with -fast-isel-abort=3

[aengelke]

(no change, re-trigger CI)

[efriedma]

LGTM

llvm/lib/Target/AArch64/AArch64FastISel.cpp

private:
   unsigned materializeFP(const ConstantFP *CFP, MVT VT);
   unsigned materializeGV(const GlobalValue *GV);
 
   // Call handling routines.
 private:
   CCAssignFn *CCAssignFnForCall(CallingConv::ID CC) const;
   bool processCallArgs(CallLoweringInfo &CLI, SmallVectorImpl<MVT> &ArgVTs,
                        unsigned &NumBytes);
-  bool finishCall(CallLoweringInfo &CLI, MVT RetVT, unsigned NumBytes);
+  bool finishCall(CallLoweringInfo &CLI, unsigned NumBytes);
 
 public:
   // Backend specific FastISel code.
   unsigned fastMaterializeAlloca(const AllocaInst *AI) override;
   unsigned fastMaterializeConstant(const Constant *C) override;
   unsigned fastMaterializeFloatZero(const ConstantFP* CF) override;
 
   explicit AArch64FastISel(FunctionLoweringInfo &FuncInfo,
if (VA.isRegLoc() && !VA.needsCustom()) {
 
       if (!emitStore(ArgVT, ArgReg, Addr, MMO))
         return false;
     }
   }
   return true;
 }
 
-bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT,
-                                 unsigned NumBytes) {
+bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, unsigned NumBytes) {
   CallingConv::ID CC = CLI.CallConv;
 
   // Issue CALLSEQ_END
   unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackUp))
     .addImm(NumBytes).addImm(0);
 
-  // Now the return value.
-  if (RetVT != MVT::isVoid) {
-    SmallVector<CCValAssign, 16> RVLocs;
-    CCState CCInfo(CC, false, *FuncInfo.MF, RVLocs, *Context);
-    CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC));
+  // Now the return values.
+  CCInfo.AnalyzeCallResult(CLI.Ins, CCAssignFnForCall(CC));
 
-    // Only handle a single return value.
-    if (RVLocs.size() != 1)
-      return false;
-
-    // Copy all of the result registers out of their specified physreg.
-    MVT CopyVT = RVLocs[0].getValVT();
+  Register ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
+  for (unsigned i = 0; i != RVLocs.size(); ++i) {
+    CCValAssign &VA = RVLocs[i];
+    MVT CopyVT = VA.getValVT();
+    unsigned CopyReg = ResultReg + i;
 
     // TODO: Handle big-endian results
     if (CopyVT.isVector() && !Subtarget->isLittleEndian())
       return false;
 
-    Register ResultReg = createResultReg(TLI.getRegClassFor(CopyVT));
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
-            TII.get(TargetOpcode::COPY), ResultReg)
-        .addReg(RVLocs[0].getLocReg());
-    CLI.InRegs.push_back(RVLocs[0].getLocReg());
+    // Copy result out of their specified physreg.
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
+            CopyReg)
+        .addReg(VA.getLocReg());
+    CLI.InRegs.push_back(VA.getLocReg());
+  }
 
-    CLI.ResultReg = ResultReg;
-    CLI.NumResultRegs = 1;
-  }
+  CLI.NumResultRegs = RVLocs.size();
 
   return true;
 }
 
 bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   CallingConv::ID CC  = CLI.CallConv;
   bool IsTailCall     = CLI.IsTailCall;
   bool IsVarArg       = CLI.IsVarArg;
bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   // FIXME: Add large code model support for ELF.
   if (CM == CodeModel::Large && !Subtarget->isTargetMachO())
     return false;
 
   // Let SDISel handle vararg functions.
   if (IsVarArg)
     return false;
 
-  // FIXME: Only handle *simple* calls for now.
-  MVT RetVT;
-  if (CLI.RetTy->isVoidTy())
-    RetVT = MVT::isVoid;
-  else if (!isTypeLegal(CLI.RetTy, RetVT))
-    return false;
-
   for (auto Flag : CLI.OutFlags)
     if (Flag.isInReg() || Flag.isSRet() || Flag.isNest() || Flag.isByVal() ||
         Flag.isSwiftSelf() || Flag.isSwiftAsync() || Flag.isSwiftError())
       return false;
 
   // Set up the argument vectors.
   SmallVector<MVT, 16> OutVTs;
   OutVTs.reserve(CLI.OutVals.size());
bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
 
   // Add a register mask with the call-preserved registers.
   // Proper defs for return values will be added by setPhysRegsDeadExcept().
   MIB.addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
 
   CLI.Call = MIB;
 
   // Finish off the call including any return values.
-  return finishCall(CLI, RetVT, NumBytes);
+  return finishCall(CLI, NumBytes);
 }
 
 bool AArch64FastISel::isMemCpySmall(uint64_t Len, MaybeAlign Alignment) {
   if (Alignment)
     return Len / Alignment->value() <= 4;
   else
     return Len < 32;
 }

llvm/test/CodeGen/AArch64/fast-isel-call-struct-return-fallback.ll

+; RUN: llc -fast-isel -pass-remarks-missed=isel < %s 2>&1 >/dev/null | FileCheck -check-prefix=STDERR -allow-empty %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+declare { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s10i64()
+
+define i64 @call_ret_s10i64() {
+; STDERR: FastISel missed call:   %ret = call { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s10i64() (in function: call_ret_s10i64)
+  %ret = call { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s10i64()
+  %ext0 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 0
+  %ext1 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 1
+  %sum = add i64 %ext0, %ext1
+  ret i64 %sum
+}

llvm/test/CodeGen/AArch64/fast-isel-call-struct-return.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no-generate-body-for-unused-prefixes
+; RUN: llc -fast-isel -fast-isel-abort=3 < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+declare { i64, i64 } @ret_s2i64()
+
+define i64 @call_ret_s2i64() {
+; CHECK-LABEL: call_ret_s2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl ret_s2i64
+; CHECK-NEXT:    add x0, x0, x1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ret = call { i64, i64 } @ret_s2i64()
+  %ext0 = extractvalue { i64, i64 } %ret, 0
+  %ext1 = extractvalue { i64, i64 } %ret, 1
+  %sum = add i64 %ext0, %ext1
+  ret i64 %sum
+}
+
+declare { i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s8i64()
+
+define i64 @call_ret_s8i64() {
+; CHECK-LABEL: call_ret_s8i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl ret_s8i64
+; CHECK-NEXT:    add x0, x0, x7
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ret = call { i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s8i64()
+  %ext0 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 0
+  %ext7 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 7
+  %sum = add i64 %ext0, %ext7
+  ret i64 %sum
+}