[ARM] Fix tail call validity checking for varargs calls.

[efriedma]

Summary

If a varargs function is calling a non-varargs function, or vice versa, make sure we use the correct "varargs" bit for each.

Fixes https://bugs.llvm.org/show_bug.cgi?id=45234

[DavidSpickett]

llvm/test/CodeGen/ARM/tail-call-results.ll

5

Can you add a comment here with the purpose of the test?

If I understand correctly it's checking that vararg->vararg, or normal->normal will tail call but a mix of the two won't.

[efriedma]

Update test; hopefully the comments make sense.

[DavidSpickett]

LGTM

llvm/lib/Target/ARM/ARMISelLowering.cpp

if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
     const Triple &TT = getTargetMachine().getTargetTriple();
     if (GV->hasExternalWeakLinkage() &&
         (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
       return false;
   }
 
   // Check that the call results are passed in the same way.
   LLVMContext &C = *DAG.getContext();
-  if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
-                                  CCAssignFnForReturn(CalleeCC, isVarArg),
-                                  CCAssignFnForReturn(CallerCC, isVarArg)))
+  if (!CCState::resultsCompatible(
+          getEffectiveCallingConv(CalleeCC, isVarArg),
+          getEffectiveCallingConv(CallerCC, CallerF.isVarArg()), MF, C, Ins,
+          CCAssignFnForReturn(CallerCC, CallerF.isVarArg())))
     return false;
   // The callee has to preserve all registers the caller needs to preserve.
   const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
   if (CalleeCC != CallerCC) {
     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
       return false;

llvm/test/CodeGen/ARM/tail-call-results.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv7-linux-gnueabihf %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv7-linux-gnueabi %s -o - | FileCheck -check-prefix=SOFTFLOAT %s
+; RUN: llc -mtriple=thumbv7-linux-gnueabihf -disable-tail-calls %s -o - | FileCheck -check-prefix=HF-NOTAIL %s
+
+; On hard-float targets, the register used to store a float return value
+; changes if the call signature is varargs. The HF-NOTAIL lines are there to
+; easily see when this happens.
+
+declare float @callee_float()
+declare i32 @callee_int()
+declare float @callee_float_vararg(i32, ...)
+declare i32 @callee_int_vararg(i32, ...)
+
+define float @caller_float__callee_float() {
+; CHECK-LABEL: caller_float__callee_float:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    b callee_float
+;
+; SOFTFLOAT-LABEL: caller_float__callee_float:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    b callee_float
+;
+; HF-NOTAIL-LABEL: caller_float__callee_float:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    bl callee_float
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call float @callee_float()
+  ret float %r
+}
+
+define float @caller_float__callee_float_vararg() {
+; CHECK-LABEL: caller_float__callee_float_vararg:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    bl callee_float_vararg
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    pop {r7, pc}
+;
+; SOFTFLOAT-LABEL: caller_float__callee_float_vararg:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    movs r0, #0
+; SOFTFLOAT-NEXT:    b callee_float_vararg
+;
+; HF-NOTAIL-LABEL: caller_float__callee_float_vararg:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    movs r0, #0
+; HF-NOTAIL-NEXT:    bl callee_float_vararg
+; HF-NOTAIL-NEXT:    vmov s0, r0
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call float (i32, ...) @callee_float_vararg(i32 0)
+  ret float %r
+}
+
+define float @caller_float_vararg__callee_float(i32, ...) {
+; CHECK-LABEL: caller_float_vararg__callee_float:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    bl callee_float
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    pop {r7, pc}
+;
+; SOFTFLOAT-LABEL: caller_float_vararg__callee_float:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    b callee_float
+;
+; HF-NOTAIL-LABEL: caller_float_vararg__callee_float:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    bl callee_float
+; HF-NOTAIL-NEXT:    vmov r0, s0
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call float @callee_float()
+  ret float %r
+}
+
+define float @caller_float_vararg__callee_float_vararg(i32, ...) {
+; CHECK-LABEL: caller_float_vararg__callee_float_vararg:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    b callee_float_vararg
+;
+; SOFTFLOAT-LABEL: caller_float_vararg__callee_float_vararg:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    movs r0, #0
+; SOFTFLOAT-NEXT:    b callee_float_vararg
+;
+; HF-NOTAIL-LABEL: caller_float_vararg__callee_float_vararg:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    movs r0, #0
+; HF-NOTAIL-NEXT:    bl callee_float_vararg
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call float (i32, ...) @callee_float_vararg(i32 0)
+  ret float %r
+}
+
+define i32 @caller_int__callee_int() {
+; CHECK-LABEL: caller_int__callee_int:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    b callee_int
+;
+; SOFTFLOAT-LABEL: caller_int__callee_int:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    b callee_int
+;
+; HF-NOTAIL-LABEL: caller_int__callee_int:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    bl callee_int
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call i32 @callee_int()
+  ret i32 %r
+}
+
+define i32 @caller_int__callee_int_vararg() {
+; CHECK-LABEL: caller_int__callee_int_vararg:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    b callee_int_vararg
+;
+; SOFTFLOAT-LABEL: caller_int__callee_int_vararg:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    movs r0, #0
+; SOFTFLOAT-NEXT:    b callee_int_vararg
+;
+; HF-NOTAIL-LABEL: caller_int__callee_int_vararg:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    movs r0, #0
+; HF-NOTAIL-NEXT:    bl callee_int_vararg
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call i32 (i32, ...) @callee_int_vararg(i32 0)
+  ret i32 %r
+}
+
+define i32 @caller_int_vararg__callee_int(i32, ...) {
+; CHECK-LABEL: caller_int_vararg__callee_int:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    b callee_int
+;
+; SOFTFLOAT-LABEL: caller_int_vararg__callee_int:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    b callee_int
+;
+; HF-NOTAIL-LABEL: caller_int_vararg__callee_int:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    bl callee_int
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call i32 @callee_int()
+  ret i32 %r
+}
+
+define i32 @caller_int_vararg__callee_int_vararg(i32, ...) {
+; CHECK-LABEL: caller_int_vararg__callee_int_vararg:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    b callee_int_vararg
+;
+; SOFTFLOAT-LABEL: caller_int_vararg__callee_int_vararg:
+; SOFTFLOAT:       @ %bb.0:
+; SOFTFLOAT-NEXT:    movs r0, #0
+; SOFTFLOAT-NEXT:    b callee_int_vararg
+;
+; HF-NOTAIL-LABEL: caller_int_vararg__callee_int_vararg:
+; HF-NOTAIL:       @ %bb.0:
+; HF-NOTAIL-NEXT:    .save {r7, lr}
+; HF-NOTAIL-NEXT:    push {r7, lr}
+; HF-NOTAIL-NEXT:    movs r0, #0
+; HF-NOTAIL-NEXT:    bl callee_int_vararg
+; HF-NOTAIL-NEXT:    pop {r7, pc}
+  %r = tail call i32 (i32, ...) @callee_int_vararg(i32 0)
+  ret i32 %r
+}