[InlineCost] Mark functions accessing varargs as not viable.

[fhahn]

Summary

This prevents functions accessing varargs from being inlined if they
have the alwaysinline attribute.

[efriedma]

(The original patch didn't make it into 6.0, so we don't need to cherry-pick this, I think.)

Do we need to check for musttail calls, since they can also access varargs?

[fhahn]

In D42556#988451, @efriedma wrote:

(The original patch didn't make it into 6.0, so we don't need to cherry-pick this, I think.)

Ah yes, the original patch went in just after the branch.

Do we need to check for musttail calls, since they can also access varargs?

I do not think so: when inlining a function containing a musttail call, InlineFunction should forward the varargs appropriately.

[efriedma]

LGTM.

I do not think so: when inlining a function containing a musttail call, InlineFunction should forward the varargs appropriately.

Right.

lib/Analysis/InlineCost.cpp

for (auto &II : *BI) {
         return false;
 
       // Disallow calls which expose returns-twice to a function not previously
       // attributed as such.
       if (!ReturnsTwice && CS.isCall() &&
           cast<CallInst>(CS.getInstruction())->canReturnTwice())
         return false;
 
+      if (CS.getCalledFunction())
+        switch (CS.getCalledFunction()->getIntrinsicID()) {
+        default:
+          break;
         // Disallow inlining functions that call @llvm.localescape. Doing this
         // correctly would require major changes to the inliner.
-      if (CS.getCalledFunction() &&
-          CS.getCalledFunction()->getIntrinsicID() ==
-              llvm::Intrinsic::localescape)
+        case llvm::Intrinsic::localescape:
+        // Disallow inlining of functions that access VarArgs.
+        case llvm::Intrinsic::vastart:
+        case llvm::Intrinsic::vaend:
           return false;
         }
     }
+  }
 
   return true;
 }
 
 // APIs to create InlineParams based on command line flags and/or other
 // parameters.
 
 InlineParams llvm::getInlineParams(int Threshold) {

test/Transforms/Inline/inline-varargs.ll

entry:
   %vargs = alloca i8*, align 8
   %vargs.ptr = bitcast i8** %vargs to i8*
   call void @llvm.va_start(i8* %vargs.ptr)
   %va1 = va_arg i8** %vargs, i32
   call void @llvm.va_end(i8* %vargs.ptr)
   ret i32 %va1
 }
 
+define internal i32 @varg_accessed_alwaysinline(...) alwaysinline {
+entry:
+  %vargs = alloca i8*, align 8
+  %vargs.ptr = bitcast i8** %vargs to i8*
+  call void @llvm.va_start(i8* %vargs.ptr)
+  %va1 = va_arg i8** %vargs, i32
+  call void @llvm.va_end(i8* %vargs.ptr)
+  ret i32 %va1
+}
+
 define i32 @call_vargs() {
-  %res = call i32 (...) @varg_accessed(i32 10)
+  %res1 = call i32 (...) @varg_accessed(i32 10)
+  %res2 = call i32 (...) @varg_accessed_alwaysinline(i32 15)
+  %res = add i32 %res1, %res2
   ret i32 %res
 }
 ; CHECK-LABEL: @call_vargs
-; CHECK: %res = call i32 (...) @varg_accessed(i32 10)
+; CHECK: %res1 = call i32 (...) @varg_accessed(i32 10)
+; CHECK-NEXT: %res2 = call i32 (...) @varg_accessed_alwaysinline(i32 15)
 
 declare void @llvm.va_start(i8*)
 declare void @llvm.va_end(i8*)
 
 ; CHECK: attributes [[FN_ATTRS]] = { "foo"="bar" }
 attributes #0 = { "foo"="bar" }