[GlobalOpt] Don't remove inalloca from varargs functions

[aeubanks]

Summary

Varargs and inalloca have a weird interaction where varargs are actually
passed via the inalloca alloca. Removing inalloca breaks the varargs
because they're still not passed as separate arguments.

Fixes #58718

[rnk]

lgtm

llvm/lib/Transforms/IPO/GlobalOpt.cpp

if (!F.hasLocalLinkage())
       continue;
 
     // If we have an inalloca parameter that we can safely remove the
     // inalloca attribute from, do so. This unlocks optimizations that
     // wouldn't be safe in the presence of inalloca.
     // FIXME: We should also hoist alloca affected by this to the entry
     // block if possible.
     if (F.getAttributes().hasAttrSomewhere(Attribute::InAlloca) &&
-        !F.hasAddressTaken() && !hasMustTailCallers(&F)) {
+        !F.hasAddressTaken() && !hasMustTailCallers(&F) && !F.isVarArg()) {
       RemoveAttribute(&F, Attribute::InAlloca);
       Changed = true;
     }
 
     // FIXME: handle invokes
     // FIXME: handle musttail
     if (F.getAttributes().hasAttrSomewhere(Attribute::Preallocated)) {
       if (!F.hasAddressTaken() && !hasMustTailCallers(&F) &&

llvm/test/Transforms/GlobalOpt/inalloca-varargs.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt -passes=globalopt -S < %s | FileCheck %s
+
+define i32 @main(ptr %a) {
+; CHECK-LABEL: define {{[^@]+}}@main
+; CHECK-SAME: (ptr [[A:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ ptr, i32 }>, align 4
+; CHECK-NEXT:    store ptr [[A]], ptr [[ARGMEM]], align 8
+; CHECK-NEXT:    [[G0:%.*]] = getelementptr inbounds <{ ptr, i32 }>, ptr [[ARGMEM]], i32 0, i32 1
+; CHECK-NEXT:    store i32 5, ptr [[G0]], align 4
+; CHECK-NEXT:    [[CALL3:%.*]] = call i32 (ptr, ...) @i(ptr inalloca(ptr) [[ARGMEM]])
+; CHECK-NEXT:    ret i32 [[CALL3]]
+;
+  %argmem = alloca inalloca <{ ptr, i32 }>, align 4
+  store ptr %a, ptr %argmem, align 8
+  %g0 = getelementptr inbounds <{ ptr, i32 }>, ptr %argmem, i32 0, i32 1
+  store i32 5, ptr %g0, align 4
+  %call3 = call i32 (ptr, ...) @i(ptr inalloca(ptr) %argmem)
+  ret i32 %call3
+}
+
+define internal i32 @i(ptr inalloca(ptr) %a, ...) {
+; CHECK-LABEL: define {{[^@]+}}@i
+; CHECK-SAME: (ptr inalloca(ptr) [[A:%.*]], ...) unnamed_addr {
+; CHECK-NEXT:    [[AP:%.*]] = alloca ptr, align 4
+; CHECK-NEXT:    call void @llvm.va_start(ptr [[AP]])
+; CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[AP]], align 4
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARGP_CUR]], align 4
+; CHECK-NEXT:    ret i32 [[L]]
+;
+  %ap = alloca ptr, align 4
+  call void @llvm.va_start(ptr %ap)
+  %argp.cur = load ptr, ptr %ap, align 4
+  %l = load i32, ptr %argp.cur, align 4
+  ret i32 %l
+}
+
+declare void @llvm.va_start(ptr)