[Mangler] Calculate the argument list byte count suffix correctly when returning large values

[wesleywiser]

Summary

__stdcall, __fastcall and __vectorcall return large values via a hidden pointer argument. However, the size of that argument should not be included in the argument list byte count suffix added to the function's decorated name.

This patch fixes that issue so that LLVM generates the same decorated name as MSVC does.

MSVC example: https://godbolt.org/z/nc35MKPhr

[rnk]

lgtm

For what it's worth, this issue is not observable from clang because clang does this mangling in the frontend. See https://godbolt.org/z/GM1Evjj3Y, it produces the function name \01_return_large@0. Looking at the code suggests that this is just an accident. It works because it is driven by the AST, which doesn't have the implicit sret parameter.

[rnk]

Should I go ahead and push this for you? I will include your email in the commit Author record unless you ask me not to.

[wesleywiser]

In D110719#3031053, @rnk wrote:

Should I go ahead and push this for you? I will include your email in the commit Author record unless you ask me not to.

That would be great, thank you!

Are you concerned about the build failure? I've started trying to repro locally but I don't see off hand how my changes could cause that.

[rnk]

In D110719#3031074, @wesleywiser wrote:

In D110719#3031053, @rnk wrote:

Should I go ahead and push this for you? I will include your email in the commit Author record unless you ask me not to.

That would be great, thank you!

Thanks for the patch!

Are you concerned about the build failure? I've started trying to repro locally but I don't see off hand how my changes could cause that.

I'm pretty confident that it's unrelated. The sanitizer tests are probably flaky.

llvm/lib/IR/Mangler.cpp

 static void addByteCountSuffix(raw_ostream &OS, const Function *F,
                                const DataLayout &DL) {
   // Calculate arguments size total.
   unsigned ArgWords = 0;
 
   const unsigned PtrSize = DL.getPointerSize();
 
   for (const Argument &A : F->args()) {
+    // For the purposes of the byte count suffix, structs returned by pointer
+    // do not count as function arguments.
+    if (A.hasStructRetAttr())
+      continue;
+
     // 'Dereference' type in case of byval or inalloca parameter attribute.
     uint64_t AllocSize = A.hasPassPointeeByValueCopyAttr() ?
       A.getPassPointeeByValueCopySize(DL) :
       DL.getTypeAllocSize(A.getType());
 
     // Size should be aligned to pointer size.
     ArgWords += alignTo(AllocSize, PtrSize);
   }

llvm/test/CodeGen/X86/stdcall.ll

 ; PR14410
 define x86_stdcallcc i32 @"\01DoNotMangle"(i32 %a) {
 ; CHECK: DoNotMangle:
 ; CHECK: retl $4
 entry:
   ret i32 %a
 }
 
+%struct.large_type = type { i64, i64, i64 }
+
+define x86_stdcallcc void @ReturnLargeType(%struct.large_type* noalias nocapture sret(%struct.large_type) align 8 %agg.result) {
+; CHECK: ReturnLargeType@0:
+; CHECK: retl
+entry:
+  %a = getelementptr inbounds %struct.large_type, %struct.large_type* %agg.result, i32 0, i32 0
+  store i64 123, i64* %a, align 8
+  %b = getelementptr inbounds %struct.large_type, %struct.large_type* %agg.result, i32 0, i32 1
+  store i64 456, i64* %b, align 8
+  %c = getelementptr inbounds %struct.large_type, %struct.large_type* %agg.result, i32 0, i32 2
+  store i64 789, i64* %c, align 8
+  ret void
+}
+
 @B = global %0 { void (...)* bitcast (void ()* @MyFunc to void (...)*) }, align 4
 ; CHECK: _B:
 ; CHECK: .long _MyFunc@0
 

llvm/test/CodeGen/X86/vectorcall.ll

entry:
   ret %struct.HVA4 %2
 }
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
 
 define x86_vectorcallcc void @test_mixed_7(%struct.HVA5* noalias sret(%struct.HVA5) %agg.result) {
-; X86-LABEL: test_mixed_7@@4
-; X64-LABEL: test_mixed_7@@8
+; CHECK-LABEL: test_mixed_7@@0
 ; X64:         mov{{[ql]}}  %rcx, %rax
 ; CHECK:       movaps %xmm{{[0-9]}}, 64(%{{rcx|eax}})
 ; CHECK:       movaps %xmm{{[0-9]}}, 48(%{{rcx|eax}})
 ; CHECK:       movaps %xmm{{[0-9]}}, 32(%{{rcx|eax}})
 ; CHECK:       movaps %xmm{{[0-9]}}, 16(%{{rcx|eax}})
 ; CHECK:       movaps %xmm{{[0-9]}}, (%{{rcx|eax}})
 ; CHECK:       ret{{[ql]}}
 entry: