[X86][fastcall][vectorcall] Move capability check before free register update

[pengfei]

Summary

When passing arguments with __fastcall or __vectorcall in 32-bit MSVC, the following arguments have chance to be passed by register if the current one failed. __regcall from ICC is on the contrary: https://godbolt.org/z/4MPbzhaMG
All the three calling conversions are not supported in GCC.

Fixes: #57737

[hans]

Instead of just referring to the bug in the commit message, can you add some text explaining the effects of this change, e.g. on the __fastcall void f(unsigned long long a, int b, int c) function in the bug and that this matches MSVC?

Did you double check whether this matches GCC too?

Please also add a release note about this since it affects the ABI.

clang/test/CodeGen/mangle-windows.c

50

Should we add tests to cover the vectorcall and regcall cases too?

[rnk]

clang/lib/CodeGen/TargetInfo.cpp

1780

I think we could improve readability here with some named variable booleans and by avoiding the lambda. These come to mind:

bool IsPtrOrInt =
bool CCUsesInReg = ...

[pengfei]

Address review comments.

[rnk]

Thanks, I have more comments, sorry I didn't add them on the first review.

clang/lib/CodeGen/TargetInfo.cpp

1782–1783

I see, no intended behavior change for regcall.

1785–1789

Seems like this can be simplified to:

// Return true to apply inreg to all legal parameters except on for MCU targets.
return !IsMCUABI;

clang/test/CodeGen/mangle-windows.c

50

This file is intended to test the C mangling. Most of the other tests don't look at the IR prototype. I think a better home for both tests is:
../clang/test/CodeGen/stdcall-fastcall.c
../clang/test/CodeGen/vectorcall.c

Do we already have coverage for this case for regcall? Please ensure we have coverage or add it.

[pengfei]

Address review comments. Thanks @rnk!

[pengfei]

Missing a !.

[rnk]

lgtm, thanks for the fix!

clang/docs/ReleaseNotes.rst

 RISC-V Support in Clang
 -----------------------
 - ``sifive-7-rv32`` and ``sifive-7-rv64`` are no longer supported for ``-mcpu``.
   Use ``sifive-e76``, ``sifive-s76``, or ``sifive-u74`` instead.
 
 X86 Support in Clang
 --------------------
 - Support ``-mindirect-branch-cs-prefix`` for call and jmp to indirect thunk.
+- Fix 32-bit ``__fastcall`` and ``__vectorcall`` ABI mismatch with MSVC.
 
 DWARF Support in Clang
 ----------------------
 
 Arm and AArch64 Support in Clang
 --------------------------------
 - ``-march`` values for targeting armv2, armv2A, armv3 and armv3M have been removed.
   Their presence gave the impression that Clang can correctly generate code for

clang/lib/CodeGen/TargetInfo.cpp

if (State.CC == llvm::CallingConv::X86_FastCall ||
 
     return false;
   }
 
   return true;
 }
 
 bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
-  if (!updateFreeRegs(Ty, State))
-    return false;
+  bool IsPtrOrInt = (getContext().getTypeSize(Ty) <= 32) &&
+                    (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
+                     Ty->isReferenceType());
 
-  if (IsMCUABI)
+  if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
+                      State.CC == llvm::CallingConv::X86_VectorCall))
     return false;
 
-  if (State.CC == llvm::CallingConv::X86_FastCall ||
-      State.CC == llvm::CallingConv::X86_VectorCall ||
-      State.CC == llvm::CallingConv::X86_RegCall) {
-    if (getContext().getTypeSize(Ty) > 32)
-      return false;
+  if (!updateFreeRegs(Ty, State))
 
-    return (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
-        Ty->isReferenceType());
-  }
+  if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
+    return false;
 
-  return true;
+  // Return true to apply inreg to all legal parameters except for MCU targets.
+  return !IsMCUABI;
 }
 
 void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
   // Vectorcall x86 works subtly different than in x64, so the format is
   // a bit different than the x64 version.  First, all vector types (not HVAs)
   // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
   // This differs from the x64 implementation, where the first 6 by INDEX get
   // registers.

clang/test/CodeGen/mangle-windows.c

 // CHECK: define dso_local x86_fastcallcc void @"\01@f7@4"
 // X64: define dso_local void @f7(
 
 void __fastcall f8(long long a) {}
 // CHECK: define dso_local x86_fastcallcc void @"\01@f8@8"
 // X64: define dso_local void @f8(
 
 void __fastcall f9(long long a, char b, char c, short d) {}
-// CHECK: define dso_local x86_fastcallcc void @"\01@f9@20"(i64 noundef %a, i8 noundef signext %b, i8 noundef signext %c, i16 noundef signext %d)
+// CHECK: define dso_local x86_fastcallcc void @"\01@f9@20"(i64 noundef %a, i8 inreg noundef signext %b, i8 inreg noundef signext %c, i16 noundef signext %d)
 // X64: define dso_local void @f9(
 
 void f12(void) {}
 // CHECK: define dso_local void @f12(
 // X64: define dso_local void @f12(
 
 void __vectorcall v1(void) {}
 // CHECK: define dso_local x86_vectorcallcc void @"\01v1@@0"(

 
 void __vectorcall v5(long long a) {}
 // CHECK: define dso_local x86_vectorcallcc void @"\01v5@@8"(
 // X64: define dso_local x86_vectorcallcc void @"\01v5@@8"(
 
 void __vectorcall v6(char a, char b) {}
 // CHECK: define dso_local x86_vectorcallcc void @"\01v6@@8"(
 // X64: define dso_local x86_vectorcallcc void @"\01v6@@16"(
+
+void __vectorcall v7(long long a, char b, char c, short d) {}
+// CHECK: define dso_local x86_vectorcallcc void @"\01v7@@20"(i64 noundef %a, i8 inreg noundef signext %b, i8 inreg noundef signext %c, i16 noundef signext %d)

clang/test/CodeGen/regcall.c

 
 struct Large { int a[5]; };
 void __regcall v4(int a, struct Large b, int c) {}
 // Win32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, %struct.Large* noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // Lin32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, %struct.Large* noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
 // Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, %struct.Large* noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
+void __regcall v5(long long a, int b, int c) {}
+// X86: define dso_local x86_regcallcc void @__regcall3__v5(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
+// X64: define dso_local x86_regcallcc void @__regcall3__v5(i64 noundef %a, i32 noundef %b, i32 noundef %c)
+
 struct HFA2 { double x, y; };
 struct HFA4 { double w, x, y, z; };
 struct HFA5 { double v, w, x, y, z; };
 
 void __regcall hfa1(int a, struct HFA4 b, int c) {}
 // X86: define dso_local x86_regcallcc void @__regcall3__hfa1(i32 inreg noundef %a, double %b.0, double %b.1, double %b.2, double %b.3, i32 inreg noundef %c)
 // X64: define dso_local x86_regcallcc void @__regcall3__hfa1(i32 noundef %a, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, i32 noundef %c)
 

clang/test/CodeGen/stdcall-fastcall.c

struct S3 {
   float x;
 };
 void __attribute__((fastcall)) foo12(struct S3 y, int x);
 void bar12(struct S3 y, int x) {
   // CHECK-LABEL: define{{.*}} void @bar12
   // CHECK: call x86_fastcallcc void @foo12(float %{{.*}}, i32 inreg noundef %
   foo12(y, x);
 }
+
+void __attribute__((fastcall)) foo13(long long a, int b, int c);
+void bar13(long long a, int b, int c) {
+  // CHECK-LABEL: define{{.*}} void @bar13
+  // CHECK: call x86_fastcallcc void @foo13(i64 noundef %{{.*}}, i32 inreg noundef %{{.*}}, i32 inreg noundef %
+  foo13(a, b, c);
+}

clang/test/CodeGen/vectorcall.c

 // X86: define dso_local x86_vectorcallcc void @"\01v3@@12"(i32 inreg noundef %a, i32 %b.0, i32 inreg noundef %c)
 // X64: define dso_local x86_vectorcallcc void @"\01v3@@24"(i32 noundef %a, i32 %b.coerce, i32 noundef %c)
 
 struct Large { int a[5]; };
 void __vectorcall v4(int a, struct Large b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v4@@28"(i32 inreg noundef %a, %struct.Large* noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, %struct.Large* noundef %b, i32 noundef %c)
 
+void __vectorcall v5(long long a, int b, int c) {}
+// X86: define dso_local x86_vectorcallcc void @"\01v5@@16"(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01v5@@24"(i64 noundef %a, i32 noundef %b, i32 noundef %c)
+
 struct HFA2 { double x, y; };
 struct HFA4 { double w, x, y, z; };
 struct HFA5 { double v, w, x, y, z; };
 
 void __vectorcall hfa1(int a, struct HFA4 b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01hfa1@@40"(i32 inreg noundef %a, %struct.HFA4 inreg %b.coerce, i32 inreg noundef %c)
 // X64: define dso_local x86_vectorcallcc void @"\01hfa1@@48"(i32 noundef %a, %struct.HFA4 inreg %b.coerce, i32 noundef %c)