[Reland] "Do not apply calling conventions to MSVC entry points"

[eandrews]

Summary

This patch is a second attempt at fixing a link error for MSVC entry points when calling conventions are specified using a flag.

Calling conventions specified using flags should not be applied to MSVC entry points. The default calling convention is set in this case. The default calling convention for MSVC entry points main and wmain is cdecl. For WinMain, wWinMain and DllMain, the default calling convention is stdcall on 32 bit Windows.

Explicitly specified calling conventions are applied to MSVC entry points.

For MinGW, the default calling convention for all MSVC entry points is __cdecl.

First attempt: 4cff1b40dacf6 (https://reviews.llvm.org/D87701)
Revert of first attempt: bebfc3b92d5e8

[rnk]

Seems reasonable

clang/lib/Sema/SemaDecl.cpp

11214–11215

This should only be done on 32-bit x86 platforms. I think i686-window-msvc is the more special case platform here, so I would recommend making the helper something like isDefaultStdCall, which is true for i686-windows-msvc, false for mingw and non-x86 triples, and false for main/wmain.

[eandrews]

Implemented review comment to restrict __stdcall default calling convention (for WinMain, wWinMain, and DllMain) to 32 bit Windows.

[eandrews]

clang/lib/Sema/SemaDecl.cpp

11214–11215

Thanks for the review @rnk! I think I made the change you requested.

[rnk]

lgtm, thanks!

clang/lib/Sema/SemaDecl.cpp

void Sema::CheckMain(FunctionDecl* FD, const DeclSpec& DS) {
   }
 
   if (!FD->isInvalidDecl() && FD->getDescribedFunctionTemplate()) {
     Diag(FD->getLocation(), diag::err_mainlike_template_decl) << FD;
     FD->setInvalidDecl();
   }
 }
 
+static bool isDefaultStdCall(FunctionDecl *FD, Sema &S) {
+
+  // Default calling convention for main and wmain is __cdecl
+  if (FD->getName() == "main" || FD->getName() == "wmain")
+    return false;
+
+  // Default calling convention for MinGW is __cdecl
+  const llvm::Triple &T = S.Context.getTargetInfo().getTriple();
+  if (T.isWindowsGNUEnvironment())
+    return false;
+
+  // Default calling convention for WinMain, wWinMain and DllMain
+  // is __stdcall on 32 bit Windows
+  if (T.isOSWindows() && T.getArch() == llvm::Triple::x86)
+    return true;
+
+  return false;
+}
+
 void Sema::CheckMSVCRTEntryPoint(FunctionDecl *FD) {
   QualType T = FD->getType();
   assert(T->isFunctionType() && "function decl is not of function type");
   const FunctionType *FT = T->castAs<FunctionType>();
 
   // Set an implicit return of 'zero' if the function can return some integral,
   // enumeration, pointer or nullptr type.
   if (FT->getReturnType()->isIntegralOrEnumerationType() ||
       FT->getReturnType()->isAnyPointerType() ||
       FT->getReturnType()->isNullPtrType())
     // DllMain is exempt because a return value of zero means it failed.
     if (FD->getName() != "DllMain")
       FD->setHasImplicitReturnZero(true);
 
+  // Explicity specified calling conventions are applied to MSVC entry points
+  if (!hasExplicitCallingConv(T)) {
+    if (isDefaultStdCall(FD, *this)) {
+      if (FT->getCallConv() != CC_X86StdCall) {
+        FT = Context.adjustFunctionType(
+            FT, FT->getExtInfo().withCallingConv(CC_X86StdCall));
+        FD->setType(QualType(FT, 0));
+      }
+    } else if (FT->getCallConv() != CC_C) {
+      FT = Context.adjustFunctionType(FT,
+                                      FT->getExtInfo().withCallingConv(CC_C));
+      FD->setType(QualType(FT, 0));
+    }
+  }
+
   if (!FD->isInvalidDecl() && FD->getDescribedFunctionTemplate()) {
     Diag(FD->getLocation(), diag::err_mainlike_template_decl) << FD;
     FD->setInvalidDecl();
   }
 }
 
 bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
   // FIXME: Need strict checking.  In C89, we need to check for

clang/test/CodeGenCXX/default_calling_conv.cpp

 // RUN: %clang_cc1 -triple i386-unknown-linux-gnu -fdefault-calling-conv=cdecl -emit-llvm -o - %s | FileCheck %s --check-prefix=CDECL --check-prefix=ALL
 // RUN: %clang_cc1 -triple i786-unknown-linux-gnu -target-feature +sse4.2 -fdefault-calling-conv=fastcall -emit-llvm -o - %s | FileCheck %s --check-prefix=FASTCALL --check-prefix=ALL
 // RUN: %clang_cc1 -triple i486-unknown-linux-gnu -fdefault-calling-conv=stdcall -emit-llvm -o - %s | FileCheck %s --check-prefix=STDCALL --check-prefix=ALL
 // RUN: %clang_cc1 -triple i486-unknown-linux-gnu -mrtd -emit-llvm -o - %s | FileCheck %s --check-prefix=STDCALL --check-prefix=ALL
 // RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=vectorcall -emit-llvm -o - %s | FileCheck %s --check-prefix=VECTORCALL --check-prefix=ALL
 // RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=regcall -emit-llvm -o - %s | FileCheck %s --check-prefix=REGCALL --check-prefix=ALL
+// RUN: %clang_cc1 -triple i686-pc-win32 -fdefault-calling-conv=vectorcall -emit-llvm -o - %s -DWINDOWS | FileCheck %s --check-prefix=WIN32
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fdefault-calling-conv=vectorcall -emit-llvm -o - %s -DWINDOWS | FileCheck %s --check-prefix=WIN64
+// RUN: %clang_cc1 -triple i686-pc-win32 -emit-llvm -o - %s -DEXPLICITCC | FileCheck %s --check-prefix=EXPLICITCC
 
 // CDECL: define{{.*}} void @_Z5test1v
 // FASTCALL: define{{.*}} x86_fastcallcc void @_Z5test1v
 // STDCALL: define{{.*}} x86_stdcallcc void @_Z5test1v
 // VECTORCALL: define{{.*}} x86_vectorcallcc void @_Z5test1v
 // REGCALL: define{{.*}} x86_regcallcc void @_Z17__regcall3__test1v
 void test1() {}
 
void test() {
   A a;
   a.test_member();
 }
 
 // ALL: define{{.*}} i32 @main
 int main() {
   return 1;
 }
+
+#ifdef WINDOWS
+// WIN32: define dso_local i32 @wmain
+// WIN64: define dso_local i32 @wmain
+int wmain() {
+  return 1;
+}
+// WIN32: define dso_local x86_stdcallcc i32 @WinMain
+// WIN64: define dso_local i32 @WinMain
+int WinMain() {
+  return 1;
+}
+// WIN32: define dso_local x86_stdcallcc i32 @wWinMain
+// WIN64: define dso_local i32 @wWinMain
+int wWinMain() {
+  return 1;
+}
+// WIN32: define dso_local x86_stdcallcc i32 @DllMain
+// WIN64: define dso_local i32 @DllMain
+int DllMain() {
+  return 1;
+}
+#endif // Windows
+
+#ifdef EXPLICITCC
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @wmain
+int __fastcall wmain() {
+  return 1;
+}
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @WinMain
+int __fastcall WinMain() {
+  return 1;
+}
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @wWinMain
+int __fastcall wWinMain() {
+  return 1;
+}
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @DllMain
+int __fastcall DllMain() {
+  return 1;
+}
+#endif // ExplicitCC