lld-link: Emit a warning if one of {main,wmain} and {WinMain,wWinMain} are both present and no explicit /subsystem: flag is passed.

[thakis]

Summary

Similar to link.exe's LNK4031.

[ruiu]

LGTM

[thakis]

r340420, thanks!

COFF/Driver.cpp

StringRef LinkerDriver::findDefaultEntry() {
   if (findUnderscoreMangle(FindMain ? "wmain" : "wmainCRTStartup"))
     return mangle("wmainCRTStartup");
   return "";
 }
 
 WindowsSubsystem LinkerDriver::inferSubsystem() {
   if (Config->DLL)
     return IMAGE_SUBSYSTEM_WINDOWS_GUI;
-  if (findUnderscoreMangle("main") || findUnderscoreMangle("wmain"))
+  bool HaveMain = findUnderscoreMangle("main");
+  bool HaveWMain = findUnderscoreMangle("wmain");
+  bool HaveWinMain = findUnderscoreMangle("WinMain");
+  bool HaveWWinMain = findUnderscoreMangle("wWinMain");
+  if (HaveMain || HaveWMain) {
+    if (HaveWinMain || HaveWWinMain) {
+      warn("found " + std::string(HaveMain ? "main" : "wmain") + " and " +
+           std::string(HaveWinMain ? "WinMain" : "wWinMain") +
+           "; defaulting to /subsystem:console");
+    }
     return IMAGE_SUBSYSTEM_WINDOWS_CUI;
-  if (findUnderscoreMangle("WinMain") || findUnderscoreMangle("wWinMain"))
+  }
+  if (HaveWinMain || HaveWWinMain)
     return IMAGE_SUBSYSTEM_WINDOWS_GUI;
   return IMAGE_SUBSYSTEM_UNKNOWN;
 }
 
 static uint64_t getDefaultImageBase() {
   if (Config->is64())
     return Config->DLL ? 0x180000000 : 0x140000000;
   return Config->DLL ? 0x10000000 : 0x400000;

test/COFF/subsystem-inference2.test

+# RUN: yaml2obj %s > %t.obj
+# RUN: lld-link /out:%t.exe %t.obj 2>&1 | FileCheck -check-prefix=WARN %s
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck %s
+
+# WARN: warning: found main and WinMain; defaulting to /subsystem:console
+# CHECK:     Subsystem: IMAGE_SUBSYSTEM_WINDOWS_CUI
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: []
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     B82A000000C3
+symbols:
+  - Name:            .text
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          6
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          0
+  - Name:            main
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            WinMain
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            mainCRTStartup
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            WinMainCRTStartup
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...