lld-link: Fix subsystem inference for non-console apps on 32-bit, and fix entry point inference on 32-bit with /nodefaultlib

[thakis]

Summary

LinkerDriver::inferSubsystem() used to do Symtab->findUnderscore("WinMain"), but WinMain is stdcall in 32-bit and is hence is called _WinMain@16. Instead, Symtab->findMangle(mangle("WinMain")) needs to be called.

But since LinkerDriver::inferSubsystem() and LinkerDriver::findDefaultEntry() both need to call this, introduce a common helper function for this and call it from both places. (Also call it for "main" for consistency, even though findUnderscore() is enough for main since that's __cdecl on 32-bit).

This also exposed a bug for /nodefaultlib entrypoint inference: The code here called findMangle(Sym) instead of findMangle(mangle(Sym)), again doing the wrong thing on 32-bit. Fix that too.

[ruiu]

LGTM

COFF/Driver.cpp

418

I believe this can be a non-member static function.

[thakis]

comment

[thakis]

Thanks!

COFF/Driver.cpp

418

Yes, if I make ::mangle() a local static too (which in return requires moving it up a bit). Want me to do this in this patch? I'm attaching the new diff so that you can see how it looks.

[thakis]

Landed 2nd version in r338877.

COFF/Driver.h

private:
   // We don't want to add the same file more than once.
   // Files are uniquified by their filesystem and file number.
   std::set<llvm::sys::fs::UniqueID> VisitedFiles;
 
   std::set<std::string> VisitedLibs;
 
   Symbol *addUndefined(StringRef Sym);
   StringRef mangle(StringRef Sym);
+  bool findUnderscoreMangle(StringRef Sym);
 
   // Windows specific -- "main" is not the only main function in Windows.
   // You can choose one from these four -- {w,}{WinMain,main}.
   // There are four different entry point functions for them,
   // {w,}{WinMain,main}CRTStartup, respectively. The linker needs to
   // choose the right one depending on which "main" function is defined.
   // This function looks up the symbol table and resolve corresponding
   // entry point name.

COFF/Driver.cpp

Symbol *LinkerDriver::addUndefined(StringRef Name) {
   Symbol *B = Symtab->addUndefined(Name);
   if (!B->IsGCRoot) {
     B->IsGCRoot = true;
     Config->GCRoot.push_back(B);
   }
   return B;
 }
 
-// Symbol names are mangled by appending "_" prefix on x86.
+// Symbol names are mangled by prepending "_" on x86.
 StringRef LinkerDriver::mangle(StringRef Sym) {
   assert(Config->Machine != IMAGE_FILE_MACHINE_UNKNOWN);
   if (Config->Machine == I386)
     return Saver.save("_" + Sym);
   return Sym;
 }
 
+bool LinkerDriver::findUnderscoreMangle(StringRef Sym) {
+  StringRef Entry = Symtab->findMangle(mangle(Sym));
+  return !Entry.empty() && !isa<Undefined>(Symtab->find(Entry));
+}
+
 // Windows specific -- find default entry point name.
 //
 // There are four different entry point functions for Windows executables,
 // each of which corresponds to a user-defined "main" function. This function
 // infers an entry point from a user-defined "main" function.
 StringRef LinkerDriver::findDefaultEntry() {
   // As a special case, if /nodefaultlib is given, we directly look for an
   // entry point. This is because, if no default library is linked, users
   // need to define an entry point instead of a "main".
   if (Config->NoDefaultLibAll) {
     for (StringRef S : {"mainCRTStartup", "wmainCRTStartup",
                         "WinMainCRTStartup", "wWinMainCRTStartup"}) {
-      StringRef Entry = Symtab->findMangle(S);
-      if (!Entry.empty() && !isa<Undefined>(Symtab->find(Entry)))
+      if (findUnderscoreMangle(S))
         return mangle(S);
     }
     return "";
   }
 
   // User-defined main functions and their corresponding entry points.
   static const char *Entries[][2] = {
       {"main", "mainCRTStartup"},
       {"wmain", "wmainCRTStartup"},
       {"WinMain", "WinMainCRTStartup"},
       {"wWinMain", "wWinMainCRTStartup"},
   };
   for (auto E : Entries) {
-    StringRef Entry = Symtab->findMangle(mangle(E[0]));
-    if (!Entry.empty() && !isa<Undefined>(Symtab->find(Entry)))
+    if (findUnderscoreMangle(E[0]))
       return mangle(E[1]);
   }
   return "";
 }
 
 WindowsSubsystem LinkerDriver::inferSubsystem() {
   if (Config->DLL)
     return IMAGE_SUBSYSTEM_WINDOWS_GUI;
-  if (Symtab->findUnderscore("main") || Symtab->findUnderscore("wmain"))
+  if (findUnderscoreMangle("main") || findUnderscoreMangle("wmain"))
     return IMAGE_SUBSYSTEM_WINDOWS_CUI;
-  if (Symtab->findUnderscore("WinMain") || Symtab->findUnderscore("wWinMain"))
+  if (findUnderscoreMangle("WinMain") || findUnderscoreMangle("wWinMain"))
     return IMAGE_SUBSYSTEM_WINDOWS_GUI;
   return IMAGE_SUBSYSTEM_UNKNOWN;
 }
 
 static uint64_t getDefaultImageBase() {
   if (Config->is64())
     return Config->DLL ? 0x180000000 : 0x140000000;
   return Config->DLL ? 0x10000000 : 0x400000;

test/COFF/entry-inference3.test

-# RUN: yaml2obj < %s > %t.obj
+# RUN: sed -e s/ENTRYNAME/mainCRTStartup/ %s | yaml2obj > %t.obj
+# RUN: not lld-link /out:%t.exe %t.obj /verbose /nodefaultlib > %t.log 2>&1
+# RUN: FileCheck %s < %t.log
+
+# RUN: sed -e s/ENTRYNAME/?mainCRTStartup@@YAHXZ/ %s | yaml2obj > %t.obj
 # RUN: not lld-link /out:%t.exe %t.obj /verbose /nodefaultlib > %t.log 2>&1
 # RUN: FileCheck %s < %t.log
 
 # CHECK: Entry name inferred: mainCRTStartup
 
 --- !COFF
 header:
   Machine:         IMAGE_FILE_MACHINE_AMD64
- Name:            .text
     ComplexType:     IMAGE_SYM_DTYPE_NULL
     StorageClass:    IMAGE_SYM_CLASS_STATIC
     SectionDefinition:
       Length:          6
       NumberOfRelocations: 0
       NumberOfLinenumbers: 0
       CheckSum:        0
       Number:          0
-  - Name:            mainCRTStartup
+  - Name:            "ENTRYNAME"
     Value:           0
     SectionNumber:   1
     SimpleType:      IMAGE_SYM_TYPE_NULL
     ComplexType:     IMAGE_SYM_DTYPE_NULL
     StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
 ...

test/COFF/entry-inference332.test

+# RUN: sed -e s/ENTRYNAME/_mainCRTStartup/ %s | yaml2obj > %t.obj
+# RUN: not lld-link /out:%t.exe %t.obj /verbose /nodefaultlib > %t.log 2>&1
+# RUN: FileCheck %s < %t.log
+
+# RUN: sed -e s/ENTRYNAME/?mainCRTStartup@@YAHXZ/ %s | yaml2obj > %t.obj
+# RUN: not lld-link /out:%t.exe %t.obj /verbose /nodefaultlib > %t.log 2>&1
+# RUN: FileCheck %s < %t.log
+
+# CHECK: Entry name inferred: _mainCRTStartup
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_I386
+  Characteristics: []
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     B82A000000C3
+symbols:
+  - Name:            .text
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          6
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          0
+  - Name:            "ENTRYNAME"
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...

test/COFF/subsystem-inference32.test

+# RUN: sed -e s/ENTRYNAME/_main/ %s | yaml2obj > %t.obj
+# RUN: lld-link /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=MAIN %s
+
+# RUN: sed s/ENTRYNAME/_wmain/ %s | yaml2obj > %t.obj
+# RUN: lld-link /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=WMAIN %s
+
+# RUN: sed s/ENTRYNAME/_WinMain@16/ %s | yaml2obj > %t.obj
+# RUN: lld-link /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=WINMAIN %s
+
+# RUN: sed s/ENTRYNAME/_wWinMain@16/ %s | yaml2obj > %t.obj
+# RUN: lld-link /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=WWINMAIN %s
+
+# MAIN:     Subsystem: IMAGE_SUBSYSTEM_WINDOWS_CUI
+# WMAIN:    Subsystem: IMAGE_SUBSYSTEM_WINDOWS_CUI
+# WINMAIN:  Subsystem: IMAGE_SUBSYSTEM_WINDOWS_GUI
+# WWINMAIN: Subsystem: IMAGE_SUBSYSTEM_WINDOWS_GUI
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_I386
+  Characteristics: []
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     B82A000000C3
+symbols:
+  - Name:            .text
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          6
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          0
+  - Name:            ENTRYNAME
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            _mainCRTStartup
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            _wmainCRTStartup
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            _WinMainCRTStartup
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            _wWinMainCRTStartup
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...