MS ABI: Mangle null pointer-to-member-functions compatibly

[majnemer]

Summary

Previously, we would mangle nullptr pointer-to-member-functions in class
templates with a mangling we invented because contemporary versions of
MSVC would crash when trying to compile such code.

However, VS "14" can successfully compile these sorts of template
instantiations. This commit updates our mangling to be compatible with
theirs.

[rnk]

lgtm

Feel free to commit with a test for the null data memptr of a non-dynamic class.

lib/AST/MicrosoftMangle.cpp

539

What happens if -1 is the null representation? This happens for data memptrs of non-dynamic classes.

[majnemer]

Closed by commit rL210637 (authored by @majnemer).

lib/AST/MicrosoftMangle.cpp

 
 void
 MicrosoftCXXNameMangler::mangleMemberFunctionPointer(const CXXRecordDecl *RD,
                                                      const CXXMethodDecl *MD) {
   // <member-function-pointer> ::= $1? <name>
   //                           ::= $H? <name> <number>
   //                           ::= $I? <name> <number> <number>
   //                           ::= $J? <name> <number> <number> <number>
-  //                           ::= $0A@
 
   MSInheritanceAttr::Spelling IM = RD->getMSInheritanceModel();
 
-  // The null member function pointer is $0A@ in function templates and crashes
-  // MSVC when used in class templates, so we don't know what they really look
-  // like.
-  if (!MD) {
-    Out << "$0A@";
-    return;
-  }
-
   char Code = '\0';
   switch (IM) {
   case MSInheritanceAttr::Keyword_single_inheritance:      Code = '1'; break;
   case MSInheritanceAttr::Keyword_multiple_inheritance:    Code = 'H'; break;
   case MSInheritanceAttr::Keyword_virtual_inheritance:     Code = 'I'; break;
   case MSInheritanceAttr::Keyword_unspecified_inheritance: Code = 'J'; break;
   }
 
-  Out << '$' << Code << '?';
-
   // If non-virtual, mangle the name.  If virtual, mangle as a virtual memptr
   // thunk.
   uint64_t NVOffset = 0;
   uint64_t VBTableOffset = 0;
   uint64_t VBPtrOffset = 0;
+  if (MD) {
+    Out << '$' << Code << '?';
     if (MD->isVirtual()) {
       MicrosoftVTableContext *VTContext =
           cast<MicrosoftVTableContext>(getASTContext().getVTableContext());
       const MicrosoftVTableContext::MethodVFTableLocation &ML =
           VTContext->getMethodVFTableLocation(GlobalDecl(MD));
       mangleVirtualMemPtrThunk(MD, ML);
       NVOffset = ML.VFPtrOffset.getQuantity();
       VBTableOffset = ML.VBTableIndex * 4;
       if (ML.VBase) {
         const ASTRecordLayout &Layout = getASTContext().getASTRecordLayout(RD);
         VBPtrOffset = Layout.getVBPtrOffset().getQuantity();
       }
     } else {
       mangleName(MD);
       mangleFunctionEncoding(MD);
     }
+  } else {
+    // Null single inheritance member functions are encoded as a simple nullptr.
+    if (IM == MSInheritanceAttr::Keyword_single_inheritance) {
+      Out << "$0A@";
+      return;
+    }
+    if (IM == MSInheritanceAttr::Keyword_unspecified_inheritance)
+      VBTableOffset = -1;
+    Out << '$' << Code;
+  }
 
   if (MSInheritanceAttr::hasNVOffsetField(/*IsMemberFunction=*/true, IM))
     mangleNumber(NVOffset);
   if (MSInheritanceAttr::hasVBPtrOffsetField(IM))
     mangleNumber(VBPtrOffset);
   if (MSInheritanceAttr::hasVBTableOffsetField(IM))
     mangleNumber(VBTableOffset);
 }

test/CodeGenCXX/mangle-ms-templates-memptrs-2.cpp

 // RUN: %clang_cc1 -Wno-microsoft -fms-extensions -fno-rtti -std=c++11 -emit-llvm %s -o - -triple=i386-pc-win32 | FileCheck %s
 
+template <typename T, int (T::*)() = nullptr>
+struct J {};
+
+struct __single_inheritance M;
+J<M> m;
+// CHECK-DAG: @"\01?m@@3U?$J@UM@@$0A@@@A"
+
+struct __multiple_inheritance N;
+J<N> n;
+// CHECK-DAG: @"\01?n@@3U?$J@UN@@$HA@@@A"
+
+struct __virtual_inheritance O;
+J<O> o;
+// CHECK-DAG: @"\01?o@@3U?$J@UO@@$IA@A@@@A"
+
+struct P;
+J<P> p;
+// CHECK-DAG: @"\01?p@@3U?$J@UP@@$JA@A@?0@@A"
+
 #pragma pointers_to_members(full_generality, virtual_inheritance)
 
 struct S {
   int a, b;
   void f();
   virtual void g();
 };