Give microsoftDemangle() an outparam for how many input bytes were consumed.

[thakis]

Summary

Demangling Itanium symbols either consumes the whole input or fails,
but Microsoft symbols can be successfully demangled with just some
of the input.

Add an outparam that enables clients to know how much of the input was
consumed, andn use this flag to give llvm-undname an opt-in warning
on partially consumed symbols.

[hans]

Seems reasonable, but can you expand on when this comes up in practice?

llvm/include/llvm/Demangle/Demangle.h

50

s/to/too/

[thakis]

Sure! The motivation is that if you want to write something like c++filt that finds all symbols in text and replaces them with the demangled form, you'll have a loop like:

for each word w
  try to demangle w to demangled_w
  if that worked,
    print demangled_w
  else
    print w

For Itanium this works great since as far as I know if w is demangled successfully, it always consumes all of w. But with the Microsoft scheme, it's possible that we get a demangled_w successfully but only consume some prefix of w, and it'd be good to print the unconsumed suffix too. With the current API that's not (*) possible.

*: Well I suppose for each w that successfully demangles, you could hand in only the first half of w and see if that demangles too and binary search over prefixes to find the suffix. But since the data is right there in the demangler and we only have to get it out, adding a way to get it out seems better to me.

[hans]

Thanks! lgtm

nit in the commit message: "consumed, *andn* use this"

lldb/source/Core/Mangled.cpp

if (name) {
     m_demangled.Clear();
     m_mangled.Clear();
   }
 }
 
 // Local helpers for different demangling implementations.
 static char *GetMSVCDemangledStr(const char *M) {
   char *demangled_cstr = llvm::microsoftDemangle(
-      M, nullptr, nullptr, nullptr,
+      M, nullptr, nullptr, nullptr, nullptr,
       llvm::MSDemangleFlags(llvm::MSDF_NoAccessSpecifier |
                             llvm::MSDF_NoCallingConvention |
                             llvm::MSDF_NoMemberType));
 
   if (Log *log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_DEMANGLE)) {
     if (demangled_cstr && demangled_cstr[0])
       LLDB_LOGF(log, "demangled msvc: %s -> \"%s\"", M, demangled_cstr);
     else

llvm/include/llvm/Demangle/Demangle.h

 enum MSDemangleFlags {
   MSDF_None = 0,
   MSDF_DumpBackrefs = 1 << 0,
   MSDF_NoAccessSpecifier = 1 << 1,
   MSDF_NoCallingConvention = 1 << 2,
   MSDF_NoReturnType = 1 << 3,
   MSDF_NoMemberType = 1 << 4,
 };
-char *microsoftDemangle(const char *mangled_name, char *buf, size_t *n,
+
+/// Demangles the Microsoft symbol pointed at by mangled_name and returns it.
+/// Returns a pointer to the start of a null-terminated demangled string on
+/// success, or nullptr on error.
+/// If n_read is non-null and demangling was successful, it receives how many
+/// bytes of the input string were consumed.
+/// buf can point to a *n_buf bytes large buffer where the demangled name is
+/// stored. If the buffer is too small, it is grown with realloc(). If buf is
+/// nullptr, then this malloc()s memory for the result.
+/// *n_buf stores the size of buf on input if buf is non-nullptr, and it
+/// receives the size of the demangled string on output if n_buf is not nullptr.
+/// status receives one of the demangle_ enum entries above if it's not nullptr.
+/// Flags controls various details of the demangled representation.
+char *microsoftDemangle(const char *mangled_name, size_t *n_read,
+                        char *buf, size_t *n_buf,
                         int *status, MSDemangleFlags Flags = MSDF_None);
 
 /// Attempt to demangle a string using different demangling schemes.
 /// The function uses heuristics to determine which demangling scheme to use.
 /// \param MangledName - reference to string to demangle.
 /// \returns - the demangled string, or a copy of the input string if no
 /// demangling occurred.
 std::string demangle(const std::string &MangledName);

llvm/lib/DebugInfo/Symbolize/Symbolize.cpp

if (Name.substr(0, 2) == "_Z") {
     free(DemangledName);
     return Result;
   }
 
   if (!Name.empty() && Name.front() == '?') {
     // Only do MSVC C++ demangling on symbols starting with '?'.
     int status = 0;
     char *DemangledName = microsoftDemangle(
-        Name.c_str(), nullptr, nullptr, &status,
+        Name.c_str(), nullptr, nullptr, nullptr, &status,
         MSDemangleFlags(MSDF_NoAccessSpecifier | MSDF_NoCallingConvention |
                         MSDF_NoMemberType | MSDF_NoReturnType));
     if (status != 0)
       return Name;
     std::string Result = DemangledName;
     free(DemangledName);
     return Result;
   }
 
   if (DbiModuleDescriptor && DbiModuleDescriptor->isWin32Module())
     return std::string(demanglePE32ExternCFunc(Name));
   return Name;
 }
 
 } // namespace symbolize
 } // namespace llvm

llvm/lib/Demangle/Demangle.cpp

static bool isItaniumEncoding(const std::string &MangledName) {
   return Pos > 0 && Pos <= 4 && MangledName[Pos] == 'Z';
 }
 
 std::string llvm::demangle(const std::string &MangledName) {
   char *Demangled;
   if (isItaniumEncoding(MangledName))
     Demangled = itaniumDemangle(MangledName.c_str(), nullptr, nullptr, nullptr);
   else
-    Demangled =
-        microsoftDemangle(MangledName.c_str(), nullptr, nullptr, nullptr);
+    Demangled = microsoftDemangle(MangledName.c_str(), nullptr, nullptr,
+                                  nullptr, nullptr);
 
   if (!Demangled)
     return MangledName;
 
   std::string Ret = Demangled;
   free(Demangled);
   return Ret;
 }

llvm/lib/Demangle/MicrosoftDemangle.cpp

void Demangler::dumpBackReferences() {
   for (size_t I = 0; I < Backrefs.NamesCount; ++I) {
     std::printf("  [%d] - %.*s\n", (int)I, (int)Backrefs.Names[I]->Name.size(),
                 Backrefs.Names[I]->Name.begin());
   }
   if (Backrefs.NamesCount > 0)
     std::printf("\n");
 }
 
-char *llvm::microsoftDemangle(const char *MangledName, char *Buf, size_t *N,
+char *llvm::microsoftDemangle(const char *MangledName, size_t *NMangled,
+                              char *Buf, size_t *N,
                               int *Status, MSDemangleFlags Flags) {
-  int InternalStatus = demangle_success;
   Demangler D;
   OutputStream S;
 
   StringView Name{MangledName};
   SymbolNode *AST = D.parse(Name);
+  if (!D.Error && NMangled)
+    *NMangled = Name.begin() - MangledName;
 
   if (Flags & MSDF_DumpBackrefs)
     D.dumpBackReferences();
 
   OutputFlags OF = OF_Default;
   if (Flags & MSDF_NoCallingConvention)
     OF = OutputFlags(OF | OF_NoCallingConvention);
   if (Flags & MSDF_NoAccessSpecifier)
     OF = OutputFlags(OF | OF_NoAccessSpecifier);
   if (Flags & MSDF_NoReturnType)
     OF = OutputFlags(OF | OF_NoReturnType);
   if (Flags & MSDF_NoMemberType)
     OF = OutputFlags(OF | OF_NoMemberType);
 
+  int InternalStatus = demangle_success;
   if (D.Error)
     InternalStatus = demangle_invalid_mangled_name;
   else if (!initializeOutputStream(Buf, N, S, 1024))
     InternalStatus = demangle_memory_alloc_failure;
   else {
     AST->output(S, OF);
     S += '\0';
     if (N != nullptr)
       *N = S.getCurrentPosition();
     Buf = S.getBuffer();
   }
 
   if (Status)
     *Status = InternalStatus;
   return InternalStatus == demangle_success ? Buf : nullptr;
 }

llvm/test/Demangle/warn-trailing.test

+; RUN: llvm-undname -warn-trailing 2>&1 < %s | FileCheck %s
+
+?x@@3HAasdf
+; CHECK: ?x@@3HAasdf
+; CHECK-NEXT: int x
+; CHECK-NEXT: warning: trailing characters: asdf

llvm/tools/llvm-microsoft-demangle-fuzzer/llvm-microsoft-demangle-fuzzer.cpp

 
 #include <cstdint>
 #include <cstdlib>
 #include <string>
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   std::string NullTerminatedString((const char *)Data, Size);
   free(llvm::microsoftDemangle(NullTerminatedString.c_str(), nullptr, nullptr,
-                               nullptr));
+                               nullptr, nullptr));
   return 0;
 }

llvm/tools/llvm-objdump/COFFDump.cpp

outs() << "[" << format("%2d", SI) << "]"
            << "(fl 0x00)" // Flag bits, which COFF doesn't have.
            << "(ty " << format("%3x", unsigned(Symbol->getType())) << ")"
            << "(scl " << format("%3x", unsigned(Symbol->getStorageClass()))
            << ") "
            << "(nx " << unsigned(Symbol->getNumberOfAuxSymbols()) << ") "
            << "0x" << format("%08x", unsigned(Symbol->getValue())) << " "
            << Name;
     if (Demangle && Name.startswith("?")) {
-      char *DemangledSymbol = nullptr;
-      size_t Size = 0;
       int Status = -1;
-      DemangledSymbol =
-          microsoftDemangle(Name.data(), DemangledSymbol, &Size, &Status);
+      char *DemangledSymbol =
+          microsoftDemangle(Name.data(), nullptr, nullptr, nullptr, &Status);
 
       if (Status == 0 && DemangledSymbol) {
         outs() << " (" << StringRef(DemangledSymbol) << ")";
         std::free(DemangledSymbol);
       } else {
         outs() << " (invalid mangled name)";
       }
     }

llvm/tools/llvm-undname/llvm-undname.cpp

 cl::opt<bool> NoReturnType("no-return-type", cl::Optional,
                            cl::desc("skip return types"), cl::Hidden,
                            cl::init(false));
 cl::opt<bool> NoMemberType("no-member-type", cl::Optional,
                            cl::desc("skip member types"), cl::Hidden,
                            cl::init(false));
 cl::opt<std::string> RawFile("raw-file", cl::Optional,
                              cl::desc("for fuzzer data"), cl::Hidden);
+cl::opt<bool> WarnTrailing("warn-trailing", cl::Optional,
+                           cl::desc("warn on trailing characters"), cl::Hidden,
+                           cl::init(false));
 cl::list<std::string> Symbols(cl::Positional, cl::desc("<input symbols>"),
                               cl::ZeroOrMore);
 
 static bool msDemangle(const std::string &S) {
   int Status;
   MSDemangleFlags Flags = MSDF_None;
   if (DumpBackReferences)
     Flags = MSDemangleFlags(Flags | MSDF_DumpBackrefs);
   if (NoAccessSpecifier)
     Flags = MSDemangleFlags(Flags | MSDF_NoAccessSpecifier);
   if (NoCallingConvention)
     Flags = MSDemangleFlags(Flags | MSDF_NoCallingConvention);
   if (NoReturnType)
     Flags = MSDemangleFlags(Flags | MSDF_NoReturnType);
   if (NoMemberType)
     Flags = MSDemangleFlags(Flags | MSDF_NoMemberType);
 
+  size_t NRead;
   char *ResultBuf =
-      microsoftDemangle(S.c_str(), nullptr, nullptr, &Status, Flags);
+      microsoftDemangle(S.c_str(), &NRead, nullptr, nullptr, &Status, Flags);
   if (Status == llvm::demangle_success) {
     outs() << ResultBuf << "\n";
     outs().flush();
+    if (WarnTrailing && NRead < S.size())
+      WithColor::warning() << "trailing characters: " << S.c_str() + NRead
+                           << "\n";
   } else {
     WithColor::error() << "Invalid mangled name\n";
   }
   std::free(ResultBuf);
   return Status == llvm::demangle_success;
 }
 
 int main(int argc, char **argv) {