Change the implementation of --dynamic-list to use linker script parsing

[• rafael]

Summary

The feature is documented as

---

The format of the dynamic list is the same as the version node
  without scope and node name.  See *note VERSION:: for more
  information.

---

And indeed qt uses a dynamic list with an 'extern "C++"' in it. With this patch we support that

The change to gc-sections-shared makes us match bfd. Just because we kept bar doesn't mean it has to be in the dynamic symbol table.

The changes to invalid-dynamic-list.test and reproduce.s are because of the new parser.

The changes to version-script.s are the only case where we change behavior with regards to bfd, but I would like to see a mix of --version-script and --dynamic-list used in the wild before complicating the code.

[ruiu]

ELF/Driver.cpp

634

Remove {

642–654

This needs a comment.

643–653

What are false and false? Please add a comment.

645–646

Use hasArg.

[• rafael]

Rebaseed and implemented requests.

[ruiu]

LGTM

[pcc]

In gc-sections-shared.s don't we need to keep bar to maintain function pointer identity? In fact, this is what both bfd and gold do on my machine.

[pcc]

In D27578#617334, @pcc wrote:

In gc-sections-shared.s don't we need to keep bar to maintain function pointer identity? In fact, this is what both bfd and gold do on my machine.

Perhaps more significantly, this seems like it would break function preemption. The sanitizers use --dynamic-list, so I think this would break preemption in any program built with a sanitizer.

[• espindola]

289082

ELF/Config.h

struct Configuration {
   llvm::StringRef LTOAAPipeline;
   llvm::StringRef LTONewPmPasses;
   llvm::StringRef OutputFile;
   llvm::StringRef SoName;
   llvm::StringRef Sysroot;
   std::string RPath;
   std::vector<VersionDefinition> VersionDefinitions;
   std::vector<llvm::StringRef> AuxiliaryList;
-  std::vector<llvm::StringRef> DynamicList;
   std::vector<llvm::StringRef> SearchPaths;
   std::vector<llvm::StringRef> Undefined;
   std::vector<SymbolVersion> VersionScriptGlobals;
   std::vector<SymbolVersion> VersionScriptLocals;
   std::vector<uint8_t> BuildIdVector;
   bool AllowMultipleDefinition;
   bool AsNeeded = false;
   bool Bsymbolic;

ELF/Driver.h

enum {
   OPT_INVALID = 0,
 #define OPTION(_1, _2, ID, _4, _5, _6, _7, _8, _9, _10, _11) OPT_##ID,
 #include "Options.inc"
 #undef OPTION
 };
 
 void printHelp(const char *Argv0);
 std::vector<uint8_t> parseHexstring(StringRef S);
-void parseDynamicList(MemoryBufferRef MB);
 
 std::string createResponseFile(const llvm::opt::InputArgList &Args);
 
 llvm::Optional<std::string> findFromSearchPaths(StringRef Path);
 llvm::Optional<std::string> searchLibrary(StringRef Path);
 
 } // namespace elf
 } // namespace lld
 
 #endif

ELF/Driver.cpp

void LinkerDriver::readConfigs(opt::InputArgList &Args) {
   for (auto *Arg : Args.filtered(OPT_auxiliary))
     Config->AuxiliaryList.push_back(Arg->getValue());
   if (!Config->Shared && !Config->AuxiliaryList.empty())
     error("-f may not be used without -shared");
 
   for (auto *Arg : Args.filtered(OPT_undefined))
     Config->Undefined.push_back(Arg->getValue());
 
   if (auto *Arg = Args.getLastArg(OPT_dynamic_list))
     if (Optional<MemoryBufferRef> Buffer = readFile(Arg->getValue()))
-      parseDynamicList(*Buffer);
+      readDynamicList(*Buffer);
 
   if (auto *Arg = Args.getLastArg(OPT_symbol_ordering_file))
     if (Optional<MemoryBufferRef> Buffer = readFile(Arg->getValue()))
       parseSymbolOrderingList(*Buffer);
 
   for (auto *Arg : Args.filtered(OPT_export_dynamic_symbol))
-    Config->DynamicList.push_back(Arg->getValue());
+    Config->VersionScriptGlobals.push_back(
+        {Arg->getValue(), /*IsExternCpp*/ false, /*HasWildcard*/ false});
+
+  // Dynamic lists are a simplified linker script that doesn't need the
+  // "global:" and implicitly ends with a "local:*". Set the variables needed to
+  // simulate that.
+  if (Args.hasArg(OPT_dynamic_list) || Args.hasArg(OPT_export_dynamic_symbol)) {
+    Config->ExportDynamic = true;
+    if (!Config->Shared)
+      Config->DefaultSymbolVersion = VER_NDX_LOCAL;
+  }
 
   if (auto *Arg = Args.getLastArg(OPT_version_script))
     if (Optional<MemoryBufferRef> Buffer = readFile(Arg->getValue()))
       readVersionScript(*Buffer);
 }
 
 // Returns a value of "-format" option.
 static bool getBinaryOption(StringRef S) {
   if (S == "binary")
if (Symtab.find(Config->Entry))
     Symtab.addUndefined(Config->Entry);
 
   // Return if there were name resolution errors.
   if (ErrorCount)
     return;
 
   Symtab.scanUndefinedFlags();
   Symtab.scanShlibUndefined();
-  Symtab.scanDynamicList();
   Symtab.scanVersionScript();
 
   Symtab.addCombinedLTOObject();
   if (ErrorCount)
     return;
 
   for (auto *Arg : Args.filtered(OPT_wrap))
     Symtab.wrap(Arg->getValue());

ELF/DriverUtils.cpp

opt::InputArgList ELFOptTable::parse(ArrayRef<const char *> Argv) {
   if (MissingCount)
     error(Twine(Args.getArgString(MissingIndex)) + ": missing argument");
 
   for (auto *Arg : Args.filtered(OPT_UNKNOWN))
     error("unknown argument: " + Arg->getSpelling());
   return Args;
 }
 
-// Parse the --dynamic-list argument.  A dynamic list is in the form
-//
-//  { symbol1; symbol2; [...]; symbolN };
-//
-// Multiple groups can be defined in the same file, and they are merged
-// into a single group.
-void elf::parseDynamicList(MemoryBufferRef MB) {
-  class Parser : public ScriptParserBase {
-  public:
-    Parser(MemoryBufferRef MB) : ScriptParserBase(MB) {}
-
-    void run() {
-      while (!atEOF()) {
-        expect("{");
-        while (!Error && !consume("}")) {
-          Config->DynamicList.push_back(unquote(next()));
-          expect(";");
-        }
-        expect(";");
-      }
-    }
-  };
-
-  Parser(MB).run();
-}
-
 void elf::printHelp(const char *Argv0) {
   ELFOptTable Table;
   Table.PrintHelp(outs(), Argv0, "lld", false);
 }
 
 // Reconstructs command line arguments so that so that you can re-run
 // the same command with the same inputs. This is for --reproduce.
 std::string elf::createResponseFile(const opt::InputArgList &Args) {

ELF/LinkerScript.h

 
 // Parses a linker script. Calling this function updates
 // Config and ScriptConfig.
 void readLinkerScript(MemoryBufferRef MB);
 
 // Parses a version script.
 void readVersionScript(MemoryBufferRef MB);
 
+void readDynamicList(MemoryBufferRef MB);
+
 // This enum is used to implement linker script SECTIONS command.
 // https://sourceware.org/binutils/docs/ld/SECTIONS.html#SECTIONS
 enum SectionsCommandKind {
   AssignmentKind, // . = expr or <sym> = expr
   OutputSectionKind,
   InputSectionKind,
   AssertKind,   // ASSERT(expr)
   BytesDataKind // BYTE(expr), SHORT(expr), LONG(expr) or QUAD(expr)

ELF/LinkerScript.cpp

 
 public:
   ScriptParser(MemoryBufferRef MB)
       : ScriptParserBase(MB),
         IsUnderSysroot(isUnderSysroot(MB.getBufferIdentifier())) {}
 
   void readLinkerScript();
   void readVersionScript();
+  void readDynamicList();
 
 private:
   void addFile(StringRef Path);
 
   void readAsNeeded();
   void readEntry();
   void readExtern();
   void readGroup();
private:
   void readVersionDeclaration(StringRef VerStr);
   std::vector<SymbolVersion> readSymbols();
 
   ScriptConfiguration &Opt = *ScriptConfig;
   bool IsUnderSysroot;
   std::vector<std::unique_ptr<MemoryBuffer>> OwningMBs;
 };
 
+void ScriptParser::readDynamicList() {
+  expect("{");
+  readAnonymousDeclaration();
+  if (!atEOF())
+    setError("EOF expected, but got " + next());
+}
+
 void ScriptParser::readVersionScript() {
   readVersionScriptCommand();
   if (!atEOF())
     setError("EOF expected, but got " + next());
 }
 
 void ScriptParser::readVersionScriptCommand() {
   if (consume("{")) {

 }
 
 // Reads an "extern C++" directive, e.g.,
 // "extern "C++" { ns::*; "f(int, double)"; };"
 std::vector<SymbolVersion> ScriptParser::readVersionExtern() {
   StringRef Tok = next();
   bool IsCXX = Tok == "\"C++\"";
   if (!IsCXX && Tok != "\"C\"")
-    setError("Unknown Language");
+    setError("Unknown language");
   expect("{");
 
   std::vector<SymbolVersion> Ret;
   while (!Error && peek() != "}") {
     StringRef Tok = next();
     bool HasWildcard = !Tok.startswith("\"") && hasWildcard(Tok);
     Ret.push_back({unquote(Tok), IsCXX, HasWildcard});
     expect(";");
   }
 
   expect("}");
   expect(";");
   return Ret;
 }
 
 void elf::readLinkerScript(MemoryBufferRef MB) {
   ScriptParser(MB).readLinkerScript();
 }
 
 void elf::readVersionScript(MemoryBufferRef MB) {
   ScriptParser(MB).readVersionScript();
 }
 
+void elf::readDynamicList(MemoryBufferRef MB) {
+  ScriptParser(MB).readDynamicList();
+}
+
 template class elf::LinkerScript<ELF32LE>;
 template class elf::LinkerScript<ELF32BE>;
 template class elf::LinkerScript<ELF64LE>;
 template class elf::LinkerScript<ELF64BE>;

ELF/SymbolTable.h

Symbol *addBitcode(StringRef Name, uint8_t Binding, uint8_t StOther,
                      uint8_t Type, bool CanOmitFromDynSym, BitcodeFile *File);
 
   Symbol *addCommon(StringRef N, uint64_t Size, uint64_t Alignment,
                     uint8_t Binding, uint8_t StOther, uint8_t Type,
                     InputFile *File);
 
   void scanUndefinedFlags();
   void scanShlibUndefined();
-  void scanDynamicList();
   void scanVersionScript();
 
   SymbolBody *find(StringRef Name);
 
   void trace(StringRef Name);
   void wrap(StringRef Name);
 
   std::vector<InputSectionBase<ELFT> *> Sections;

ELF/SymbolTable.cpp

 template <class ELFT> void SymbolTable<ELFT>::scanShlibUndefined() {
   for (SharedFile<ELFT> *File : SharedFiles)
     for (StringRef U : File->getUndefinedSymbols())
       if (SymbolBody *Sym = find(U))
         if (Sym->isDefined())
           Sym->symbol()->ExportDynamic = true;
 }
 
-// This function processes --export-dynamic-symbol and --dynamic-list.
-template <class ELFT> void SymbolTable<ELFT>::scanDynamicList() {
-  for (StringRef S : Config->DynamicList)
-    if (SymbolBody *B = find(S))
-      B->symbol()->ExportDynamic = true;
-}
-
 // Initialize DemangledSyms with a map from demangled symbols to symbol
 // objects. Used to handle "extern C++" directive in version scripts.
 //
 // The map will contain all demangled symbols. That can be very large,
 // and in LLD we generally want to avoid do anything for each symbol.
 // Then, why are we doing this? Here's why.
 //
 // Users can use "extern C++ {}" directive to match against demangled

test/ELF/gc-sections-shared.s

 # CHECK-NEXT:     Value:
 # CHECK-NEXT:     Size:
 # CHECK-NEXT:     Binding: Local
 # CHECK-NEXT:     Type:
 # CHECK-NEXT:     Other:
 # CHECK-NEXT:     Section: Undefined (0x0)
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
-# CHECK-NEXT:     Name: bar
-# CHECK-NEXT:     Value:
-# CHECK-NEXT:     Size:
-# CHECK-NEXT:     Binding: Global
-# CHECK-NEXT:     Type:
-# CHECK-NEXT:     Other:
-# CHECK-NEXT:     Section: .text
-# CHECK-NEXT:   }
-# CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name: bar2
 # CHECK-NEXT:     Value:
 # CHECK-NEXT:     Size:
 # CHECK-NEXT:     Binding: Global
 # CHECK-NEXT:     Type:
 # CHECK-NEXT:     Other:
 # CHECK-NEXT:     Section: Undefined
 # CHECK-NEXT:   }

test/ELF/invalid-dynamic-list.test

 
 ## Missing \" in foobar definition
 # RUN echo "{ \"foobar; };" > %t1
 # RUN: not ld.lld --dynamic-list %t1 2>&1 | FileCheck -check-prefix=ERR5 %s
 # ERR5: {{.*}}:1: unexpected EOF
 
 # RUN: echo "{ extern \"BOGUS\" { test }; };" > %t1
 # RUN: not ld.lld --dynamic-list %t1 2>&1 | FileCheck -check-prefix=ERR6 %s
-# ERR6: {{.*}}:1: ; expected, but got "BOGUS"
+# ERR6: {{.*}}:1: Unknown language

test/ELF/reproduce.s

 # RUN: mkdir -p %t.dir/build2/a/b/c
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build2/foo.o
 # RUN: cd %t.dir/build2/a/b/c
 # RUN: env LLD_REPRODUCE=repro ld.lld ./../../../foo.o -o bar -shared --as-needed
 # RUN: cpio -id < repro.cpio
 # RUN: diff %t.dir/build2/foo.o repro/%:t.dir/build2/foo.o
 
 # RUN: echo "{ local: *; };" >  ver
-# RUN: echo > dyn
+# RUN: echo "{};" > dyn
 # RUN: echo > file
 # RUN: echo > file2
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o 'foo bar'
 # RUN: ld.lld --reproduce repro2 'foo bar' -L"foo bar" -Lfile -Tfile2 \
 # RUN:   --dynamic-list dyn -rpath file --script=file --version-script ver \
 # RUN:   --dynamic-linker "some unusual/path" -soname 'foo bar' -soname='foo bar'
 # RUN: cpio -id < repro2.cpio
 # RUN: FileCheck %s --check-prefix=RSP2 < repro2/response.txt

test/ELF/version-script.s

 # REQUIRES: x86
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/shared.s -o %t2.o
 # RUN: ld.lld -shared %t2.o -soname shared -o %t2.so
 
 # RUN: echo "{ global: foo1; foo3; local: *; };" > %t.script
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
 # RUN: ld.lld --version-script %t.script -shared %t.o %t2.so -o %t.so
 # RUN: llvm-readobj -dyn-symbols %t.so | FileCheck --check-prefix=DSO %s
 
 # RUN: echo "# comment" > %t3.script
 # RUN: echo "{ local: *; # comment" >> %t3.script
 # RUN: echo -n "}; # comment" >> %t3.script
 # RUN: ld.lld --version-script %t3.script -shared %t.o %t2.so -o %t3.so
 # RUN: llvm-readobj -dyn-symbols %t3.so | FileCheck --check-prefix=DSO2 %s
 
-# --version-script filters --dynamic-list.
-# RUN: echo "{ foo1; foo2; };" > %t.list
-# RUN: ld.lld --version-script %t.script --dynamic-list %t.list %t.o %t2.so -o %t
-# RUN: llvm-readobj -dyn-symbols %t | FileCheck --check-prefix=EXE %s
-
 # RUN: echo "VERSION_1.0 { global: foo1; local: *; };" > %t4.script
 # RUN: echo "VERSION_2.0 { global: foo3; local: *; };" >> %t4.script
 # RUN: ld.lld --version-script %t4.script -shared %t.o %t2.so -o %t4.so
 # RUN: llvm-readobj -dyn-symbols %t4.so | FileCheck --check-prefix=VERDSO %s
 
 # RUN: echo "VERSION_1.0 { global: foo1; local: *; };" > %t5.script
 # RUN: echo "{ global: foo3; local: *; };" >> %t5.script
 # RUN: not ld.lld --version-script %t5.script -shared %t.o %t2.so -o %t5.so 2>&1 | \
 # RUN:   FileCheck -check-prefix=ERR1 %s
 # ERR1: anonymous version definition is used in combination with other version definitions
 
 # RUN: echo "{ global: foo1; local: *; };" > %t5.script
 # RUN: echo "VERSION_2.0 { global: foo3; local: *; };" >> %t5.script
 # RUN: not ld.lld --version-script %t5.script -shared %t.o %t2.so -o %t5.so 2>&1 | \
 # RUN:   FileCheck -check-prefix=ERR2 %s
 # ERR2: EOF expected, but got VERSION_2.0
 
 # RUN: echo "VERSION_1.0 { global: foo1; local: *; };" > %t6.script
 # RUN: echo "VERSION_2.0 { global: foo1; local: *; };" >> %t6.script
 # RUN: ld.lld --version-script %t6.script -shared %t.o %t2.so -o %t6.so 2>&1 | \
 # RUN:   FileCheck -check-prefix=WARN2 %s
 # WARN2: duplicate symbol 'foo1' in version script
 
+# RUN: echo "{ foo1; foo2; };" > %t.list
 # RUN: ld.lld --version-script %t.script --dynamic-list %t.list %t.o %t2.so -o %t2
 # RUN: llvm-readobj %t2 > /dev/null
 
 # DSO:      DynamicSymbols [
 # DSO-NEXT:   Symbol {
 # DSO-NEXT:     Name: @
 # DSO-NEXT:     Value: 0x0
 # DSO-NEXT:     Size: 0

 # DSO2-NEXT:     Size: 0
 # DSO2-NEXT:     Binding: Global (0x1)
 # DSO2-NEXT:     Type: Function (0x2)
 # DSO2-NEXT:     Other: 0
 # DSO2-NEXT:     Section: Undefined (0x0)
 # DSO2-NEXT:   }
 # DSO2-NEXT: ]
 
-# EXE:      DynamicSymbols [
-# EXE-NEXT:   Symbol {
-# EXE-NEXT:     Name: @
-# EXE-NEXT:     Value: 0x0
-# EXE-NEXT:     Size: 0
-# EXE-NEXT:     Binding: Local (0x0)
-# EXE-NEXT:     Type: None (0x0)
-# EXE-NEXT:     Other: 0
-# EXE-NEXT:     Section: Undefined (0x0)
-# EXE-NEXT:   }
-# EXE-NEXT:   Symbol {
-# EXE-NEXT:     Name: bar@
-# EXE-NEXT:     Value: 0x0
-# EXE-NEXT:     Size: 0
-# EXE-NEXT:     Binding: Global (0x1)
-# EXE-NEXT:     Type: Function (0x2)
-# EXE-NEXT:     Other: 0
-# EXE-NEXT:     Section: Undefined (0x0)
-# EXE-NEXT:   }
-# EXE-NEXT:   Symbol {
-# EXE-NEXT:     Name: foo1@
-# EXE-NEXT:     Value: 0x201000
-# EXE-NEXT:     Size: 0
-# EXE-NEXT:     Binding: Global (0x1)
-# EXE-NEXT:     Type: None (0x0)
-# EXE-NEXT:     Other: 0
-# EXE-NEXT:     Section: .text
-# EXE-NEXT:   }
-# EXE-NEXT: ]
-
 # VERDSO:      DynamicSymbols [
 # VERDSO-NEXT:   Symbol {
 # VERDSO-NEXT:     Name: @
 # VERDSO-NEXT:     Value: 0x0
 # VERDSO-NEXT:     Size: 0
 # VERDSO-NEXT:     Binding: Local
 # VERDSO-NEXT:     Type: None
 # VERDSO-NEXT:     Other: 0