LLD: Remove YAML/Native round-trip passes

[ruiu]

Summary

The round-trip passes were introduced in r193300. The intention of
the change was to make sure that LLD is capable of reading end
writing such file formats.

But that turned out to be yet another over-designed stuff that had
been slowing down everyday development.

The passes ran after the core linker and before the writer. If you
had an additional piece of information that needs to be passed from
front-end to the writer, you had to invent a way to save the data to
YAML/Native. These passes forced us to do that even if that data
was not needed to be represented neither in an object file nor in
an executable/DSO. It doesn't make sense. We don't need these passes.

[Bigcheese]

I support this, but I'd like to see the input roundtrip tests be added before we remove this.

[• rafael]

I really like this.

Every object file has peculiarities. MachO has scattered relocations. Mips
elf has many in one relocations.

If we, with the handsight of having built a linker, create a rounder wheel,
it will be just another file format. We will still have to parse it and
report broken files. It will have to be able to represent the constructs
that currently use the various features of existing formats, but calling it
"native" and requiring direct mapping to it seems counter productive.

include/lld/Passes/RoundTripNativePass.h

-//===--Passes/RoundTripNativePass.h - Write Native file/Read it back------===//
-//
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_PASSES_ROUND_TRIP_NATIVE_PASS_H
-#define LLD_PASSES_ROUND_TRIP_NATIVE_PASS_H
-
-#include "lld/Core/File.h"
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/Pass.h"
-#include <vector>
-
-namespace lld {
-class RoundTripNativePass : public Pass {
-public:
-  RoundTripNativePass(LinkingContext &context) : Pass(), _context(context) {}
-
-  /// Writes to a native file and reads the atoms from the native file back.
-  /// Replaces mergedFile with the contents of the native File.
-  void perform(std::unique_ptr<MutableFile> &mergedFile) override;
-
-  virtual ~RoundTripNativePass() {}
-
-private:
-  LinkingContext &_context;
-  // Keep the parsed file alive for the rest of the link. All atoms
-  // that are created by the RoundTripNativePass are owned by the
-  // nativeFile.
-  std::vector<std::unique_ptr<File> > _nativeFile;
-};
-
-} // namespace lld
-
-#endif // LLD_PASSES_ROUND_TRIP_NATIVE_PASS_H

include/lld/Passes/RoundTripYAMLPass.h

-//===--Passes/RoundTripYAMLPass.h- Write YAML file/Read it back-----------===//
-//
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_PASSES_ROUND_TRIP_YAML_PASS_H
-#define LLD_PASSES_ROUND_TRIP_YAML_PASS_H
-
-#include "lld/Core/File.h"
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/Pass.h"
-#include <vector>
-
-namespace lld {
-class RoundTripYAMLPass : public Pass {
-public:
-  RoundTripYAMLPass(LinkingContext &context) : Pass(), _context(context) {}
-
-  /// Writes to a YAML file and reads the atoms from the YAML file back.
-  /// Replaces the mergedFile with new contents.
-  void perform(std::unique_ptr<MutableFile> &mergedFile) override;
-
-  virtual ~RoundTripYAMLPass() {}
-
-private:
-  LinkingContext &_context;
-  // Keep the parsed file alive for the rest of the link. All atoms
-  // that are created by the RoundTripYAMLPass are owned by the
-  // yamlFile.
-  std::vector<std::unique_ptr<File> > _yamlFile;
-};
-
-} // namespace lld
-
-#endif // LLD_PASSES_ROUND_TRIP_YAML_PASS_H

lib/CMakeLists.txt

 add_subdirectory(Config)
 add_subdirectory(Core)
 add_subdirectory(Driver)
-add_subdirectory(Passes)
 add_subdirectory(ReaderWriter)

lib/Driver/CMakeLists.txt

add_llvm_library(lldDriver
   DarwinLdDriver.cpp
   Driver.cpp
   GnuLdDriver.cpp
   UniversalDriver.cpp
   WinLinkDriver.cpp
   WinLinkModuleDef.cpp
   LINK_LIBS
     lldConfig
-    lldPasses
     lldMachO
     lldPECOFF
     lldELF
     lldAArch64ELFTarget
     lldARMELFTarget
     lldHexagonELFTarget
     lldMipsELFTarget
     lldX86ELFTarget

lib/Driver/Driver.cpp

 #include "lld/Core/Instrumentation.h"
 #include "lld/Core/LLVM.h"
 #include "lld/Core/Parallel.h"
 #include "lld/Core/PassManager.h"
 #include "lld/Core/Reader.h"
 #include "lld/Core/Resolver.h"
 #include "lld/Core/Writer.h"
 #include "lld/Driver/Driver.h"
-#include "lld/Passes/RoundTripNativePass.h"
-#include "lld/Passes/RoundTripYAMLPass.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/raw_ostream.h"
if (!resolver.resolve())
     return false;
   std::unique_ptr<MutableFile> merged = resolver.resultFile();
   resolveTask.end();
 
   // Run passes on linked atoms.
   ScopedTask passTask(getDefaultDomain(), "Passes");
   PassManager pm;
   context.addPasses(pm);
-
-#ifndef NDEBUG
-  llvm::Optional<std::string> env =
-      llvm::sys::Process::GetEnv("LLD_RUN_ROUNDTRIP_TEST");
-
-  if (env.hasValue() && !env.getValue().empty()) {
-    pm.add(llvm::make_unique<RoundTripYAMLPass>(context));
-    pm.add(llvm::make_unique<RoundTripNativePass>(context));
-  }
-#endif
-
   pm.runOnFile(merged);
   passTask.end();
 
   // Give linked atoms to Writer to generate output file.
   ScopedTask writeTask(getDefaultDomain(), "Write");
   if (std::error_code ec = context.writeFile(*merged)) {
     diagnostics << "Failed to write file '" << context.outputPath()
                 << "': " << ec.message() << "\n";
     return false;
   }
 
   return true;
 }
 
 } // namespace

lib/Passes/CMakeLists.txt

-add_llvm_library(lldPasses
-  RoundTripNativePass.cpp
-  RoundTripYAMLPass.cpp
-  LINK_LIBS
-    lldCore
-    lldNative
-    lldYAML
-    LLVMSupport
-  )

lib/Passes/Makefile

-##===- lld/lib/Passes/Makefile ---------------------------*- Makefile -*-===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LLD_LEVEL := ../..
-LIBRARYNAME := lldPasses
-
-include $(LLD_LEVEL)/Makefile

lib/Passes/RoundTripNativePass.cpp

-//===--Passes/RoundTripNativePass.cpp - Write Native file/Read it back-----===//
-//
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/Instrumentation.h"
-#include "lld/Core/Simple.h"
-#include "lld/Core/Writer.h"
-#include "lld/Passes/RoundTripNativePass.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Path.h"
-#include <memory>
-
-using namespace lld;
-
-#define DEBUG_TYPE "RoundTripNativePass"
-
-/// Perform the actual pass
-void RoundTripNativePass::perform(std::unique_ptr<MutableFile> &mergedFile) {
-  ScopedTask task(getDefaultDomain(), "RoundTripNativePass");
-  std::unique_ptr<Writer> nativeWriter = createWriterNative(_context);
-  SmallString<128> tmpNativeFile;
-  // Separate the directory from the filename
-  StringRef outFile = llvm::sys::path::filename(_context.outputPath());
-  if (llvm::sys::fs::createTemporaryFile(outFile, "native", tmpNativeFile))
-    return;
-  DEBUG(llvm::dbgs() << "RoundTripNativePass: " << tmpNativeFile << "\n");
-
-  // The file that is written would be kept around if there is a problem
-  // writing to the file or when reading atoms back from the file.
-  nativeWriter->writeFile(*mergedFile, tmpNativeFile.str());
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mb =
-      MemoryBuffer::getFile(tmpNativeFile.str());
-  if (!mb)
-    return;
-
-  std::error_code ec = _context.registry().loadFile(
-      std::move(mb.get()), _nativeFile);
-  if (ec) {
-    // Note: we need a way for Passes to report errors.
-    llvm_unreachable("native reader not registered or read error");
-  }
-  File *objFile = _nativeFile[0].get();
-  if (objFile->parse())
-    llvm_unreachable("native reader parse error");
-  mergedFile.reset(new SimpleFile(objFile->path()));
-  copyAtoms(mergedFile.get(), objFile);
-  llvm::sys::fs::remove(tmpNativeFile.str());
-}

lib/Passes/RoundTripYAMLPass.cpp

-//===--Passes/RoundTripYAMLPass.cpp - Write YAML file/Read it back---------===//
-//
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/Instrumentation.h"
-#include "lld/Core/Simple.h"
-#include "lld/Core/Writer.h"
-#include "lld/Passes/RoundTripYAMLPass.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Path.h"
-#include <memory>
-
-using namespace lld;
-
-#define DEBUG_TYPE "RoundTripYAMLPass"
-
-/// Perform the actual pass
-void RoundTripYAMLPass::perform(std::unique_ptr<MutableFile> &mergedFile) {
-  ScopedTask task(getDefaultDomain(), "RoundTripYAMLPass");
-  std::unique_ptr<Writer> yamlWriter = createWriterYAML(_context);
-  SmallString<128> tmpYAMLFile;
-  // Separate the directory from the filename
-  StringRef outFile = llvm::sys::path::filename(_context.outputPath());
-  if (llvm::sys::fs::createTemporaryFile(outFile, "yaml", tmpYAMLFile))
-    return;
-  DEBUG(llvm::dbgs() << "RoundTripYAMLPass: " << tmpYAMLFile << "\n");
-
-  // The file that is written would be kept around if there is a problem
-  // writing to the file or when reading atoms back from the file.
-  yamlWriter->writeFile(*mergedFile, tmpYAMLFile.str());
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mb =
-      MemoryBuffer::getFile(tmpYAMLFile.str());
-  if (!mb)
-    return;
-
-  std::error_code ec = _context.registry().loadFile(
-      std::move(mb.get()), _yamlFile);
-  if (ec) {
-    // Note: we need a way for Passes to report errors.
-    llvm_unreachable("yaml reader not registered or read error");
-  }
-  File *objFile = _yamlFile[0].get();
-  if (objFile->parse())
-    llvm_unreachable("native reader parse error");
-  mergedFile.reset(new SimpleFile(objFile->path()));
-  copyAtoms(mergedFile.get(), objFile);
-  llvm::sys::fs::remove(tmpYAMLFile.str());
-}

lib/ReaderWriter/CMakeLists.txt

 add_subdirectory(ELF)
 add_subdirectory(MachO)
 add_subdirectory(Native)
 add_subdirectory(PECOFF)
 add_subdirectory(YAML)
 
 if (MSVC)
   add_definitions(-wd4062) # Suppress 'warning C4062: Enumerator has no associated handler in a switch statement.'
 endif()
 
 add_llvm_library(lldReaderWriter
   CoreLinkingContext.cpp
   FileArchive.cpp
   LinkerScript.cpp
   LINK_LIBS
     lldCore
-    lldPasses
     lldYAML
     LLVMObject
     LLVMSupport
   )

lib/ReaderWriter/CoreLinkingContext.cpp

 //===- lib/ReaderWriter/CoreLinkingContext.cpp ----------------------------===//
 //
 //                             The LLVM Linker
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "lld/Core/DefinedAtom.h"
 #include "lld/Core/File.h"
 #include "lld/Core/Pass.h"
 #include "lld/Core/PassManager.h"
 #include "lld/Core/Simple.h"
-#include "lld/Passes/RoundTripYAMLPass.h"
 #include "lld/ReaderWriter/CoreLinkingContext.h"
 #include "llvm/ADT/ArrayRef.h"
 
 using namespace lld;
 
 namespace {
 
 /// \brief Simple atom created by the stubs pass.

lib/ReaderWriter/ELF/CMakeLists.txt

 add_llvm_library(lldELF
   ELFLinkingContext.cpp
   Reader.cpp
   Writer.cpp
   LINK_LIBS
     lldCore
-    lldPasses
     lldYAML
     LLVMSupport
   )
 
 include_directories(.)
 
 add_subdirectory(X86)
 add_subdirectory(X86_64)
 add_subdirectory(Mips)
 add_subdirectory(Hexagon)
 add_subdirectory(AArch64)
 add_subdirectory(ARM)

lib/ReaderWriter/ELF/ELFLinkingContext.cpp

 //===- lib/ReaderWriter/ELF/ELFLinkingContext.cpp -------------------------===//
 //
 //                             The LLVM Linker
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "lld/ReaderWriter/ELFLinkingContext.h"
 #include "ELFFile.h"
 #include "OrderPass.h"
 #include "TargetHandler.h"
 #include "lld/Core/Instrumentation.h"
 #include "lld/Core/SharedLibraryFile.h"
-#include "lld/Passes/RoundTripYAMLPass.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 
 #if defined(HAVE_CXXABI_H)

lib/ReaderWriter/ELF/Makefile

 ##===- lld/lib/ReaderWriter/ELF/Makefile --------------*- Makefile -*-===##
 #
 #                     The LLVM Compiler Infrastructure
 #
 # This file is distributed under the University of Illinois Open Source
 # License. See LICENSE.TXT for details.
 #
 ##===----------------------------------------------------------------------===##
 
 LLD_LEVEL := ../../..
 LIBRARYNAME := lldELF
-USEDLIBS = lldPasses.a
 
 CPP.Flags += -I$(PROJ_SRC_DIR)/$(LLD_LEVEL)/lib/ReaderWriter/ELF
 
 # these link against this lib
 PARALLEL_DIRS := Hexagon X86 X86_64 Mips AArch64 ARM
 
 include $(LLD_LEVEL)/Makefile

lib/ReaderWriter/MachO/CMakeLists.txt

add_llvm_library(lldMachO
   MachONormalizedFileFromAtoms.cpp
   MachONormalizedFileToAtoms.cpp
   MachONormalizedFileYAML.cpp
   ShimPass.cpp
   StubsPass.cpp
   WriterMachO.cpp
   LINK_LIBS
     lldCore
-    lldPasses
     lldYAML
     LLVMObject
     LLVMSupport
   )
 
 include_directories(.)

lib/ReaderWriter/MachO/MachOLinkingContext.cpp

 #include "File.h"
 #include "MachONormalizedFile.h"
 #include "MachOPasses.h"
 #include "lld/Core/ArchiveLibraryFile.h"
 #include "lld/Core/PassManager.h"
 #include "lld/Core/Reader.h"
 #include "lld/Core/Writer.h"
 #include "lld/Driver/Driver.h"
-#include "lld/Passes/RoundTripYAMLPass.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MachO.h"
 #include "llvm/Support/Path.h"

lib/ReaderWriter/PECOFF/CMakeLists.txt

 add_llvm_library(lldPECOFF
   EdataPass.cpp
   IdataPass.cpp
   LinkerGeneratedSymbolFile.cpp
   LoadConfigPass.cpp
   PECOFFLinkingContext.cpp
   Pass.cpp
   ReaderCOFF.cpp
   ReaderImportHeader.cpp
   WriterImportLibrary.cpp
   WriterPECOFF.cpp
   LINK_LIBS
     lldCore
-    lldPasses
     LLVMObject
     LLVMSupport
   )

test/elf/Mips/pc23-range.test

 # Check that LLD shows an error if ADDIUPC immediate is out of range.
 
 # RUN: yaml2obj -format=elf %s > %t-obj
-# RUN: env LLD_RUN_ROUNDTRIP_TEST= \
-# RUN:   lld -flavor gnu -target mipsel -o %t-exe %t-obj 2>&1 | FileCheck %s
+# RUN: lld -flavor gnu -target mipsel -o %t-exe %t-obj 2>&1 | FileCheck %s
 
 # CHECK: The addiupc instruction immediate 0x02000008 is out of range
 
 !ELF
 FileHeader: !FileHeader
   Class:   ELFCLASS32
   Data:    ELFDATA2LSB
   Type:    ET_REL

test/elf/roundtrip.test

-# This tests the functionality of the RoundTrip Passes and verifies
-# that the atoms belong to the native file after the passes finish
-
-# REQUIRES: asserts
-
-RUN: lld -flavor gnu -target x86_64 %p/Inputs/foo.o.x86-64 --noinhibit-exec  \
-RUN: --output-filetype=yaml -o %t1
-RUN: FileCheck %s < %t1
-
-CHECK:path:{{.*}}.native
-

test/lit.cfg

 config.test_format = lit.formats.ShTest(execute_external)
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.objtxt', '.test']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)
 
-# run RoundTrip{YAML,Native}Tests.
-config.environment['LLD_RUN_ROUNDTRIP_TEST'] = '1'
-
 # test_exec_root: The root path where tests should be run.
 lld_obj_root = getattr(config, 'lld_obj_root', None)
 if lld_obj_root is not None:
     config.test_exec_root = os.path.join(lld_obj_root, 'test')
 
 # Set llvm_{src,obj}_root for use by others.
 config.llvm_src_root = getattr(config, 'llvm_src_root', None)
 config.llvm_obj_root = getattr(config, 'llvm_obj_root', None)

tools/lld/Makefile

 TOOL_NO_EXPORTS = 1
 
 #include /Makefile.config
 LEVEL := $(LLD_LEVEL)/../..
 include $(LEVEL)/Makefile.config
 
 LINK_COMPONENTS := $(TARGETS_TO_BUILD)
 USEDLIBS = lldDriver.a lldConfig.a \
-           lldELF.a lldMachO.a lldPasses.a lldPECOFF.a lldYAML.a \
+           lldELF.a lldMachO.a lldPECOFF.a lldYAML.a \
            lldReaderWriter.a lldCore.a lldNative.a \
            lldHexagonELFTarget.a lldMipsELFTarget.a \
            lldX86ELFTarget.a lldX86_64ELFTarget.a lldAArch64ELFTarget.a \
            lldARMELFTarget.a \
            LLVMOption.a
 
 include $(LLD_LEVEL)/Makefile

unittests/DriverTests/Makefile

 ##===---- unittests/DriverTests/Makefile ----------------*- Makefile -*-===##
 #
 #                     The LLVM Compiler Infrastructure
 #
 # This file is distributed under the University of Illinois Open Source
 # License. See LICENSE.TXT for details.
 #
 ##===-------------------------------------------------------------------===##
 
 LLD_LEVEL = ../..
 TESTNAME = DriverTests
 USEDLIBS = lldDriver.a lldConfig.a \
-           lldELF.a lldMachO.a lldPasses.a lldPECOFF.a \
+           lldELF.a lldMachO.a lldPECOFF.a \
            lldCore.a lldNative.a lldReaderWriter.a \
            lldHexagonELFTarget.a lldMipsELFTarget.a \
            lldX86ELFTarget.a lldX86_64ELFTarget.a lldYAML.a \
            lldAArch64ELFTarget.a lldARMELFTarget.a \
            LLVMObject.a LLVMMCParser.a LLVMMC.a LLVMBitReader.a \
            LLVMCore.a LLVMOption.a LLVMSupport.a
 include $(LLD_LEVEL)/unittests/Makefile