Fixes to distribution example for X86_64 Arch Linux

[winksaville]

Summary

Add runtime libcxxabi, use gold linker and create LLVMgold.so.

Tested with on monorepo with:

mkdir build && cd build && \
cmake ../llvm -G Ninja \
  -C ../clang/cmake/caches/DistributionExample.cmake \
  -DCMAKE_INSTALL_PREFIX=local-opt \
  && \
ninja stage2-distribution && \
cd tools/clang/stage2-bins && \
ninja check-all && \
ninja install-distribution

[winksaville]

This may not "correct" but I had to do these to get ninja stage2-distribution to complete on my computer.

[beanz]

Adding "libcxxabi" to LLVM_ENABLE_RUNTIMES is fine, but the other changes to the DistributionExample are only needed because you chose to use gold, which is a configuration-specific decision that is not representative of how most people will build, therefore it shouldn't be in the example.

I'm also not sure the PLUGIN_TOOL line is correct. That seems to assume that you either set LLVM_ENABLE_LLVM_DYLIB, or have libLLVM pre-installed, which I don't think most people do.

[winksaville]

In D62215#1510933, @beanz wrote:

Adding "libcxxabi" to LLVM_ENABLE_RUNTIMES is fine, but the other changes to the DistributionExample are only needed because you chose to use gold, which is a configuration-specific decision that is not representative of how most people will build, therefore it shouldn't be in the example.

I'm also not sure the PLUGIN_TOOL line is correct. That seems to assume that you either set LLVM_ENABLE_LLVM_DYLIB, or have libLLVM pre-installed, which I don't think most people do.

Actually I used ld.gold and LLVMgold.so just to make the DistributionExample work,
it was not that I wanted or desired to use it.

Here were the initial steps.

I cloned llvm/llvm-project and I'm at sha1 6e19534a
(Note: wink@wink-desktop:~/prgs/llvm/llvm-project-2 (master) is part of my command line prompt):

wink@wink-desktop:~/prgs/llvm/llvm-project-2 (master)
$ git log -1
commit 6e19543a2a2013bd357eb15e383b435cd0cbb810 (HEAD -> master, upstream/master, origin/master, origin/HEAD)
Author: Yi-Hong Lyu <[e-mail]>
Date:   Tue May 21 19:42:57 2019 +0000

    [PowerPC][NFC] Add a tests for Reordering CSR reloads in epilogue to follow the same order as CSR saves in the prologue
    
    llvm-svn: 361299

I then used cmake and ninja stage2-distribution and it fails when trying to build llvm-tblgen with
error loading plugin: ... LLVMgold.so: cannot open shared object file: No such file or directory.
Note: There are no modifications and there is no explicit intention of using ld.gold:

wink@wink-desktop:~/prgs/llvm/llvm-project-2/build-master (master)
$ cmake ../llvm -G Ninja -C ../clang/cmake/caches/DistributionExample.cmake -DCMAKE_INSTALL_PREFIX=~/local-master
...
-- Configuring done
-- Generating done
-- Build files have been written to: /home/wink/prgs/llvm/llvm-project-2/build-master
wink@wink-desktop:~/prgs/llvm/llvm-project-2/build-master (master)
$ ninja -j5 stage2-distribution
[186/2251] Generating VCSRevision.h
-- Found Git: /usr/bin/git (found version "2.21.0") 
[218/2251] Building CXX object lib/IR/CMakeFiles/LLVMCore.dir/Core.cpp.o
...
-- Build files have been written to: /home/wink/prgs/llvm/llvm-project-2/build-master/runtimes/runtimes-bins
[2249/2251] Performing configure step for 'stage2'
loading initial cache file /home/wink/prgs/llvm/llvm-project-2/clang/cmake/caches/DistributionExample-stage2.cmake
-- The C compiler identification is Clang 9.0.0
-- The CXX compiler identification is Clang 9.0.0
-- The ASM compiler identification is Clang
..
[256/2445] Linking CXX executable bin/llvm-tblgen
FAILED: bin/llvm-tblgen 
: && /home/wink/prgs/llvm/llvm-project-2/build-master/./bin/clang++  -fPIC -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -std=c++11 -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wstring-conversion -fdiagnostics-color -ffunction-sections -fdata-sections -flto -O3 -gline-tables-only -DNDEBUG  -flto -Wl,-allow-shlib-undefined    -Wl,-O3 -Wl,--gc-sections utils/TableGen/CMakeFiles/llvm-tblgen.dir/AsmMatcherEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/AsmWriterEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/AsmWriterInst.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/Attributes.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CallingConvEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeEmitterGen.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenDAGPatterns.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenHwModes.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenInstruction.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenMapTable.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenRegisters.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenSchedule.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenTarget.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcherEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcherGen.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcherOpt.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcher.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DFAPacketizerEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DisassemblerEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/ExegesisEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/FastISelEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/FixedLenDecoderEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/GlobalISelEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/InfoByHwMode.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/InstrInfoEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/InstrDocsEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/IntrinsicEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/OptParserEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/PredicateExpander.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/PseudoLoweringEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/RISCVCompressInstEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/RegisterBankEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/RegisterInfoEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SDNodeProperties.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SearchableTableEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SubtargetEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SubtargetFeatureInfo.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/TableGen.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/Types.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86DisassemblerTables.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86EVEX2VEXTablesEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86FoldTablesEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86ModRMFilters.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86RecognizableInstr.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/WebAssemblyDisassemblerEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CTagsEmitter.cpp.o  -o bin/llvm-tblgen  -Wl,-rpath,"\$ORIGIN/../lib" lib/libLLVMSupport.a lib/libLLVMTableGen.a -lpthread lib/libLLVMSupport.a -lz -lrt -ldl -ltinfo -lpthread -lm /usr/lib/libz3.so lib/libLLVMDemangle.a && :
/usr/bin/ld: /home/wink/prgs/llvm/llvm-project-2/build-master/bin/../lib/LLVMgold.so: error loading plugin: /home/wink/prgs/llvm/llvm-project-2/build-master/bin/../lib/LLVMgold.so: cannot open shared object file: No such file or directory

With this error I guessed that I needed to build LLVMgold.so, but then I also determined
I needed to link everything with ld.gold to complete the build process, which is why I
uploaded this patch. Anyway, since you don't seem to like using ld.gold I thought I'd look
into how we might work around that. After spending a yesterday and this morning looking
that I identified that using LTO seems to be causing the need for LLVMgold.so:

My "proof" is that if I take the command line above and just remove the two instances of -flto:
-fdata-sections -flto -O3 -gline-tables-only -DNDEBUG -flto -Wl,-allow-shlib-undefined

So that that section of the command line is now:
-fdata-sections -O3 -gline-tables-only -DNDEBUG -Wl,-allow-shlib-undefined

I then validate llvm-tblgen doesn't exist:

wink@wink-desktop:~/prgs/llvm/llvm-project-2/build-master
$ ./bin/llvm-tblgen --version
bash: ./bin/llvm-tblgen: No such file or directory

Then compile without the two -flto options:

wink@wink-desktop:~/prgs/llvm/llvm-project-2/build-master
$ /home/wink/prgs/llvm/llvm-project-2/build-master/./bin/clang++ -fPIC -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -std=c++11 -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wstring-conversion -fdiagnostics-color -ffunction-sections -fdata-sections -O3 -gline-tables-only -DNDEBUG -Wl,-allow-shlib-undefined    -Wl,-O3 -Wl,--gc-sections utils/TableGen/CMakeFiles/llvm-tblgen.dir/AsmMatcherEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/AsmWriterEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/AsmWriterInst.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/Attributes.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CallingConvEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeEmitterGen.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenDAGPatterns.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenHwModes.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenInstruction.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenMapTable.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenRegisters.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenSchedule.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CodeGenTarget.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcherEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcherGen.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcherOpt.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DAGISelMatcher.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DFAPacketizerEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/DisassemblerEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/ExegesisEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/FastISelEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/FixedLenDecoderEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/GlobalISelEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/InfoByHwMode.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/InstrInfoEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/InstrDocsEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/IntrinsicEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/OptParserEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/PredicateExpander.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/PseudoLoweringEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/RISCVCompressInstEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/RegisterBankEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/RegisterInfoEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SDNodeProperties.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SearchableTableEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SubtargetEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/SubtargetFeatureInfo.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/TableGen.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/Types.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86DisassemblerTables.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86EVEX2VEXTablesEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86FoldTablesEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86ModRMFilters.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/X86RecognizableInstr.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/WebAssemblyDisassemblerEmitter.cpp.o utils/TableGen/CMakeFiles/llvm-tblgen.dir/CTagsEmitter.cpp.o  -o bin/llvm-tblgen  -Wl,-rpath,"\$ORIGIN/../lib" lib/libLLVMSupport.a lib/libLLVMTableGen.a -lpthread lib/libLLVMSupport.a -lz -lrt -ldl -ltinfo -lpthread -lm /usr/lib/libz3.so lib/libLLVMDemangle.a
wink@wink-desktop:~/prgs/llvm/llvm-project-2/build-master

And finally run llvm-tblgen:

$ ./bin/llvm-tblgen --version
LLVM.org LLVM version 9.0.0svn
  Optimized build.
  Default target: x86_64-unknown-linux-gnu
  Host CPU: haswell
wink@wink-desktop:~/prgs/llvm/llvm-project-2/build-master
$ 

This shows to me that we need LLVMgold.so because we're linking with LTO.

So I've found a couple hacks that partially work:

1. Build LLVMgold.so and use ld.gold
2. Don't use LTO

How would you like to proceed?

Note: My solution 1 was able to compile and clang can run, but I tried
ninja check-all and it failed:

[101/341] Performing configure step for 'runtimes'
-- Linker detection: GNU ld
-- Building with -fPIC
CMake Warning (dev) at /home/wink/prgs/llvm/llvm-project/compiler-rt/CMakeLists.txt:64 (set):
  implicitly converting 'BOOLEAN' to 'STRING' type.
This warning is for project developers.  Use -Wno-dev to suppress it.

CMake Warning (dev) at /home/wink/prgs/llvm/llvm-project/compiler-rt/CMakeLists.txt:67 (set):
  implicitly converting 'BOOLEAN' to 'STRING' type.
This warning is for project developers.  Use -Wno-dev to suppress it.

-- Compiler-RT supported architectures: x86_64
-- Linker detection: GNU ld
-- Linker detection: GNU ld
-- Generated Sanitizer SUPPORTED_TOOLS list on "Linux" is "asan;lsan;msan;tsan;ubsan"
-- sanitizer_common tests on "Linux" will run against "asan;lsan;msan;tsan;ubsan"
-- check-shadowcallstack does nothing.
CMake Warning at /home/wink/prgs/llvm/llvm-project/libcxx/cmake/Modules/CheckLibcxxAtomic.cmake:51 (message):
  Host compiler must support std::atomic!
Call Stack (most recent call first):
  /home/wink/prgs/llvm/llvm-project/libcxx/cmake/config-ix.cmake:63 (include)
  /home/wink/prgs/llvm/llvm-project/libcxx/CMakeLists.txt:461 (include)


-- Linker detection: GNU ld
-- Linker detection: GNU ld
-- Linker detection: GNU ld
-- Configuring done
-- Generating done
CMake Warning:
  Manually-specified variables were not used by the project:

    CMAKE_NM


-- Build files have been written to: /home/wink/prgs/llvm/llvm-project/build-dist-a/tools/clang/stage2-bins/runtimes/runtimes-bins
[102/341] cd /home/wink/prgs/llvm/llvm-project/build-dist-a...times-bins/ --target check-runtimes --config RelWithDebInfo
ninja: error: '/home/wink/prgs/llvm/llvm-project/build-dist-a/tools/clang/stage2-bins/lib/libgtest.a', needed by 'compiler-rt/lib/asan/tests/ASAN_INST_TEST_OBJECTS.gtest-all.cc.x86_64-calls.o', missing and no known rule to make it
FAILED: runtimes/CMakeFiles/check-runtimes 
cd /home/wink/prgs/llvm/llvm-project/build-dist-a/tools/clang/stage2-bins/runtimes/runtimes-bins && /usr/bin/cmake --build /home/wink/prgs/llvm/llvm-project/build-dist-a/tools/clang/stage2-bins/runtimes/runtimes-bins/ --target check-runtimes --config RelWithDebInfo
[107/341] Linking CXX executable bin/opt
ninja: build stopped: subcommand failed.

[beanz]

In D62215#1512543, @winksaville wrote:

With this error I guessed that I needed to build LLVMgold.so, but then I also determined
I needed to link everything with ld.gold to complete the build process, which is why I
uploaded this patch.

The issue is actually that your system linker isn't compatible with LLVM/master for LTO, and the LLVM build system doesn't know how to setup linux system linkers automatically. We do properly configure ld64, and lld on Linux and Windows if you configure to tell LLVM to use LLD.

Anyway, since you don't seem to like using ld.gold

It isn't that I don't "like" gold, it is that gold isn't installed by default on most systems, and I'm pretty sure it doesn't exist for Darwin or Win32.

So I've found a couple hacks that partially work:

1. Build LLVMgold.so and use ld.gold

This is a viable configuration, but cannot be part of the example because it isn't portable.

2. Don't use LTO

The real underlying issue is that LLVM's build system isn't producing reasonable errors for when the system linker doesn't support LTO, and it doesn't seem to know how to configure a gold-based LTO bootstrap.

CMake Warning at /home/wink/prgs/llvm/llvm-project/libcxx/cmake/Modules/CheckLibcxxAtomic.cmake:51 (message):

Host compiler must support std::atomic!

This is caused by the issue that I'm trying to address in D62155, and will impact the correctness of the runtime libraries you build.

ninja: error: '/home/wink/prgs/llvm/llvm-project/build-dist-a/tools/clang/stage2-bins/lib/libgtest.a', needed by 'compiler-rt/lib/asan/tests/ASAN_INST_TEST_OBJECTS.gtest-all.cc.x86_64-calls.o', missing and no known rule to make it

This looks like a missing build dependency between the runtime check target and gtest, which should be easy enough to fix. I'll put up a patch.

I actually really appreciate you trying this out and reporting back with such detailed feedback on your experience. Thank you for your patience, and I'm sorry if I'm being a bit of a pain.

I think the correct fix for this is to use lld on non-Darwin platforms, and that can be done by adding the following code to DistributionExample.cmake:

if (NOT APPLE)
  set(BOOTSTRAP_LLVM_ENABLE_LLD On CACHE BOOL "")
endif()

This will force using LLD on non-Darwin builds. This combined with adding "libcxxabi" to LLVM_ENABLE_RUNTIMES should actually get your build working without any further changes (none of the gold stuff should be needed).

Additionally we should probably consider adding better error checking to the bootstrap configurations to error out during configuration on some of these problems. For example we should be able to verify that when BOOTSTRAP_LLVM_ENABLE_LTO=On the following things should be true.

• If your host is Windows, in stage 1 you must be building lld, and in stage 2 you must be linking with it
• If your host is Linux, in stage 1 you must be building lld, and in stage 2 you must be linking with it -or- you must build the Gold plugin in stage1 and link with ld.gold in stage 2.
• If your host is Darwin, you must not be using lld

[beanz]

D62269 should fix the missing dependency on gtest.

[winksaville]

I actually really appreciate you trying this out and reporting back with such detailed feedback on your experience. Thank you for your patience, and I'm sorry if I'm being a bit of a pain.

Been a good learning experince for me, both LLVM and cmake, I hope you don't think I'm being pedantic or odd,
but I'm just trying to verify how things should work. Additionally, I want to get Arch Linux to provide static and
dynamic libraries for clang.

I think the correct fix for this is to use lld on non-Darwin platforms, and that can be done by adding the following code to DistributionExample.cmake:

if (NOT APPLE)
  set(BOOTSTRAP_LLVM_ENABLE_LLD On CACHE BOOL "")
endif()

This will force using LLD on non-Darwin builds. This combined with adding "libcxxabi" to LLVM_ENABLE_RUNTIMES should actually get your build working without any further changes (none of the gold stuff should be needed).

This seems a reasonable solution,

Additionally we should probably consider adding better error checking to the bootstrap configurations to error out during configuration on some of these problems. For example we should be able to verify that when BOOTSTRAP_LLVM_ENABLE_LTO=On the following things should be true.

• If your host is Windows, in stage 1 you must be building lld, and in stage 2 you must be linking with it
• If your host is Linux, in stage 1 you must be building lld, and in stage 2 you must be linking with it -or- you must build the Gold plugin in stage1 and link with ld.gold in stage 2.
• If your host is Darwin, you must not be using lld

IIRC, on linux if LTO is ON I had to use ld.gold for both stage 1 and 2, although that maybe
just the way it ended up and it wasn't necessary. But I'll test whatever you come up with.

[beanz]

In D62215#1512849, @winksaville wrote:

IIRC, on linux if LTO is ON I had to use ld.gold for both stage 1 and 2, although that maybe
just the way it ended up and it wasn't necessary. But I'll test whatever you come up with.

I will try and spin up a VM today to try and test this on Linux using GNU ld for the stage 1. I don't usually do much work on Linux, but when I do I always setup lld as my system linker because it is way more efficient both in speed and memory usage. I don't think I've ever managed to get GNU ld to link a non-debug build of LLVM.

[winksaville]

Rather than creating LLVMgold.so as this change does, @beanz suggested we use a known LTO capable linker, lld. See D62279..

clang/cmake/caches/DistributionExample-stage2.cmake

 # This file sets up a CMakeCache for the second stage of a simple distribution
 # bootstrap build.
 
 set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld" CACHE STRING "")
-set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "")
+
+set(LLVM_USE_LINKER "gold" CACHE STRING "")
 
 set(LLVM_TARGETS_TO_BUILD X86;ARM;AArch64 CACHE STRING "")
 
 set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -gline-tables-only -DNDEBUG" CACHE STRING "")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -gline-tables-only -DNDEBUG" CACHE STRING "")
 
 # setup toolchain

clang/cmake/caches/DistributionExample.cmake

 # This file sets up a CMakeCache for a simple distribution bootstrap build.
 
 #Enable LLVM projects and runtimes
 set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld" CACHE STRING "")
-set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "")
 
 # Only build the native target in stage1 since it is a throwaway build.
 set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
 
 # Optimize the stage1 compiler, but don't LTO it because that wastes time.
 set(CMAKE_BUILD_TYPE Release CACHE STRING "")
 
 # Setup vendor-specific settings.
set(CLANG_BOOTSTRAP_TARGETS
   llvm-test-depends
   clang-test-depends
   distribution
   install-distribution
   clang CACHE STRING "")
 
 # Setup the bootstrap build.
 set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
+set(LLVM_USE_LINKER "gold" CACHE STRING "")
+set(LLVM_BINUTILS_INCDIR "/usr/include" CACHE PATH "")
+set(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS ON CACHE BOOL "")
 
 if(STAGE2_CACHE_FILE)
   set(CLANG_BOOTSTRAP_CMAKE_ARGS
     -C ${STAGE2_CACHE_FILE}
     CACHE STRING "")
 else()
   set(CLANG_BOOTSTRAP_CMAKE_ARGS
     -C ${CMAKE_CURRENT_LIST_DIR}/DistributionExample-stage2.cmake
     CACHE STRING "")
 endif()

llvm/tools/gold/CMakeLists.txt

 set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/gold.exports)
 
 if( LLVM_ENABLE_PIC AND LLVM_BINUTILS_INCDIR )
   include_directories( ${LLVM_BINUTILS_INCDIR} )
 
   set(LLVM_LINK_COMPONENTS
      ${LLVM_TARGETS_TO_BUILD}
      Linker
      LTO
      BitWriter
      IPO
      )
 
   add_llvm_library(LLVMgold MODULE
     gold-plugin.cpp
+    PLUGIN_TOOL LLVM
     )
 
 endif()