[gn build] Make build fully deterministic

[thakis]

Summary

This follows http://blog.llvm.org/2019/11/deterministic-builds-with-clang-and-lld.html
after ead0b76382a5a to make the GN build fully deterministic, instead of just locally
deterministic.

Getting a fully deterministic build requires some configuration and setup. See
llvm/utils/gn/docs/deterministic.md for details.

With this, I've built lld at two different build paths on my Windows box and got
identical binaries. (I'd expect the same to happen on Linux, and with other
binaries.)

This is not 100% complete yet. A few absolute include paths need to become sysroot-relative
if a sysroot is in use (there are FIXMEs for this), and the sysroot setup is a pain.

I'm sending this out for early feedback. Maybe I should put the sysroot bits in a separate
patch and land only the rest first.

[rnk]

llvm/utils/gn/build/BUILD.gn

21

Can this be shortened to use_relative_paths_in_debug_info? It seems descriptive enough. Are there non-deterministic relative paths?

llvm/utils/gn/docs/deterministic.md

25

I was going to ask about this, but I think I agree it should be future work after a second individual has replicated your results manually. :)

71

One thing to note is that if this points at the main MSVC installation, the link may become stale. The Visual Studio updater deletes old MSVC versions and adds new ones in a newly versioned directory. We may eventually want a script that detects the stale junctions and repoints them to the current installation.

[thakis]

Thanks for taking a look!

llvm/utils/gn/build/BUILD.gn

21

Done. I pulled the non-sysroot bits into D74519 which has fewer fixmes and is ready to land today. I'll rebase this on top of D74519 once D74519 is in.

[phosek]

llvm/utils/gn/docs/deterministic.md

27

I've build a tool for creating Debian-based sysroots that we use in Fuchsia, it's a single Go file plus YAML file that's used to describe what you want in your sysroot: https://fuchsia.googlesource.com/fuchsia/+/refs/heads/master/tools/debroot/cmd/, this might be useful in case people want to create their own sysroot and don't want to do it by hand.

[thakis]

rebase (but still incomplete)

llvm/utils/gn/.gitignore

 bin
+sysroot

llvm/utils/gn/build/BUILD.gn

declare_args() {
   llvm_build_instrumented_coverage = false
 
   # If set, puts relative paths in debug info.
   # Makes the build output independent of the build directory, but makes
   # most debuggers harder to use. See "Getting to local determinism" and
   # "Getting debuggers to work well with locally deterministic builds" in
   # http://blog.llvm.org/2019/11/deterministic-builds-with-clang-and-lld.html
   # for more information.
   use_relative_paths_in_debug_info = false
+
+  # If set, uses a sysroot for finding headers.
+  # To have a deterministic, machine-independent path to the sysroot on
+  # compile commands, the path for the sysroot is always
+  # llvm/utils/gn/sysroot/linux/{x64,arm,arm64,...} (linux),
+  # llvm/utils/gn/sysroot/mac (mac),
+  # llvm/utils/gn/sysroot/win/sdk/{include,lib} and
+  # llvm/utils/gn/sysroot/win/msvc/{include,lib} (win). See
+  # docs/deterministic.md for ways to set up these directories.
+  use_sysroot = false
 }
 
 assert(!llvm_build_instrumented_coverage || is_clang,
        "llvm_build_instrumented_coverage requires clang as host compiler")
 
 config("compiler_defaults") {
   defines = []
 
   if (!llvm_enable_assertions) {
     defines += [ "NDEBUG" ]
   }
 
   asmflags = target_flags
   cflags = target_flags
   ldflags = target_flags + target_ldflags
 
-  if (host_os == "mac" && clang_base_path != "") {
+  if (host_os == "mac" && clang_base_path != "" && !use_sysroot) {
     cflags += [
       "-isysroot",
       mac_sdk_path,
     ]
   }
 
   if (host_os != "win") {
     if (is_debug) {
if (is_clang) {
     }
     if (use_relative_paths_in_debug_info) {
       cflags += [
         "-fdebug-compilation-dir",
         ".",
       ]
     }
   }
+  if (use_sysroot) {
+    if (current_os == "linux") {
+      sysroot = "//llvm/utils/gn/sysroot/linux"
+      if (current_cpu == "x64") {
+        sysroot += "/x64"
+      } else {
+        assert(false, "don't know how to use linux sysroot for " + current_cpu)
+      }
+      cflags += [ "--sysroot=" + rebase_path(sysroot, root_build_dir) ]
+    } else if (current_os == "mac") {
+      cflags += [
+        "-isysroot",
+        rebase_path("//llvm/utils/gn/sysroot/mac", root_build_dir),
+      ]
+    } else if (current_os == "win") {
+      # Headers.
+      assert(is_clang, "use_sysroot only works with clang-cl on Windows")
+      sdk_inc = "//llvm/utils/gn/sysroot/win/sdk/include"
+      cflags += [
+        "/X",
+        "-imsvc" +
+            rebase_path("//llvm/utils/gn/sysroot/win/msvc/include",
+                        root_build_dir),
+        "-imsvc" + rebase_path(sdk_inc + "/shared", root_build_dir),
+        "-imsvc" + rebase_path(sdk_inc + "/ucrt", root_build_dir),
+        "-imsvc" + rebase_path(sdk_inc + "/um", root_build_dir),
+        "-imsvc" + rebase_path(sdk_inc + "/winrt", root_build_dir),
+      ]
+
+      # Libraries.
+      sdk_lib = "//llvm/utils/gn/sysroot/win/sdk/lib"
+      msvc_lib = "//llvm/utils/gn/sysroot/win/msvc/lib"
+      if (current_cpu == "x64") {
+        arch = "/x64"
+      } else {
+        assert(false, "don't know how to use win sysroot for " + current_cpu)
+      }
+      ldflags +=  [
+        "/lldignoreenv",
+        "/libpath:" + rebase_path(msvc_lib + arch , root_build_dir),
+        "/libpath:" + rebase_path(sdk_lib + "/shared" + arch, root_build_dir),
+        "/libpath:" + rebase_path(sdk_lib + "/ucrt" + arch, root_build_dir),
+        "/libpath:" + rebase_path(sdk_lib + "/um" + arch, root_build_dir),
+        # XXX why do i need winrt in include but not in lib?
+      ]
+    } else {
+      # FIXME: android?
+      assert(false, "don't know how to use sysroot for " + current_os)
+    }
+  }
 }
 
 config("no_exceptions") {
   if (host_os != "win") {
     cflags_cc = [ "-fno-exceptions" ]
   }
 }
 

llvm/utils/gn/build/libs/xml/BUILD.gn

 import("//llvm/utils/gn/build/libs/xml/enable.gni")
 import("//llvm/utils/gn/build/mac_sdk.gni")
 
 config("xml_config") {
   visibility = [ ":xml" ]
   libs = [ "xml2" ]
   if (host_os == "mac") {
+    # FIXME: sysroot
     include_dirs = [ "$mac_sdk_path/usr/include/libxml2" ]
   } else {
+    # FIXME: sysroot
     include_dirs = [ "/usr/include/libxml2" ]
   }
 }
 
 group("xml") {
   if (llvm_enable_libxml2) {
     public_configs = [ ":xml_config" ]
   }
 }

llvm/utils/gn/docs/deterministic.md

 Deterministic builds with LLVM's GN build
 =========================================
 
-Summary: Use the following args.gn.
+Summary: Use the following args.gn, and create a bunch of symlinks so that
+the relative compiler path and the default relative symlink paths work.
 
+    clang_base_path = "../../llvm/utils/gn/toolchain/win"
     use_relative_paths_in_debug_info = true
+    use_sysroot = true
 
-It is possible to produce [locally deterministic][1] builds of LLVM
+It is possible to produce [universally deterministic][1] builds of LLVM
 with the GN build. It requires some configuration though.
 
 1. Make debug info use relative paths by setting
    `use_relative_paths_in_debug_info = true` in your `args.gn` file. With this
    set, current debuggers need minor configuration to keep working.  See
    "Getting to local determinism" and "Getting debuggers to work well with
    locally deterministic builds" in the [deterministic builds][1] documentation
    for details.
 
+2. Make the build use a sysroot for headers by setting `use_sysroot = true`
+   in your `args.gn` file. You need to create a symlink to your sysroot
+   at a fixed location for things to work.
+
+   FIXME: Add scripts to make this less of a drag to set up.
+
+   FIXME: This points to the chromium sysroots as examples for now.
+
+   FIXME: The chromium debian sid sysroot does not contain ICU headers,
+   which libxml2 depends on by default.
+
+    * Linux: `ln -s .../real/path/to/sysroot llvm/utils/gn/sysroot/linux/x64`
+
+          $ mkdir -p llvm/utils/gn/sysroot/linux
+          $ ln -s $PWD/../chrome/src/build/linux/debian_sid_amd64-sysroot \
+                  llvm/utils/gn/sysroot/linux/x64
+
+    * Mac: `ln -s .../real/path/to/mac/sdk llvm/utils/gn/sysroot/mac`. You could
+      use `ln -s $(xcrun -show-sdk-path) llvm/utils/gn/sysroot/mac`, but the
+      idea in using a sysroot is that you don't need to register your sdk of
+      choice with `xcrun`, so normally it'd just be a local directory
+      containing a fixed SDK.
+
+    * Windows:
+
+          >mkdir llvm\utils\gn\sysroot\win\sdk\include
+          
+          > set P=c:\src\chrome\src\third_party\depot_tools\win_toolchain\vs_files\8f58c55897a3282ed617055775a77ec3db771b88
+          
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\include\ucrt ^
+                     %P%\win_sdk\Include\10.0.18362.0\ucrt
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\include\shared ^
+                     %P%\win_sdk\Include\10.0.18362.0\shared
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\include\um ^
+                     %P%\win_sdk\Include\10.0.18362.0\um
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\include\winrt ^
+                     %P%\win_sdk\Include\10.0.18362.0\winrt
+          
+          >mkdir llvm\utils\gn\sysroot\win\sdk\lib
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\lib\ucrt ^
+                     %P%\win_sdk\Lib\10.0.18362.0\ucrt
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\lib\shared ^
+                     %P%\win_sdk\Lib\10.0.18362.0\shared
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\lib\um ^
+                     %P%\win_sdk\Lib\10.0.18362.0\um
+          >mklink /j llvm\utils\gn\sysroot\win\sdk\lib\winrt ^
+                     %P%\win_sdk\Lib\10.0.18362.0\winrt
+          
+          >mkdir llvm\utils\gn\sysroot\win\msvc
+          >mklink /j llvm\utils\gn\sysroot\win\msvc\include ^
+                     %P%\VC\Tools\MSVC\14.23.28105\include
+          >mklink /j llvm\utils\gn\sysroot\win\msvc\lib ^
+                     %P%\VC\Tools\MSVC\14.23.28105\lib
+
+
+3. Use the same host compiler / linker. (FIXME: Bootstrap builds)
+   Use a relative path for this! Else the path to the host compiler won't
+   be directory-independent.
+   Use e.g. `clang_base_path = "../../llvm/utils/gn/toolchain/win"`
+
+        >mklink /j llvm\utils\gn\toolchain\win ^
+                   "c:\src\chrome\src\third_party\llvm-build\Release+Asserts"
+
+        $ mkdir llvm/utils/gn/toolchain
+        $ ln -s $PWD/../chrome/src/third_party/llvm-build/Release+Asserts \
+                llvm/utils/gn/toolchain/linux
+
+
+4. FIXME: The mac build still uses system ld64, which makes links
+   system-dependent.
+
 1: http://blog.llvm.org/2019/11/deterministic-builds-with-clang-and-lld.html

llvm/utils/gn/secondary/llvm/lib/DebugInfo/PDB/enable_dia.gni

 declare_args() {
   # Whether to build code that requires the Microsoft DIA SDK.
   # If this is set, %INCLUDE% must contain ".../DIA SDK/include"
   # and %LIB% must contain ".../DIA SKD/lib/amd64".
+  # XXX use_sysroot
   llvm_enable_dia_sdk = false
 }