gn build: Add a stage2 toolchain for Android.

[pcc]

Summary

This makes it possible to build llvm-symbolizer for Android, which
is one of the prerequisites for running the sanitizer tests on Android.

Depends on D56427

Depends on D56575

Depends on D56576

[thakis]

Super cool!

llvm/utils/gn/build/libs/pthread/BUILD.gn

9 ↗

(On Diff #181208)

Maybe add a # On Android, bionic has built-in support for pthreads. like comment (I had to look that up just now).

llvm/utils/gn/build/toolchain/BUILD.gn

49 ↗

(On Diff #181208)

(this might be a git merge issue -- be sure to run git ls-files '*.gn' '*.gni' | xargs -n 1 gn format though :-)

158 ↗

(On Diff #181208)

Whoa, do deps on toolchains work now? Is that new?

169 ↗

(On Diff #181208)

That's alright for now, but I think eventually we want to remove the target_foo stuff from the toolchain definition again and instead use configs for this (like https://cs.chromium.org/chromium/src/build/config/compiler/BUILD.gn?type=cs&q=build/config/compiler&sq=package:chromium&g=0&l=217)

172 ↗

(On Diff #181208)

The -fuse-ld=lld will probably need revisiting -- probably want to also add that on Linux when e.g. clang_base_path is set, and maybe if some use_lld arg is explicitly set to on. But ok for now.

[phosek]

llvm/utils/gn/build/toolchain/BUILD.gn

158 ↗

(On Diff #181208)

It's been around for a while, but it's discouraged. gn help toolchain says for toolchain deps:

This concept is somewhat inefficient to express in Ninja (it requires a lot
of duplicate of rules) so should only be used when absolutely necessary.

I don't think there's a cleaner way for multi-stage builds though since you need to somehow express that dependency.

169 ↗

(On Diff #181208)

That would be my preference. Rather than combining different ways of specifying flags, we should just introduce default configs for each platform and then add those to defaults for each target type (e.g. https://fuchsia.googlesource.com/build/+/master/config/BUILDCONFIG.gn#146)

[phosek]

llvm/utils/gn/build/toolchain/BUILD.gn

172 ↗

(On Diff #181208)

I think that should also be a config that's set as a default. If it's done that way, targets can then opt-out by using:

configs -= ["//build/config:lld"]

[pcc]

llvm/utils/gn/build/toolchain/BUILD.gn

49 ↗

(On Diff #181208)

Yes, I think I might have forgotten to format D56576. This change restores the correct formatting though.

169 ↗

(On Diff #181208)

We'll see how that goes. One issue is that we'll need some way of writing the target flags to a file to run the tests:
http://llvm-cs.pcc.me.uk/projects/compiler-rt/test/hwasan/lit.site.cfg.in#5
and I'm not sure how easy that will be with a config.

llvm/trunk/utils/gn/build/BUILDCONFIG.gn

 }
 set_defaults("shared_library") {
   configs = shared_binary_target_configs
 }
 set_defaults("source_set") {
   configs = shared_binary_target_configs
 }
 
+if (target_os == "") {
+  target_os = host_os
+}
+if (current_os == "") {
+  current_os = target_os
+}
+
 if (host_os == "win") {
   host_toolchain = "//llvm/utils/gn/build/toolchain:win"
 } else {
   host_toolchain = "//llvm/utils/gn/build/toolchain:unix"
 }
 
 set_default_toolchain(host_toolchain)

llvm/trunk/utils/gn/build/libs/pthread/BUILD.gn

 import("//llvm/utils/gn/build/libs/pthread/enable.gni")
 
 config("pthread_config") {
   visibility = [ ":pthread" ]
   libs = [ "pthread" ]
 }
 
 group("pthread") {
-  if (llvm_enable_threads && host_os != "win") {
+  # On Android, bionic has built-in support for pthreads.
+  if (llvm_enable_threads && current_os != "win" && current_os != "android") {
     public_configs = [ ":pthread_config" ]
   }
 }

llvm/trunk/utils/gn/build/toolchain/BUILD.gn

 import("//llvm/utils/gn/build/toolchain/compiler.gni")
 
 declare_args() {
   # If is_goma is true, the location of the goma client install.
   if (host_os == "win") {
     goma_dir = "c:\src\goma\goma-win64"
   } else {
     goma_dir = getenv("HOME") + "/goma"
   }
 }
 
 template("unix_toolchain") {
   toolchain(target_name) {
     forward_variables_from(invoker, "*")
+    if (!defined(target_cflags)) {
+      target_cflags = ""
+    }
+    if (!defined(target_ldflags)) {
+      target_ldflags = ""
+    }
 
     tool("cc") {
       depfile = "{{output}}.d"
-      command = "$cc -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}}"
+      command = "$cc -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} $target_cflags"
       depsformat = "gcc"
       description = "CC {{output}}"
       outputs = [
         "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
       ]
     }
 
     tool("cxx") {
       depfile = "{{output}}.d"
-      command = "$cxx -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}}"
+      command = "$cxx -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} $target_cflags"
       depsformat = "gcc"
       description = "CXX {{output}}"
       outputs = [
         "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
       ]
     }
 
     tool("alink") {
       if (current_os == "mac") {
         command = "libtool -static -no_warning_for_no_symbols {{arflags}} -o {{output}} {{inputs}}"
       } else {
         # Remove the output file first so that ar doesn't try to modify the
         # existing file.
-        command = "rm -f {{output}} && $ar rcsDT {{arflags}} {{output}} {{inputs}}"
+        command =
+            "rm -f {{output}} && $ar rcsDT {{arflags}} {{output}} {{inputs}}"
       }
       description = "AR {{output}}"
       outputs = [
         "{{output_dir}}/{{target_output_name}}.a",
       ]
       output_prefix = "lib"
       default_output_dir = "{{root_out_dir}}/lib"
     }
 
     tool("solink") {
       outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
       if (current_os == "mac") {
-        command = "$ld -shared {{ldflags}} -o $outfile {{libs}} {{inputs}}"
+        command = "$ld -shared {{ldflags}} -o $outfile {{libs}} {{inputs}} $target_ldflags"
         default_output_extension = ".dylib"
       } else {
-        command =
-            "$ld -shared {{ldflags}} -Wl,-z,defs -o $outfile {{libs}} {{inputs}}"
+        command = "$ld -shared {{ldflags}} -Wl,-z,defs -o $outfile {{libs}} {{inputs}} $target_ldflags"
         default_output_extension = ".so"
       }
       description = "SOLINK $outfile"
       outputs = [
         outfile,
       ]
       lib_switch = "-l"
       output_prefix = "lib"
       default_output_dir = "{{root_out_dir}}/lib"
     }
 
     tool("solink_module") {
       outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
       if (current_os == "mac") {
-        command = "$ld -shared {{ldflags}} -Wl,-flat_namespace -Wl,-undefined,suppress -o $outfile {{libs}} {{inputs}}"
+        command = "$ld -shared {{ldflags}} -Wl,-flat_namespace -Wl,-undefined,suppress -o $outfile {{libs}} {{inputs}} $target_ldflags"
         default_output_extension = ".dylib"
       } else {
-        command = "$ld -shared {{ldflags}} -o $outfile {{libs}} {{inputs}}"
+        command = "$ld -shared {{ldflags}} -o $outfile {{libs}} {{inputs}} $target_ldflags"
         default_output_extension = ".so"
       }
       description = "SOLINK $outfile"
       outputs = [
         outfile,
       ]
       lib_switch = "-l"
       default_output_dir = "{{root_out_dir}}/lib"
     }
 
     tool("link") {
       outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
       if (current_os == "mac") {
-        command = "$ld {{ldflags}} -o $outfile {{libs}} {{inputs}}"
+        command =
+            "$ld {{ldflags}} -o $outfile {{libs}} {{inputs}} $target_ldflags"
       } else {
-        command = "$ld {{ldflags}} -o $outfile {{libs}} -Wl,--start-group {{inputs}} -Wl,--end-group"
+        command = "$ld {{ldflags}} -o $outfile {{libs}} -Wl,--start-group {{inputs}} -Wl,--end-group $target_ldflags"
       }
       description = "LINK $outfile"
       outputs = [
         outfile,
       ]
       lib_switch = "-l"
 
       # Setting this allows targets to override the default executable output by
unix_toolchain("unix") {
 
   ar = "ar"
 
   toolchain_args = {
     current_os = host_os
   }
 }
 
+if (android_ndk_path != "") {
+  unix_toolchain("stage2_android_aarch64") {
+    cc = "bin/clang"
+    cxx = "bin/clang++"
+    ld = cxx
+    ar = "bin/llvm-ar"
+
+    deps = [
+      "//:clang($host_toolchain)",
+      "//:lld($host_toolchain)",
+      "//:llvm-ar($host_toolchain)",
+    ]
+
+    toolchain_args = {
+      current_os = "android"
+    }
+
+    libcxx_path = "$android_ndk_path/sources/cxx-stl/llvm-libc++"
+    platform_lib_path =
+        "$android_ndk_path/platforms/android-21/arch-arm64/usr/lib"
+    libgcc_path = "$android_ndk_path/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x"
+
+    target_flags =
+        "--target=aarch64-linux-android21 --sysroot=$android_ndk_path/sysroot"
+    target_cflags = "$target_flags -isystem $libcxx_path/include"
+    target_ldflags = "$target_flags -fuse-ld=lld -B$platform_lib_path -L$platform_lib_path -L$libgcc_path"
+    target_ldflags +=
+        " -nostdlib++ -L$libcxx_path/libs/arm64-v8a -l:libc++.a.21"
+  }
+}
+
 toolchain("win") {
   cl = "cl"
   link = "link"
 
   if (clang_base_path != "") {
     cl = "$clang_base_path/bin/clang-cl"
     link = "$clang_base_path/bin/lld-link"
   }

llvm/trunk/utils/gn/build/toolchain/compiler.gni

 declare_args() {
   # Whether to use goma (https://chromium.googlesource.com/infra/goma/client/)
   use_goma = false
 
   # Set this to a clang build directory. If set, that clang is used as compiler.
   # goma only works with compiler binaries it knows about, so useful both for
   # using a goma-approved compiler and for compiling clang with a locally-built
   # clang in a different build directory.
   # On Windows, setting this also causes lld-link to be used as linker.
   # Example value: getenv("HOME") + "/src/llvm-build/Release+Asserts"
   clang_base_path = ""
+
+  # Set this to the path to Android NDK r18b. If set, cross compilation targeting
+  # Android will be enabled.
+  android_ndk_path = ""
 }
 
 declare_args() {
   # Set if the host compiler is clang.  On by default on Mac or if
   # clang_base_path is set.
   is_clang = host_os == "mac" || clang_base_path != ""
 }

llvm/trunk/utils/gn/secondary/BUILD.gn

 import("//clang/lib/ARCMigrate/enable.gni")
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+import("//llvm/utils/gn/build/toolchain/compiler.gni")
 
 group("default") {
   deps = [
     "//clang/test",
     "//lld/test",
     "//llvm/test",
   ]
+
+  if (android_ndk_path != "") {
+    # FIXME: This should be a dependency of a test target instead of being
+    # depended on from here.
+    deps += [ "//llvm/tools/llvm-symbolizer(//llvm/utils/gn/build/toolchain:stage2_android_aarch64)" ]
+  }
+
   testonly = true
 }
 
 # Symlink handling.
 # On POSIX, symlinks to the target can be created before the target exist,
 # and the target can depend on the symlink targets, so that building the
 # target ensures the symlinks exist.
 # However, symlinks didn't exist on Windows until recently, so there the

llvm/trunk/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn

values = [
     "LLVM_GISEL_COV_ENABLED=",
     "LLVM_GISEL_COV_PREFIX=",
 
     # This is both in llvm-config.h and config.h; llvm-config.h doesn't
     # define it if it's not set while config.h defines it to empty in that case.
     "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple",
   ]
 
-  if (host_os == "linux") {
+  if (current_os == "linux" || current_os == "android") {
     values += [
       "HAVE_FUTIMENS=1",
       "HAVE_LINK_H=1",
       "HAVE_LSEEK64=1",
       "HAVE_MALLINFO=1",
       "HAVE_POSIX_FALLOCATE=1",
       "HAVE_SCHED_GETAFFINITY=1",
       "HAVE_CPU_COUNT=1",
       "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC=1",
-      "HAVE_VALGRIND_VALGRIND_H=1",
     ]
   } else {
     values += [
       "HAVE_FUTIMENS=",
       "HAVE_LINK_H=",
       "HAVE_LSEEK64=",
       "HAVE_MALLINFO=",
       "HAVE_POSIX_FALLOCATE=",
       "HAVE_SCHED_GETAFFINITY=",
       "HAVE_CPU_COUNT=",
       "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC=",
-      "HAVE_VALGRIND_VALGRIND_H=",
     ]
   }
 
-  if (host_os == "mac") {
+  if (current_os == "linux") {
+    values += [ "HAVE_VALGRIND_VALGRIND_H=1" ]
+  } else {
+    values += [ "HAVE_VALGRIND_VALGRIND_H=" ]
+  }
+
+  if (current_os == "mac") {
     values += [
       "HAVE_CRASHREPORTER_INFO=1",
       "HAVE_DECL_ARC4RANDOM=1",
       "HAVE_DLADDR=1",
       "HAVE_MALLOC_H=",
       "HAVE_MACH_MACH_H=1",
       "HAVE_MALLOC_MALLOC_H=1",
       "HAVE_MALLOC_ZONE_STATISTICS=1",
       "HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC=1",
     ]
   } else {
     values += [
       "HAVE_CRASHREPORTER_INFO=",
       "HAVE_DECL_ARC4RANDOM=",
       "HAVE_DLADDR=",
       "HAVE_MACH_MACH_H=",
       "HAVE_MALLOC_H=1",
       "HAVE_MALLOC_MALLOC_H=",
       "HAVE_MALLOC_ZONE_STATISTICS=",
       "HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC=",
     ]
   }
 
-  if (host_os == "win") {
+  if (current_os == "linux" || current_os == "mac") {
+    values += [
+      "HAVE_BACKTRACE=1",
+      "HAVE_POSIX_SPAWN=1",
+      "HAVE_PTHREAD_GETNAME_NP=1",
+    ]
+  } else {
     values += [
       "HAVE_BACKTRACE=",
+      "HAVE_POSIX_SPAWN=",
+      "HAVE_PTHREAD_GETNAME_NP=",
+    ]
+  }
+
+  if (current_os == "win") {
+    values += [
       "HAVE_DECL_STRERROR_S=1",
       "HAVE_DLFCN_H=",
       "HAVE_DLOPEN=",
       "HAVE_FUTIMES=",
       "HAVE_GETPAGESIZE=",
       "HAVE_GETRLIMIT=",
       "HAVE_GETRUSAGE=",
       "HAVE_ISATTY=",
       "HAVE_LIBPTHREAD=",
-      "HAVE_PTHREAD_GETNAME_NP=",
       "HAVE_PTHREAD_SETNAME_NP=",
       "HAVE_LIBZ=",
-      "HAVE_POSIX_SPAWN=",
       "HAVE_PREAD=",
       "HAVE_PTHREAD_GETSPECIFIC=",
       "HAVE_PTHREAD_H=",
       "HAVE_PTHREAD_MUTEX_LOCK=",
       "HAVE_PTHREAD_RWLOCK_INIT=",
       "HAVE_REALPATH=",
       "HAVE_SBRK=",
       "HAVE_SETENV=",
values += [
       "HAVE__CHSIZE_S=1",
       "HAVE__UNWIND_BACKTRACE=",
       "stricmp=_stricmp",
       "strdup=_strdup",
     ]
   } else {
     # POSIX-y system defaults.
     values += [
-      "HAVE_BACKTRACE=1",
       "HAVE_DECL_STRERROR_S=",
       "HAVE_DLFCN_H=1",
       "HAVE_DLOPEN=1",
       "HAVE_FUTIMES=1",
       "HAVE_GETPAGESIZE=1",
       "HAVE_GETRLIMIT=1",
       "HAVE_GETRUSAGE=1",
       "HAVE_ISATTY=1",
       "HAVE_LIBPTHREAD=1",
-      "HAVE_PTHREAD_GETNAME_NP=1",
       "HAVE_PTHREAD_SETNAME_NP=1",
       "HAVE_LIBZ=1",
-      "HAVE_POSIX_SPAWN=1",
       "HAVE_PREAD=1",
       "HAVE_PTHREAD_GETSPECIFIC=1",
       "HAVE_PTHREAD_H=1",
       "HAVE_PTHREAD_MUTEX_LOCK=1",
       "HAVE_PTHREAD_RWLOCK_INIT=1",
       "HAVE_REALPATH=1",
       "HAVE_SBRK=1",
       "HAVE_SETENV=1",
values += [
       "HAVE_ZLIB_H=1",
       "HAVE__CHSIZE_S=",
       "HAVE__UNWIND_BACKTRACE=1",
       "stricmp=",
       "strdup=",
     ]
   }
 
-  if (host_os == "linux") {
+  if (current_os == "linux" || current_os == "android") {
     values += [ "LTDL_SHLIB_EXT=.so" ]
-  } else if (host_os == "mac") {
+  } else if (current_os == "mac") {
     values += [ "LTDL_SHLIB_EXT=.dylib" ]
-  } else if (host_os == "win") {
+  } else if (current_os == "win") {
     values += [ "LTDL_SHLIB_EXT=.dll" ]
   }
 
   if (llvm_enable_libedit) {
     values += [ "HAVE_LIBEDIT=1" ]
   } else {
     values += [ "HAVE_LIBEDIT=" ]
   }
values = [
     "LLVM_USE_PERF=",
     "LLVM_VERSION_MAJOR=$llvm_version_major",
     "LLVM_VERSION_MINOR=$llvm_version_minor",
     "LLVM_VERSION_PATCH=$llvm_version_patch",
     "PACKAGE_VERSION=${llvm_version}svn",
     "LLVM_FORCE_ENABLE_STATS=",
   ]
 
-  if (host_os == "win") {
+  if (current_os == "win") {
     values += [ "LLVM_ON_UNIX=" ]
   } else {
     values += [ "LLVM_ON_UNIX=1" ]
   }
 
   if (llvm_enable_threads) {
     values += [ "LLVM_ENABLE_THREADS=1" ]
   } else {
write_cmake_config(target_name) {
     # Build something like
     # `LLVM_ENUM_ASM_PARSERS=LLVM_ASM_PARSER(ARM)\nLLVM_ASM_PARSER(X86)\n`. Note
     # that \n is a literal '\' followed by a literal 'n', not a newline
     # character.  (write_cmake_config.py replaces that with a real newline).
     value = ""
     foreach(target, llvm_targets_to_build) {
       value = "$value${invoker.value}($target)\n"
     }
-    values = [
-      "${invoker.key}=$value",
-    ]
+    values = [ "${invoker.key}=$value" ]
   }
 }
 
 write_target_def_file("AsmParsers.def") {
   key = "LLVM_ENUM_ASM_PARSERS"
   value = "LLVM_ASM_PARSER"
 }