Add llvm-go tool.

[pcc]

Summary

This tool lets us build LLVM components within the tree by setting up a
$GOPATH that resembles a tree fetched in the normal way with "go get".

It is intended that components such as the Go frontend will be built in-tree
using this tool.

TODO: autotools.

[pcc]

• Fix failure behavior, fix symlinks, allow environment variables to be set from command line

[ruiu]

bindings/go/build.sh

31 ↗

(On Diff #15229)

I think generated files shouldn't be written to the LLVM source tree. Instead it should be written to the build directory.

tools/llvm-go/llvm-go.go

35 ↗

(On Diff #15229)

compilerFlags

36 ↗

(On Diff #15229)

drop "flags" from field name.

103 ↗

(On Diff #15229)

This should be

args[i+1] = args[i+1] + " " + tag

?

120 ↗

(On Diff #15229)

You may want to a comment that this file is auto-generated by llvm-go.go.

152 ↗

(On Diff #15229)

Need to check the return value. On Windows I think it will return an error.

189 ↗

(On Diff #15229)

Check the error.

209 ↗

(On Diff #15229)

Print the help message if none above.

[ruiu]

tools/llvm-go/llvm-go.go

223 ↗

(On Diff #15286)

Go supports Java-like labeled for loop.

DONE:
for {
  switch {
  ...
  default:
    break DONE;
  }
}

[pcc]

• Address review comments, disable bindings on Windows

bindings/go/build.sh

31 ↗

(On Diff #15229)

Normally that is the case, but the build.sh script is used in the case where we go get LLVM, so there is no build directory as such.

I don't think we necessarily need to generate version.go though. Separately from this change, I'll investigate if we can change this to something else that does not require us to generate it here.

tools/llvm-go/llvm-go.go

223 ↗

(On Diff #15286)

Done.

35 ↗

(On Diff #15229)

Done.

36 ↗

(On Diff #15229)

Done.

103 ↗

(On Diff #15229)

Yes, done.

120 ↗

(On Diff #15229)

Done.

152 ↗

(On Diff #15229)

Done. That reminds me that I should disable the Go stuff altogether on Windows for now.

189 ↗

(On Diff #15229)

Done.

209 ↗

(On Diff #15229)

Done.

[ruiu]

LGTM

[pcc]

Closed by commit rL220462 (authored by @pcc).

llvm/trunk/bindings/go/build.sh

 #!/bin/sh -xe
 
-llvm_components="\
-all-targets \
-analysis \
-asmparser \
-asmprinter \
-bitreader \
-bitwriter \
-codegen \
-core \
-debuginfo \
-executionengine \
-instrumentation \
-interpreter \
-ipo \
-irreader \
-linker \
-mc \
-mcjit \
-objcarcopts \
-option \
-profiledata \
-scalaropts \
-support \
-target \
-"
-
-if [ "$1" = "--print-components" ] ; then
-  echo $llvm_components
-  exit 0
-fi
-
 gollvmdir=$(dirname "$0")/llvm
 
 workdir=$gollvmdir/workdir
 llvmdir=$gollvmdir/../../..
 llvm_builddir=$workdir/llvm_build
 
 mkdir -p $llvm_builddir
 
 cmake_flags="../../../../.. $@"
 llvm_config="$llvm_builddir/bin/llvm-config"
+llvm_go="$llvm_builddir/bin/llvm-go"
 
 if test -n "`which ninja`" ; then
   # If Ninja is available, we can speed up the build by building only the
   # required subset of LLVM.
   (cd $llvm_builddir && cmake -G Ninja $cmake_flags)
-  ninja -C $llvm_builddir llvm-config
+  ninja -C $llvm_builddir llvm-config llvm-go
+  llvm_components="$($llvm_go print-components)"
   llvm_buildtargets="$($llvm_config --libs $llvm_components | sed -e 's/-l//g')"
   ninja -C $llvm_builddir $llvm_buildtargets FileCheck
 else
   (cd $llvm_builddir && cmake $cmake_flags)
   make -C $llvm_builddir -j4
 fi
 
+$llvm_go print-config > $gollvmdir/llvm_config.go
+
 llvm_version="$($llvm_config --version)"
-llvm_cflags="$($llvm_config --cppflags)"
-llvm_ldflags="$($llvm_config --ldflags) $($llvm_config --libs $llvm_components) $($llvm_config --system-libs)"
-if [ $(uname) != "Darwin" ]; then
-  # OS X doesn't like -rpath with cgo. See:
-  # https://code.google.com/p/go/issues/detail?id=7293
-  llvm_ldflags="-Wl,-rpath,$($llvm_config --libdir) $llvm_ldflags"
-fi
-sed -e "s#@LLVM_CFLAGS@#$llvm_cflags#g; s#@LLVM_LDFLAGS@#$llvm_ldflags#g" $gollvmdir/llvm_config.go.in > \
-  $gollvmdir/llvm_config.go
 printf "package llvm\n\nconst Version = \"%s\"\n" "$llvm_version" > $gollvmdir/version.go

llvm/trunk/cmake/config-ix.cmake

if (LLVM_ENABLE_SPHINX)
   if (LLVM_BUILD_DOCS)
     add_custom_target(sphinx ALL)
   endif()
 else()
   message(STATUS "Sphinx disabled.")
 endif()
 
 set(LLVM_BINDINGS "")
+if(WIN32)
+  message(STATUS "Go bindings disabled.")
+else()
   find_program(GO_EXECUTABLE NAMES go DOC "go executable")
   if(GO_EXECUTABLE STREQUAL "GO_EXECUTABLE-NOTFOUND")
     message(STATUS "Go bindings disabled.")
   else()
     execute_process(COMMAND ${GO_EXECUTABLE} run ${CMAKE_SOURCE_DIR}/bindings/go/conftest.go
                     RESULT_VARIABLE GO_CONFTEST)
     if(GO_CONFTEST STREQUAL "0")
       set(LLVM_BINDINGS "${LLVM_BINDINGS} go")
       message(STATUS "Go bindings enabled.")
     else()
       message(STATUS "Go bindings disabled, need at least Go 1.2.")
     endif()
   endif()
+endif()

llvm/trunk/test/Bindings/Go/go.test

-; RUN: cd %S/../../../bindings/go/llvm && \
-; RUN: env CGO_CPPFLAGS="$(llvm-config --cppflags)" \
-; RUN:     CGO_CXXFLAGS=-std=c++11 \
-; RUN:     CGO_LDFLAGS="$(llvm-config --ldflags --libs --system-libs \
-; RUN:                                $(../build.sh --print-components)) $CGO_LDFLAGS" \
-; RUN:     %go test -tags byollvm .
+; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm
 
 ; REQUIRES: shell

llvm/trunk/test/CMakeLists.txt

set(LLVM_TEST_DEPENDS
           lli-child-target
           llvm-ar
           llvm-as
           llvm-bcanalyzer
           llvm-c-test
           llvm-cov
           llvm-diff
           llvm-dis
-          llvm-extract
           llvm-dwarfdump
+          llvm-extract
+          llvm-go
           llvm-link
           llvm-lto
           llvm-mc
           llvm-mcmarkup
           llvm-nm
           llvm-size
           llvm-objdump
           llvm-profdata

llvm/trunk/test/lit.cfg

for pattern in [r"\bbugpoint\b(?!-)",
                 r"\bllvm-as\b",
                 r"\bllvm-bcanalyzer\b",
                 r"\bllvm-config\b",
                 r"\bllvm-cov\b",
                 r"\bllvm-diff\b",
                 r"\bllvm-dis\b",
                 r"\bllvm-dwarfdump\b",
                 r"\bllvm-extract\b",
+                r"\bllvm-go\b",
                 r"\bllvm-link\b",
                 r"\bllvm-lto\b",
                 r"\bllvm-mc\b",
                 r"\bllvm-mcmarkup\b",
                 r"\bllvm-nm\b",
                 r"\bllvm-objdump\b",
                 r"\bllvm-profdata\b",
                 r"\bllvm-ranlib\b",

llvm/trunk/tools/CMakeLists.txt

 
 add_llvm_tool_subdirectory(llvm-symbolizer)
 
 add_llvm_tool_subdirectory(llvm-c-test)
 
 add_llvm_tool_subdirectory(obj2yaml)
 add_llvm_tool_subdirectory(yaml2obj)
 
+add_llvm_tool_subdirectory(llvm-go)
+
 if(NOT CYGWIN AND LLVM_ENABLE_PIC)
   add_llvm_tool_subdirectory(lto)
   add_llvm_tool_subdirectory(llvm-lto)
 else()
   ignore_llvm_tool_subdirectory(lto)
   ignore_llvm_tool_subdirectory(llvm-lto)
 endif()
 

llvm/trunk/tools/Makefile

 # On Win32, loadable modules can be built with ENABLE_SHARED.
 ifneq ($(ENABLE_SHARED),1)
   ifneq (,$(filter $(HOST_OS), Cygwin MingW))
     PARALLEL_DIRS := $(filter-out bugpoint-passes, \
                         $(PARALLEL_DIRS))
   endif
 endif
 
+ifneq (,$(filter go,$(BINDINGS_TO_BUILD)))
+  PARALLEL_DIRS += llvm-go
+endif
+
 include $(LEVEL)/Makefile.common

llvm/trunk/tools/llvm-go/CMakeLists.txt

+if(LLVM_BINDINGS MATCHES "go")
+  set(binpath ${CMAKE_BINARY_DIR}/bin/llvm-go${CMAKE_EXECUTABLE_SUFFIX})
+  add_custom_command(OUTPUT ${binpath}
+    COMMAND ${GO_EXECUTABLE} build -o ${binpath} llvm-go.go
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/llvm-go.go
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Building Go executable llvm-go")
+  add_custom_target(llvm-go ALL DEPENDS ${binpath})
+endif()

llvm/trunk/tools/llvm-go/Makefile

+##===- tools/llvm-go/Makefile ------------------------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+
+LEVEL := ../..
+include $(LEVEL)/Makefile.common
+
+all:: $(ToolDir)/llvm-go$(EXEEXT)
+
+$(ToolDir)/llvm-go$(EXEEXT): $(PROJ_SRC_DIR)/llvm-go.go
+  $(GO) build -o $@ $<

llvm/trunk/tools/llvm-go/llvm-go.go

+//===-- llvm-go.go - go tool wrapper for LLVM -----------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This tool lets us build LLVM components within the tree by setting up a
+// $GOPATH that resembles a tree fetched in the normal way with "go get".
+//
+//===----------------------------------------------------------------------===//
+
+package main
+
+import (
+  "fmt"
+  "io/ioutil"
+  "os"
+  "os/exec"
+  "path/filepath"
+  "runtime"
+  "strings"
+)
+
+type pkg struct {
+  llvmpath, pkgpath string
+}
+
+var packages = []pkg{
+  {"bindings/go/llvm", "llvm.org/llvm/bindings/go/llvm"},
+}
+
+type compilerFlags struct {
+  cpp, cxx, ld string
+}
+
+var components = []string{
+  "all-targets",
+  "analysis",
+  "asmparser",
+  "asmprinter",
+  "bitreader",
+  "bitwriter",
+  "codegen",
+  "core",
+  "debuginfo",
+  "executionengine",
+  "instrumentation",
+  "interpreter",
+  "ipo",
+  "irreader",
+  "linker",
+  "mc",
+  "mcjit",
+  "objcarcopts",
+  "option",
+  "profiledata",
+  "scalaropts",
+  "support",
+  "target",
+}
+
+func llvmConfig(args ...string) string {
+  configpath := os.Getenv("LLVM_CONFIG")
+  if configpath == "" {
+    // strip llvm-go, add llvm-config
+    configpath = os.Args[0][:len(os.Args[0])-7] + "llvm-config"
+  }
+
+  cmd := exec.Command(configpath, args...)
+  out, err := cmd.Output()
+  if err != nil {
+    panic(err.Error())
+  }
+
+  outstr := string(out)
+  outstr = strings.TrimSuffix(outstr, "\n")
+  return strings.Replace(outstr, "\n", " ", -1)
+}
+
+func llvmFlags() compilerFlags {
+  ldflags := llvmConfig(append([]string{"--ldflags", "--libs", "--system-libs"}, components...)...)
+  if runtime.GOOS != "darwin" {
+    // OS X doesn't like -rpath with cgo. See:
+    // https://code.google.com/p/go/issues/detail?id=7293
+    ldflags = "-Wl,-rpath," + llvmConfig("--libdir") + " " + ldflags
+  }
+  return compilerFlags{
+    cpp: llvmConfig("--cppflags"),
+    cxx: "-std=c++11",
+    ld:  ldflags,
+  }
+}
+
+func addTag(args []string, tag string) []string {
+  args = append([]string{}, args...)
+  addedTag := false
+  for i, a := range args {
+    if strings.HasPrefix(a, "-tags=") {
+      args[i] = a + " " + tag
+      addedTag = true
+    } else if a == "-tags" && i+1 < len(args) {
+      args[i+1] = args[i+1] + " " + tag
+      addedTag = true
+    }
+  }
+  if !addedTag {
+    args = append([]string{args[0], "-tags", tag}, args[1:]...)
+  }
+  return args
+}
+
+func printComponents() {
+  fmt.Println(strings.Join(components, " "))
+}
+
+func printConfig() {
+  flags := llvmFlags()
+
+  fmt.Printf(`// +build !byollvm
+
+// This file is generated by llvm-go, do not edit.
+
+package llvm
+
+/*
+#cgo CPPFLAGS: %s
+#cgo CXXFLAGS: %s
+#cgo LDFLAGS: %s
+*/
+import "C"
+
+type (run_build_sh int)
+`, flags.cpp, flags.cxx, flags.ld)
+}
+
+func runGoWithLLVMEnv(args []string, cc, cxx, cppflags, cxxflags, ldflags string) {
+  args = addTag(args, "byollvm")
+
+  srcdir := llvmConfig("--src-root")
+
+  tmpgopath, err := ioutil.TempDir("", "gopath")
+  if err != nil {
+    panic(err.Error())
+  }
+
+  for _, p := range packages {
+    path := filepath.Join(tmpgopath, "src", p.pkgpath)
+    err := os.MkdirAll(filepath.Dir(path), os.ModePerm)
+    if err != nil {
+      panic(err.Error())
+    }
+
+    err = os.Symlink(filepath.Join(srcdir, p.llvmpath), path)
+    if err != nil {
+      panic(err.Error())
+    }
+  }
+
+  newgopathlist := []string{tmpgopath}
+  newgopathlist = append(newgopathlist, filepath.SplitList(os.Getenv("GOPATH"))...)
+  newgopath := strings.Join(newgopathlist, string(filepath.ListSeparator))
+
+  flags := llvmFlags()
+
+  newenv := []string{
+    "CC=" + cc,
+    "CXX=" + cxx,
+    "CGO_CPPFLAGS=" + flags.cpp + " " + cppflags,
+    "CGO_CXXFLAGS=" + flags.cxx + " " + cxxflags,
+    "CGO_LDFLAGS=" + flags.ld + " " + ldflags,
+    "GOPATH=" + newgopath,
+  }
+  for _, v := range os.Environ() {
+    if !strings.HasPrefix(v, "CC=") &&
+      !strings.HasPrefix(v, "CXX=") &&
+      !strings.HasPrefix(v, "CGO_CPPFLAGS=") &&
+      !strings.HasPrefix(v, "CGO_CXXFLAGS=") &&
+      !strings.HasPrefix(v, "CGO_LDFLAGS=") &&
+      !strings.HasPrefix(v, "GOPATH=") {
+      newenv = append(newenv, v)
+    }
+  }
+
+  gocmdpath, err := exec.LookPath("go")
+  if err != nil {
+    panic(err.Error())
+  }
+
+  proc, err := os.StartProcess(gocmdpath, append([]string{"go"}, args...),
+    &os.ProcAttr{
+      Env:   newenv,
+      Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
+    })
+  if err != nil {
+    panic(err.Error())
+  }
+  ps, err := proc.Wait()
+  if err != nil {
+    panic(err.Error())
+  }
+
+  os.RemoveAll(tmpgopath)
+
+  if !ps.Success() {
+    os.Exit(1)
+  }
+}
+
+func usage() {
+  fmt.Println(`Usage: llvm-go subcommand [flags]
+
+Available subcommands: build get install run test print-components print-config`)
+  os.Exit(0)
+}
+
+func main() {
+  cc := os.Getenv("CC")
+  cxx := os.Getenv("CXX")
+  cppflags := os.Getenv("CGO_CPPFLAGS")
+  cxxflags := os.Getenv("CGO_CXXFLAGS")
+  ldflags := os.Getenv("CGO_LDFLAGS")
+
+  args := os.Args[1:]
+  DONE: for {
+    switch {
+    case len(args) == 0:
+      usage()
+    case strings.HasPrefix(args[0], "cc="):
+      cc = args[0][3:]
+      args = args[1:]
+    case strings.HasPrefix(args[0], "cxx="):
+      cxx = args[0][4:]
+      args = args[1:]
+    case strings.HasPrefix(args[0], "cppflags="):
+      cppflags = args[0][9:]
+      args = args[1:]
+    case strings.HasPrefix(args[0], "cxxflags="):
+      cxxflags = args[0][9:]
+      args = args[1:]
+    case strings.HasPrefix(args[0], "ldflags="):
+      ldflags = args[0][8:]
+      args = args[1:]
+    default:
+      break DONE
+    }
+  }
+
+  switch args[0] {
+  case "build", "get", "install", "run", "test":
+    runGoWithLLVMEnv(args, cc, cxx, cppflags, cxxflags, ldflags)
+  case "print-components":
+    printComponents()
+  case "print-config":
+    printConfig()
+  default:
+    usage()
+  }
+}