test-release.sh: Update to fetch source from GitHub

[tstellar]

Summary

This also changes the test-release.sh script to build using the monorepo layout instead of copying sub-projects into llvm/tools or llvm/projects.

[hans]

Thanks for digging into this!

llvm/utils/release/test-release.sh

304–305

I guess this won't work with -svn-path, which I like to use to test a branch before tagging it.

It would be great if we could rename that to -git-ref or something and have it work the same way.

306

Could we just pipe it to tar directly? Then there would be no need to come up with a name for the file or to keep it around.

309

Maybe add a comment here for those not familiar with the GitHub API?

371–372

Maybe add to the change description that this changes the build to use the monorpo layout. (It makes sense now that we're getting the source from GitHub, but I think it's worth calling it out explicitly.)
Also this is the post I'm most curious about. Hopefully it doesn't cause problems :-)

[tstellar]

Address some review comments.

[tstellar]

llvm/utils/release/test-release.sh

304–305

I've added an equivalent option called -git-ref. The only difference is if you pass it a raw sha, then it won't be able to checkout the test-suite, since it is in a different repository.

[hans]

Looks great to me. Thanks!

llvm/utils/release/test-release.sh

310

Nit: :: -> :, and achive -> archive.

llvm/utils/release/test-release.sh

 do_test_suite="yes"
 do_openmp="yes"
 do_lld="yes"
 do_lldb="no"
 do_polly="yes"
 BuildDir="`pwd`"
 ExtraConfigureFlags=""
 ExportBranch=""
+git_ref=""
 
 function usage() {
     echo "usage: `basename $0` -release X.Y.Z -rc NUM [OPTIONS]"
     echo ""
     echo " -release X.Y.Z       The release version to test."
     echo " -rc NUM              The pre-release candidate number."
     echo " -final               The final release candidate."
     echo " -triple TRIPLE       The target triple for this machine."
     echo " -j NUM               Number of compile jobs to run. [default: 3]"
     echo " -build-dir DIR       Directory to perform testing in. [default: pwd]"
     echo " -no-checkout         Don't checkout the sources from SVN."
     echo " -test-debug          Test the debug build. [default: no]"
     echo " -test-asserts        Test with asserts on. [default: no]"
     echo " -no-compare-files    Don't test that phase 2 and 3 files are identical."
     echo " -use-gzip            Use gzip instead of xz."
     echo " -use-ninja           Use ninja instead of make/gmake."
     echo " -configure-flags FLAGS  Extra flags to pass to the configure step."
-    echo " -svn-path DIR        Use the specified DIR instead of a release."
-    echo "                      For example -svn-path trunk or -svn-path branches/release_37"
+    echo " -git-ref sha         Use the specified git ref for testing instead of a release."
     echo " -no-rt               Disable check-out & build Compiler-RT"
     echo " -no-libs             Disable check-out & build libcxx/libcxxabi/libunwind"
     echo " -no-libcxxabi        Disable check-out & build libcxxabi"
     echo " -no-libunwind        Disable check-out & build libunwind"
     echo " -no-test-suite       Disable check-out & build test-suite"
     echo " -no-openmp           Disable check-out & build libomp"
     echo " -no-lld              Disable check-out & build lld"
     echo " -lldb                Enable check-out & build lldb"
case $1 in
             ;;
         -rc | --rc | -RC | --RC )
             shift
             RC="rc$1"
             ;;
         -final | --final )
             RC=final
             ;;
-        -svn-path | --svn-path )
+        -git-ref | --git-ref )
             shift
             Release="test"
             Release_no_dot="test"
             ExportBranch="$1"
             RC="`echo $ExportBranch | sed -e 's,/,_,g'`"
-            echo "WARNING: Using the branch $ExportBranch instead of a release tag"
+            git_ref="$1"
+            echo "WARNING: Using the ref $git_ref instead of a release tag"
             echo "         This is intended to aid new packagers in trialing "
             echo "         builds without requiring a tag to be created first"
             ;;
         -triple | --triple )
             shift
             Triple="$1"
             ;;
         -configure-flags | --configure-flags )

 if [ -z "$ExportBranch" ]; then
     ExportBranch="tags/RELEASE_$Release_no_dot/$RC"
 fi
 if [ -z "$Triple" ]; then
     echo "error: no target triple specified"
     exit 1
 fi
 
+if [ "$Release" != "test" ]; then
+  if [ -n "$git_ref" ]; then
+    echo "error: can't specify both -release and -git-ref"
+    exit 1
+  fi
+  git_ref=llvmorg-$Release
+  if [ "$RC" != "final" ]; then
+    git_ref="$git_ref-$RC"
+  fi
+fi
+
 # Figure out how many make processes to run.
 if [ -z "$NumJobs" ]; then
     NumJobs=`sysctl -n hw.activecpu 2> /dev/null || true`
 fi
 if [ -z "$NumJobs" ]; then
     NumJobs=`sysctl -n hw.ncpu 2> /dev/null || true`
 fi
 if [ -z "$NumJobs" ]; then
     NumJobs=`grep -c processor /proc/cpuinfo 2> /dev/null || true`
 fi
 if [ -z "$NumJobs" ]; then
     NumJobs=3
 fi
 
 # Projects list
-projects="llvm cfe clang-tools-extra"
+projects="llvm clang clang-tools-extra"
 if [ $do_rt = "yes" ]; then
   projects="$projects compiler-rt"
 fi
 if [ $do_libs = "yes" ]; then
   projects="$projects libcxx"
   if [ $do_libcxxabi = "yes" ]; then
     projects="$projects libcxxabi"
   fi

 
 if [ "$System" != "Darwin" ]; then
   check_program_exists 'file'
   check_program_exists 'objdump'
 fi
 
 check_program_exists ${MAKE}
 
-# Make sure that the URLs are valid.
-function check_valid_urls() {
-    for proj in $projects ; do
-        echo "# Validating $proj SVN URL"
-
-        if ! svn ls $Base_url/$proj/$ExportBranch > /dev/null 2>&1 ; then
-            echo "$proj does not have a $ExportBranch branch/tag!"
-            exit 1
-        fi
-    done
-}
-
 # Export sources to the build directory.
 function export_sources() {
-    check_valid_urls
-
-    for proj in $projects ; do
-        case $proj in
-        llvm)
-            projsrc=$proj.src
-            ;;
-        cfe)
-            projsrc=llvm.src/tools/clang
-            ;;
-        lld|lldb|polly)
-            projsrc=llvm.src/tools/$proj
-            ;;
-        clang-tools-extra)
-            projsrc=llvm.src/tools/clang/tools/extra
-            ;;
-        compiler-rt|libcxx|libcxxabi|libunwind|openmp)
-            projsrc=llvm.src/projects/$proj
-            ;;
-        test-suite)
-            projsrc=$proj.src
-            ;;
-        *)
-            echo "error: unknown project $proj"
-            exit 1
-            ;;
-        esac
-
-        if [ -d $projsrc ]; then
-          echo "# Reusing $proj $Release-$RC sources in $projsrc"
-          continue
-        fi
-        echo "# Exporting $proj $Release-$RC sources to $projsrc"
-        if ! svn export -q $Base_url/$proj/$ExportBranch $projsrc ; then
-            echo "error: failed to export $proj project"
-            exit 1
-        fi
-    done
+  SrcDir=$BuildDir/llvm-project
+  mkdir -p $SrcDir
+  echo "# Using git ref: $git_ref"
+
+  # GitHub allows you to download a tarball of any commit using the URL:
+  # https:://github.com/$organization/$repo/achive/$ref.tar.gz
+  curl -L https://github.com/llvm/llvm-project/archive/$git_ref.tar.gz | \
+    tar -C $SrcDir --strip-components=1 -xzf -
+
+  if [ "$do_test_suite" = "yes" ]; then
+    TestSuiteSrcDir=$BuildDir/llvm-test-suite
+    mkdir -p $TestSuiteSrcDir
+
+    # We can only use named refs, like branches and tags, that exist in
+    # both the llvm-project and test-suite repos if we want to run the
+    # test suite.
+    # If the test-suite fails to download assume we are using a ref that
+    # doesn't exist in the test suite and disable it.
+    set +e
+    curl -L https://github.com/llvm/test-suite/archive/$git_ref.tar.gz | \
+      tar -C $TestSuiteSrcDir --strip-components=1 -xzf -
+    if [ $? -ne -0 ]; then
+      echo "$git_ref not found in test-suite repo, test-suite disabled."
+      do_test_suite="no"
+    fi
+    set -e
 
-    cd $BuildDir
 }
 
 function configure_llvmCore() {
     Phase="$1"
     Flavor="$2"
     ObjDir="$3"
 
     case $Flavor in
case $Flavor in
             ;;
         * )
             echo "# Invalid flavor '$Flavor'"
             echo ""
             return
             ;;
     esac
 
+    project_list=${projects// /;}
     echo "# Using C compiler: $c_compiler"
     echo "# Using C++ compiler: $cxx_compiler"
 
     cd $ObjDir
     echo "# Configuring llvm $Release-$RC $Flavor"
 
     echo "#" env CC="$c_compiler" CXX="$cxx_compiler" \
         cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
-        $ExtraConfigureFlags $BuildDir/llvm.src \
+        -DLLVM_ENABLE_PROJECTS="$project_list" \
+        $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
     env CC="$c_compiler" CXX="$cxx_compiler" \
         cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
-        $ExtraConfigureFlags $BuildDir/llvm.src \
+        -DLLVM_ENABLE_PROJECTS="$project_list" \
+        $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
 
     cd $BuildDir
 }
 
 function build_llvmCore() {
     Phase="$1"
     Flavor="$2"

 fi
 
 # Setup the test-suite.  Do this early so we can catch failures before
 # we do the full 3 stage build.
 if [ $do_test_suite = "yes" ]; then
   SandboxDir="$BuildDir/sandbox"
   Lit=$SandboxDir/bin/lit
   TestSuiteBuildDir="$BuildDir/test-suite-build"
-  TestSuiteSrcDir="$BuildDir/test-suite.src"
+  TestSuiteSrcDir="$BuildDir/llvm-test-suite"
 
   virtualenv $SandboxDir
-  $SandboxDir/bin/python $BuildDir/llvm.src/utils/lit/setup.py install
+  $SandboxDir/bin/python $BuildDir/llvm-project/llvm/utils/lit/setup.py install
   mkdir -p $TestSuiteBuildDir
 fi
 
 (
 Flavors="Release"
 if [ "$do_debug" = "yes" ]; then
     Flavors="Debug $Flavors"
 fi