test-release.sh: Update to fetch source from GitHub

[tstellar]

Summary

This also changes the test-release.sh script to build using the monorepo layout instead of copying sub-projects into llvm/tools or llvm/projects.

[hans]

Thanks for digging into this!

llvm/utils/release/test-release.sh

293

I guess this won't work with -svn-path, which I like to use to test a branch before tagging it.

It would be great if we could rename that to -git-ref or something and have it work the same way.

299

Could we just pipe it to tar directly? Then there would be no need to come up with a name for the file or to keep it around.

302

Maybe add a comment here for those not familiar with the GitHub API?

351

Maybe add to the change description that this changes the build to use the monorpo layout. (It makes sense now that we're getting the source from GitHub, but I think it's worth calling it out explicitly.)
Also this is the post I'm most curious about. Hopefully it doesn't cause problems :-)

[tstellar]

Address some review comments.

[tstellar]

llvm/utils/release/test-release.sh

293

I've added an equivalent option called -git-ref. The only difference is if you pass it a raw sha, then it won't be able to checkout the test-suite, since it is in a different repository.

[hans]

Looks great to me. Thanks!

llvm/utils/release/test-release.sh

303

Nit: :: -> :, and achive -> archive.

llvm/utils/release/test-release.sh

 if [ -z "$NumJobs" ]; then
     NumJobs=`grep -c processor /proc/cpuinfo 2> /dev/null || true`
 fi
 if [ -z "$NumJobs" ]; then
     NumJobs=3
 fi
 
 # Projects list
-projects="llvm cfe clang-tools-extra"
+projects="llvm clang clang-tools-extra"
 if [ $do_rt = "yes" ]; then
   projects="$projects compiler-rt"
 fi
 if [ $do_libs = "yes" ]; then
   projects="$projects libcxx"
   if [ $do_libcxxabi = "yes" ]; then
     projects="$projects libcxxabi"
   fi

 
 if [ "$System" != "Darwin" ]; then
   check_program_exists 'file'
   check_program_exists 'objdump'
 fi
 
 check_program_exists ${MAKE}
 
-# Make sure that the URLs are valid.
-function check_valid_urls() {
-    for proj in $projects ; do
-        echo "# Validating $proj SVN URL"
-
-        if ! svn ls $Base_url/$proj/$ExportBranch > /dev/null 2>&1 ; then
-            echo "$proj does not have a $ExportBranch branch/tag!"
-            exit 1
-        fi
-    done
-}
-
 # Export sources to the build directory.
 function export_sources() {
-    check_valid_urls
-
-    for proj in $projects ; do
-        case $proj in
-        llvm)
-            projsrc=$proj.src
-            ;;
-        cfe)
-            projsrc=llvm.src/tools/clang
-            ;;
-        lld|lldb|polly)
-            projsrc=llvm.src/tools/$proj
-            ;;
-        clang-tools-extra)
-            projsrc=llvm.src/tools/clang/tools/extra
-            ;;
-        compiler-rt|libcxx|libcxxabi|libunwind|openmp)
-            projsrc=llvm.src/projects/$proj
-            ;;
-        test-suite)
-            projsrc=$proj.src
-            ;;
-        *)
-            echo "error: unknown project $proj"
-            exit 1
-            ;;
-        esac
-
-        if [ -d $projsrc ]; then
-          echo "# Reusing $proj $Release-$RC sources in $projsrc"
-          continue
-        fi
-        echo "# Exporting $proj $Release-$RC sources to $projsrc"
-        if ! svn export -q $Base_url/$proj/$ExportBranch $projsrc ; then
-            echo "error: failed to export $proj project"
-            exit 1
-        fi
-    done
+  tag="llvmorg-$Release"
+  if [ "$RC" != "final" ]; then
+    tag="$tag-$RC"
+  fi
+
+  SrcDir=$BuildDir/llvm-project
+  TarballName="llvm-project.tar.gz"
+  mkdir -p $SrcDir
+  echo "# Using git tag: $tag"
+  curl -L -O https://github.com/llvm/llvm-project/archive/$tag/$TarballName
+  tar -C $SrcDir --strip-components=1 -xzf $TarballName
+
+  if [ "$do_test_suite" = "yes" ]; then
+    TestSuiteSrcDir=$BuildDir/llvm-test-suite
+    TestSuiteTarballName="llvm-test-suite.tar.gz"
+    mkdir -p $TestSuiteSrcDir
+    curl -L -O https://github.com/llvm/test-suite/archive/$tag/$TestSuiteTarballName
+    tar -C $TestSuiteSrcDir --strip-components=1 -xzf $TestSuiteTarballName
 
-    cd $BuildDir
 }
 
 function configure_llvmCore() {
     Phase="$1"
     Flavor="$2"
     ObjDir="$3"
 
     case $Flavor in
case $Flavor in
             ;;
         * )
             echo "# Invalid flavor '$Flavor'"
             echo ""
             return
             ;;
     esac
 
+    project_list=${projects// /;}
     echo "# Using C compiler: $c_compiler"
     echo "# Using C++ compiler: $cxx_compiler"
 
     cd $ObjDir
     echo "# Configuring llvm $Release-$RC $Flavor"
 
     echo "#" env CC="$c_compiler" CXX="$cxx_compiler" \
         cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
-        $ExtraConfigureFlags $BuildDir/llvm.src \
+        -DLLVM_ENABLE_PROJECTS="$project_list" \
+        $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
     env CC="$c_compiler" CXX="$cxx_compiler" \
         cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
-        $ExtraConfigureFlags $BuildDir/llvm.src \
+        -DLLVM_ENABLE_PROJECTS="$project_list" \
+        $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
 
     cd $BuildDir
 }
 
 function build_llvmCore() {
     Phase="$1"
     Flavor="$2"

 fi
 
 # Setup the test-suite.  Do this early so we can catch failures before
 # we do the full 3 stage build.
 if [ $do_test_suite = "yes" ]; then
   SandboxDir="$BuildDir/sandbox"
   Lit=$SandboxDir/bin/lit
   TestSuiteBuildDir="$BuildDir/test-suite-build"
-  TestSuiteSrcDir="$BuildDir/test-suite.src"
+  TestSuiteSrcDir="$BuildDir/llvm-test-suite"
 
   virtualenv $SandboxDir
-  $SandboxDir/bin/python $BuildDir/llvm.src/utils/lit/setup.py install
+  $SandboxDir/bin/python $BuildDir/llvm-project/llvm/utils/lit/setup.py install
   mkdir -p $TestSuiteBuildDir
 fi
 
 (
 Flavors="Release"
 if [ "$do_debug" = "yes" ]; then
     Flavors="Debug $Flavors"
 fi