diff --git a/.pylintrc b/.pylintrc new file mode 100644 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,7 @@ +# add root folder to Pylint PYTHONPATH, so builders can be checked with zorg.* +# packages +[MASTER] +init-hook='import sys; sys.path.append(".")' + +[FORMAT] +max-line-length=120 \ No newline at end of file diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -231,6 +231,9 @@ 'workernames' : ["linaro-armv8-windows-msvc-01", "linaro-armv8-windows-msvc-02"], 'builddir': "clang-arm64-windows-msvc", 'factory' : ClangBuilder.getClangCMakeBuildFactory( + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, vs="manual", test=False, # Disable testing until MCJIT failures are fixed extra_cmake_args=[ @@ -271,6 +274,9 @@ 'builddir': "clang-cmake-armv7-lnt", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, checkout_lld=False, test=False, @@ -288,6 +294,9 @@ 'builddir':"clang-cmake-armv7-selfhost-neon", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, checkout_lld=False, useTwoStage=True, @@ -304,6 +313,9 @@ 'builddir':"clang-cmake-armv7-quick", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, checkout_lld=False, extra_cmake_args=["-DLLVM_TARGETS_TO_BUILD='ARM;AArch64'"])}, @@ -315,6 +327,9 @@ 'builddir':"clang-cmake-armv7-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, checkout_lld=False, test=True, @@ -332,6 +347,9 @@ 'builddir': "clang-cmake-armv7-selfhost", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, checkout_lld=False, useTwoStage=True, @@ -348,6 +366,9 @@ 'builddir': "clang-cmake-aarch64-quick", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, checkout_lld=False, test=True, @@ -365,6 +386,9 @@ 'builddir':"clang-cmake-aarch64-lld", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=True, checkout_lld=True, test=True, @@ -386,6 +410,9 @@ 'builddir': "clang-cmake-aarch64-global-isel", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, checkout_lld=False, test=True, @@ -403,6 +430,9 @@ 'builddir': "clang-cmake-armv7-full", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, checkout_compiler_rt=True, testStage1=False, @@ -423,6 +453,9 @@ 'builddir': "clang-cmake-thumbv7-full-sh", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, checkout_compiler_rt=True, testStage1=False, @@ -446,6 +479,9 @@ 'builddir': "clang-cmake-armv8-lld", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=True, checkout_lld=True, test=True, @@ -468,6 +504,9 @@ 'builddir': "clang-cmake-aarch64-full", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=True, checkout_flang=True, checkout_lld=False, @@ -489,6 +528,9 @@ 'builddir': "clang-arm64-windows-msvc-2stage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( vs="manual", + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, test=False, # Disable testing until MCJIT failures are fixed useTwoStage=True, testStage1=False, @@ -513,6 +555,9 @@ 'builddir': "clang-ppc64be-lnt", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=False, runTestSuite=True, @@ -526,6 +571,9 @@ 'builddir': "clang-ppc64be-multistage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=True, stage1_config='Release', @@ -538,6 +586,9 @@ 'builddir': "clang-ppc64le-lnt", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=False, runTestSuite=True, @@ -551,6 +602,9 @@ 'builddir': "clang-ppc64le-multistage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=True, stage1_config='Release', @@ -563,6 +617,9 @@ 'builddir': "clang-ppc64be", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=False, stage1_config='Release', @@ -574,6 +631,9 @@ 'builddir': "clang-ppc64le", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=False, stage1_config='Release', @@ -584,6 +644,9 @@ 'workernames' : ["ppc64le-clang-rhel-test"], 'builddir': "clang-ppc64le-rhel", 'factory' : ClangBuilder.getClangCMakeBuildFactory(clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_clang_tools_extra=True, checkout_compiler_rt=False, checkout_lld=True, @@ -610,6 +673,9 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=4, clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=False, stage1_config='Release', @@ -624,6 +690,9 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=4, clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=True, stage1_config='Release', @@ -637,6 +706,9 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=4, clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=False, runTestSuite=True, @@ -650,6 +722,9 @@ 'builddir': "clang-sparc64-linux-multistage", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, useTwoStage=True, stage1_config='Release', @@ -663,6 +738,9 @@ 'workernames' : ["hexagon-build-02", "hexagon-build-03"], 'builddir': "clang-hexagon-elf", 'factory' : ClangBuilder.getClangCMakeBuildFactory( + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, jobs=16, checkout_clang_tools_extra=False, checkout_compiler_rt=False, @@ -694,6 +772,9 @@ 'builddir': "clang-cmake-x86_64-avx2-linux", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, @@ -715,6 +796,9 @@ 'builddir': "clang-cmake-x86_64-avx2-linux-perf", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, @@ -738,6 +822,9 @@ 'builddir': "clang-cmake-x86_64-sde-avx512-linux", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_lld=False, @@ -762,6 +849,9 @@ 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( stage1_config='Release', clean=True, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, test=False, useTwoStage=False, @@ -787,6 +877,9 @@ 'factory' : ClangBuilder.getClangCMakeGCSBuildFactory( stage1_config='Release', clean=True, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_compiler_rt=False, test=False, useTwoStage=False, @@ -810,6 +903,9 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=8, clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, extra_cmake_args=[ '-DLLVM_ENABLE_ASSERTIONS=ON', @@ -824,6 +920,9 @@ 'builddir': "clang-x64-ninja-win7", 'factory' : ClangBuilder.getClangCMakeBuildFactory( clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, vs="autodetect", vs_target_arch='x64', @@ -1762,6 +1861,9 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=8, clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=X86', @@ -1775,6 +1877,9 @@ 'factory' : ClangBuilder.getClangCMakeBuildFactory( jobs=8, clean=False, + # Legacy implementation using `rm -rf` instead of integrated + # build step "RemoveDirectory". + clean_with_rm=True, checkout_lld=False, extra_cmake_args=['-DLLVM_ENABLE_ASSERTIONS=ON', '-DLLVM_TARGETS_TO_BUILD=Sparc', diff --git a/zorg/buildbot/builders/ClangBuilder.py b/zorg/buildbot/builders/ClangBuilder.py --- a/zorg/buildbot/builders/ClangBuilder.py +++ b/zorg/buildbot/builders/ClangBuilder.py @@ -4,6 +4,7 @@ from buildbot.process.properties import WithProperties, Property from buildbot.steps.shell import ShellCommand, SetProperty from buildbot.steps.shell import WarningCountingShellCommand +from buildbot.steps.worker import RemoveDirectory import zorg.buildbot.builders.Util as builders_util @@ -71,6 +72,9 @@ def getClangCMakeGCSBuildFactory( clean=True, + # Legacy implementation using `rm -rf` instead of integrated build + # step "RemoveDirectory". + clean_with_rm=False, test=True, cmake='cmake', jobs=None, @@ -128,10 +132,14 @@ stage1_upload_directory=stage1_upload_directory, xz_compression_factor=xz_compression_factor, use_pixz_compression=use_pixz_compression, - trigger_after_stage1=trigger_after_stage1) + trigger_after_stage1=trigger_after_stage1, + clean_with_rm=clean_with_rm) def getClangCMakeBuildFactory( clean=True, + # Legacy implementation using `rm -rf` instead of integrated build + # step "RemoveDirectory". + clean_with_rm=False, test=True, cmake='cmake', jobs=None, @@ -178,10 +186,12 @@ checkout_compiler_rt=checkout_compiler_rt, checkout_libcxx=checkout_libcxx, checkout_flang=checkout_flang, - checkout_test_suite=checkout_test_suite) + checkout_test_suite=checkout_test_suite, + clean_with_rm=clean_with_rm) def _getClangCMakeBuildFactory( clean=True, + clean_with_rm=False, test=True, cmake='cmake', jobs=None, @@ -309,15 +319,33 @@ ############# CLEANING - f.addStep(ShellCommand(name='clean stage 1', - command=['rm','-rf',stage1_build], - warnOnFailure=True, - haltOnFailure=False, - flunkOnFailure=False, - description='cleaning stage 1', - descriptionDone='clean', - workdir='.', - doStepIf=clean_build_requested)) + if clean_with_rm: + # Legacy implementation using `rm -rf` instead of integrated build step + # "RemoveDirectory". + # once all buliders are using this, it can be removed. + # Using the Unix "rm" command on Windows might cause issues with + # race conditions: https://youtu.be/uhRWMGBjlO8?t=455 + f.addStep(ShellCommand(name='clean stage 1', + command=['rm','-rf',stage1_build], + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + description='cleaning stage 1', + descriptionDone='clean', + workdir='.', + doStepIf=clean_build_requested)) + else: + # Use the buildbot build-in step "RemoveDirectory". + # http://docs.buildbot.net/current/manual/configuration/buildsteps.html#removedirectory + f.addStep(RemoveDirectory(name='clean stage 1', + dir=stage1_build, + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + description='cleaning stage 1', + descriptionDone='clean', + workdir='.', + doStepIf=clean_build_requested)) ############# STAGE 1 @@ -357,14 +385,25 @@ env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: - f.addStep(ShellCommand(name='clean stage 1 install', - command=['rm','-rf',stage1_install], - warnOnFailure=True, - haltOnFailure=False, - flunkOnFailure=False, - description='cleaning stage 1 install', - descriptionDone='clean', - workdir='.')) + if clean_with_rm: + # see comment in CLEAN step + f.addStep(ShellCommand(name='clean stage 1 install', + command=['rm','-rf',stage1_install], + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + description='cleaning stage 1 install', + descriptionDone='clean', + workdir='.')) + else: + f.addStep(RemoveDirectory(name='clean stage 1 install', + dir=stage1_install, + warnOnFailure=True, + haltOnFailure=False, + flunkOnFailure=False, + description='cleaning stage 1 install', + descriptionDone='clean', + workdir='.')) f.addStep(ShellCommand(name='install stage 1', command=ninja_install_cmd, description='ninja install', @@ -393,12 +432,21 @@ # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we # should not build in the first place. - f.addStep(ShellCommand(name='clean stage 2', - command=['rm','-rf',stage2_build], - warnOnFailure=True, - description='cleaning stage 2', - descriptionDone='clean', - workdir='.')) + if clean_with_rm: + f.addStep(ShellCommand(name='clean stage 2', + command=['rm','-rf',stage2_build], + warnOnFailure=True, + description='cleaning stage 2', + descriptionDone='clean', + workdir='.')) + else: + f.addStep(RemoveDirectory(name='clean stage 2', + dir=stage2_build, + warnOnFailure=True, + description='cleaning stage 2', + descriptionDone='clean', + workdir='.')) + # Absolute paths to just built compilers. # Note: Backslash path separators do not work well with cmake and ninja. @@ -450,12 +498,21 @@ compiler_path = stage1_install if useTwoStage: compiler_path=stage2_install - f.addStep(ShellCommand(name='clean stage 2 install', - command=['rm','-rf',stage2_install], - warnOnFailure=True, - description='cleaning stage 2 install', - descriptionDone='clean', - workdir='.')) + if clean_with_rm: + f.addStep(ShellCommand(name='clean stage 2 install', + command=['rm','-rf',stage2_install], + warnOnFailure=True, + description='cleaning stage 2 install', + descriptionDone='clean', + workdir='.')) + else: + f.addStep(RemoveDirectory(name='clean stage 2 install', + dir=stage2_install, + warnOnFailure=True, + description='cleaning stage 2 install', + descriptionDone='clean', + workdir='.')) + f.addStep(ShellCommand(name='install stage 2', command=ninja_install_cmd, description='ninja install 2', @@ -514,12 +571,21 @@ test_suite_env['CXX'] = cxx # Steps to prepare, build and run LNT - f.addStep(ShellCommand(name='clean sandbox', - command=['rm', '-rf', 'sandbox'], - haltOnFailure=True, - description='removing sandbox directory', - workdir='test', - env=env)) + if clean_with_rm: + f.addStep(ShellCommand(name='clean sandbox', + command=['rm', '-rf', 'sandbox'], + haltOnFailure=True, + description='removing sandbox directory', + workdir='test', + env=env)) + else: + f.addStep(RemoveDirectory(name='clean sandbox', + dir='sandbox', + haltOnFailure=True, + description='removing sandbox directory', + workdir='test', + env=env)) + f.addStep(ShellCommand(name='recreate sandbox', command=['virtualenv', 'sandbox'], haltOnFailure=True,