Create scheduler class LLDBTriggerable for LLDB Android builders

[chying]

Summary

• subclass of buildbot.schedulers.triggerable.Triggerable
• override trigger function to get changelist based on changes since their last build
• this solves the problem that changelist from triggerer builder be inaccurate under some circumstances, like previous build failed and didn't trigger, those changelist on the failed build won't be pass through

[chying]

A few improvements for class design

• Pass project list and max change count by argument at class initialization for flexibility
• Remove newRevision from class member, define as function variable

[chying]

Remove unused imports

buildbot/osuosl/master/master.cfg

c['schedulers'].append(SingleBranchScheduler(name="lld_scheduler",
                             change_filter=depends_on([
                                 "llvm",
                                 "lld"])))
 
 # lldb android builders will be triggered by completion of buildserver instead of by source change
 lldb_android_builders=["lldb-x86_64-ubuntu-14.04-android",
                        "lldb-windows7-android",
                        "lldb-x86_64-darwin-13.4"]
-c['schedulers'].append((triggerable.Triggerable(name="lldb_android_scheduler",
-                                                builderNames=lldb_android_builders)))
+from zorg.buildbot.schedulers.LLDBTriggerable import LLDBTriggerable
+c['schedulers'].append((LLDBTriggerable(name="lldb_android_scheduler",
 
 c['schedulers'].append(SingleBranchScheduler(name="lldb_scheduler",
                             treeStableTimer=2*60,
                             builderNames=list(set(get_all_for("lldb"))-set(lldb_android_builders)),
                             change_filter=depends_on([
                                 "llvm",
                                 "cfe",
                                 "lldb"])))

zorg/buildbot/schedulers/LLDBTriggerable.py

+import os
+from twisted.python import log
+from twisted.internet import defer, utils
+from buildbot.schedulers.triggerable import Triggerable
+from buildbot.process.properties import Properties
+
+class LLDBTriggerable(Triggerable):
+    """
+    This is the scheduler used for lldb android builders,
+    Overwrite trigger function, so the triggered builder will
+    get changelist based on new changes since their last build.
+    The origianl implementation takes changelist from upstream
+    triggerer builder, this will be inaccurate in the case that
+    some builds in upstream didn't trigger new builds due to
+    failure in their early steps.
+    """
+
+    def __init__(self, **kwargs):
+        Triggerable.__init__(self, **kwargs)
+        self.environ = os.environ.copy()
+        self.projects = ['lldb', 'cfe', 'llvm']
+        self.lastRevision = None
+        self.newRevision = None
+
+    def trigger(self, ssid, set_props=None):
+        """Trigger this scheduler with the given sourcestamp ID. Returns a
+        deferred that will fire when the buildset is finished."""
+        # properties for this buildset are composed of our own properties,
+        # potentially overridden by anything from the triggering build
+        props = Properties()
+        props.updateFromProperties(self.properties)
+        if set_props:
+            props.updateFromProperties(set_props)
+
+        def getRevision(ss):
+            self.newRevision = ss['revision']
+            return ss['revision']
+
+        def getRecentChanges(newRevision):
+            if self.lastRevision is None:
+                return None
+            return self.master.db.changes.getRecentChanges(100)
+
+        # check the last 100 changeset and pick up the ones that are between
+        # last revision and current revision and belong to interested projects
+        def selectChangeSet(changes):
+            changeids = []
+            if changes is not None:
+                for change in changes:
+                    if change['revision'] > self.newRevision or change['revision'] <= self.lastRevision or change['project'] not in self.projects:
+                        continue
+                    changeids.append(change['changeid'])
+            log.msg("LLDBTriggerable: last revision change from %s to %s" % (self.lastRevision, self.newRevision))
+            self.lastRevision = self.newRevision
+            return changeids
+
+        def addBuildset(changeids):
+            if changeids:
+                log.msg("LLDBTriggerable: addBuildsetForChanges, changeids: %s" % changeids)
+                return self.addBuildsetForChanges(reason=self.reason, changeids=changeids, properties=props)
+            elif ssid:
+                # if this is the first build after master startup, use the source stamp from triggerer build
+                # it's possible to write last revision to a file on master, so after master reconfig we could
+                # pick up the correct last revision.
+                # It's not implemented here because 1) the cases are rare that first build after master restart
+                # is preceded by failing builds on triggerer builder, 2) avoid polluting master with project
+                # specific cache files
+                log.msg("LLDBTriggerable: addBuildsetForSourceStamp")
+                return self.addBuildsetForSourceStamp(reason=self.reason, ssid=ssid, properties=props)
+            else:
+                return self.addBuildsetForLatest(reason=self.reason, properties=props)
+
+        def setup_waiter((bsid,brids)):
+            self._waiters[bsid] = d = defer.Deferred()
+            self._updateWaiters()
+            return d
+
+        d = self.master.db.sourcestamps.getSourceStamp(ssid)
+        d.addCallback(getRevision)
+        d.addCallback(getRecentChanges)
+        d.addCallback(selectChangeSet)
+        d.addCallback(addBuildset)
+        d.addCallback(setup_waiter)
+        return d
+