| # Copyright (c) 2009 Google Inc. All rights reserved. |
| # Copyright (c) 2009 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import traceback |
| import os |
| |
| from datetime import datetime |
| from optparse import make_option |
| from StringIO import StringIO |
| |
| from webkitpy.common.net.bugzilla import CommitterValidator |
| from webkitpy.common.net.statusserver import StatusServer |
| from webkitpy.common.system.executive import ScriptError |
| from webkitpy.common.system.deprecated_logging import error, log |
| from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler |
| from webkitpy.tool.bot.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate |
| from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate |
| from webkitpy.tool.grammar import pluralize |
| from webkitpy.tool.multicommandtool import Command, TryAgain |
| |
| class AbstractQueue(Command, QueueEngineDelegate): |
| watchers = [ |
| ] |
| |
| _pass_status = "Pass" |
| _fail_status = "Fail" |
| _error_status = "Error" |
| |
| def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations |
| options_list = (options or []) + [ |
| make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue. Dangerous!"), |
| make_option("--exit-after-iteration", action="store", type="int", dest="iterations", default=None, help="Stop running the queue after iterating this number of times."), |
| ] |
| Command.__init__(self, "Run the %s" % self.name, options=options_list) |
| self._iteration_count = 0 |
| |
| def _cc_watchers(self, bug_id): |
| try: |
| self.tool.bugs.add_cc_to_bug(bug_id, self.watchers) |
| except Exception, e: |
| traceback.print_exc() |
| log("Failed to CC watchers.") |
| |
| def run_webkit_patch(self, args): |
| webkit_patch_args = [self.tool.path()] |
| # FIXME: This is a hack, we should have a more general way to pass global options. |
| # FIXME: We must always pass global options and their value in one argument |
| # because our global option code looks for the first argument which does |
| # not begin with "-" and assumes that is the command name. |
| webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host] |
| webkit_patch_args.extend(args) |
| return self.tool.executive.run_and_throw_if_fail(webkit_patch_args) |
| |
| def _log_directory(self): |
| return "%s-logs" % self.name |
| |
| # QueueEngineDelegate methods |
| |
| def queue_log_path(self): |
| return os.path.join(self._log_directory(), "%s.log" % self.name) |
| |
| def work_item_log_path(self, work_item): |
| raise NotImplementedError, "subclasses must implement" |
| |
| def begin_work_queue(self): |
| log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root)) |
| if self.options.confirm: |
| response = self.tool.user.prompt("Are you sure? Type \"yes\" to continue: ") |
| if (response != "yes"): |
| error("User declined.") |
| log("Running WebKit %s." % self.name) |
| |
| def should_continue_work_queue(self): |
| self._iteration_count += 1 |
| return not self.options.iterations or self._iteration_count <= self.options.iterations |
| |
| def next_work_item(self): |
| raise NotImplementedError, "subclasses must implement" |
| |
| def should_proceed_with_work_item(self, work_item): |
| raise NotImplementedError, "subclasses must implement" |
| |
| def process_work_item(self, work_item): |
| raise NotImplementedError, "subclasses must implement" |
| |
| def handle_unexpected_error(self, work_item, message): |
| raise NotImplementedError, "subclasses must implement" |
| |
| # Command methods |
| |
| def execute(self, options, args, tool, engine=QueueEngine): |
| self.options = options |
| self.tool = tool |
| return engine(self.name, self, self.tool.wakeup_event).run() |
| |
| @classmethod |
| def _log_from_script_error_for_upload(cls, script_error, output_limit=None): |
| # We have seen request timeouts with app engine due to large |
| # log uploads. Trying only the last 512k. |
| if not output_limit: |
| output_limit = 512 * 1024 # 512k |
| output = script_error.message_with_output(output_limit=output_limit) |
| # We pre-encode the string to a byte array before passing it |
| # to status_server, because ClientForm (part of mechanize) |
| # wants a file-like object with pre-encoded data. |
| return StringIO(output.encode("utf-8")) |
| |
| @classmethod |
| def _update_status_for_script_error(cls, tool, state, script_error, is_error=False): |
| message = str(script_error) |
| if is_error: |
| message = "Error: %s" % message |
| failure_log = cls._log_from_script_error_for_upload(script_error) |
| return tool.status_server.update_status(cls.name, message, state["patch"], failure_log) |
| |
| |
| class AbstractPatchQueue(AbstractQueue): |
| def _update_status(self, message, patch=None, results_file=None): |
| self.tool.status_server.update_status(self.name, message, patch, results_file) |
| |
| def _update_work_items(self, patch_ids): |
| self.tool.status_server.update_work_items(self.name, patch_ids) |
| |
| def _did_pass(self, patch): |
| self._update_status(self._pass_status, patch) |
| |
| def _did_fail(self, patch): |
| self._update_status(self._fail_status, patch) |
| |
| def _did_error(self, patch, reason): |
| message = "%s: %s" % (self._error_status, reason) |
| self._update_status(message, patch) |
| |
| def work_item_log_path(self, patch): |
| return os.path.join(self._log_directory(), "%s.log" % patch.bug_id()) |
| |
| def log_progress(self, patch_ids): |
| log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids)))) |
| |
| |
| class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler): |
| name = "commit-queue" |
| def __init__(self): |
| AbstractPatchQueue.__init__(self) |
| |
| # AbstractPatchQueue methods |
| |
| def begin_work_queue(self): |
| AbstractPatchQueue.begin_work_queue(self) |
| self.committer_validator = CommitterValidator(self.tool.bugs) |
| |
| def _validate_patches_in_commit_queue(self): |
| # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers. |
| bug_ids = self.tool.bugs.queries.fetch_bug_ids_from_commit_queue() |
| all_patches = sum([self.tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True) for bug_id in bug_ids], []) |
| return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches) |
| |
| def _patch_cmp(self, a, b): |
| # Sort first by is_rollout, then by attach_date. |
| # Reversing the order so that is_rollout is first. |
| rollout_cmp = cmp(b.is_rollout(), a.is_rollout()) |
| if (rollout_cmp != 0): |
| return rollout_cmp |
| return cmp(a.attach_date(), b.attach_date()) |
| |
| def next_work_item(self): |
| patches = self._validate_patches_in_commit_queue() |
| patches = sorted(patches, self._patch_cmp) |
| self._update_work_items([patch.id() for patch in patches]) |
| if not patches: |
| return None |
| # Only bother logging if we have patches in the queue. |
| self.log_progress([patch.id() for patch in patches]) |
| return patches[0] |
| |
| def _can_build_and_test(self): |
| try: |
| self.run_webkit_patch([ |
| "build-and-test", |
| "--force-clean", |
| "--build", |
| "--test", |
| "--non-interactive", |
| "--no-update", |
| "--build-style=both", |
| "--quiet"]) |
| except ScriptError, e: |
| failure_log = self._log_from_script_error_for_upload(e) |
| self._update_status("Unable to successfully do a clean build and test", results_file=failure_log) |
| return False |
| return True |
| |
| def should_proceed_with_work_item(self, patch): |
| patch_text = "rollout patch" if patch.is_rollout() else "patch" |
| self._update_status("Landing %s" % patch_text, patch) |
| return True |
| |
| def _land(self, patch, first_run=False): |
| try: |
| args = [ |
| "land-attachment", |
| "--force-clean", |
| "--build", |
| "--non-interactive", |
| "--ignore-builders", |
| "--build-style=both", |
| "--quiet", |
| patch.id() |
| ] |
| # We don't bother to run tests for rollouts as that makes them too slow. |
| if not patch.is_rollout(): |
| args.append("--test") |
| if not first_run: |
| # The first time through, we don't reject the patch from the |
| # commit queue because we want to make sure we can build and |
| # test ourselves. However, the second time through, we |
| # register ourselves as the parent-command so we can reject |
| # the patch on failure. |
| args.append("--parent-command=commit-queue") |
| # The second time through, we also don't want to update so we |
| # know we're testing the same revision that we successfully |
| # built and tested. |
| args.append("--no-update") |
| self.run_webkit_patch(args) |
| self._did_pass(patch) |
| return True |
| except ScriptError, e: |
| if first_run: |
| return False |
| self._did_fail(patch) |
| raise |
| |
| def process_work_item(self, patch): |
| self._cc_watchers(patch.bug_id()) |
| if not self._land(patch, first_run=True): |
| self._update_status("Patch could not be landed with first attempt, doing a clean build as a sanity check", patch) |
| # The patch failed to land, but the bots were green. It's possible |
| # that the bots were behind. To check that case, we try to build and |
| # test ourselves. |
| if not self._can_build_and_test(): |
| return False |
| self._update_status("Clean build succeeded, trying patch again", patch) |
| # Hum, looks like the patch is actually bad. Of course, we could |
| # have been bitten by a flaky test the first time around. We try |
| # to land again. If it fails a second time, we're pretty sure its |
| # a bad test and re can reject it outright. |
| self._land(patch) |
| return True |
| |
| def handle_unexpected_error(self, patch, message): |
| self.committer_validator.reject_patch_from_commit_queue(patch.id(), message) |
| |
| # StepSequenceErrorHandler methods |
| @staticmethod |
| def _error_message_for_bug(tool, status_id, script_error): |
| if not script_error.output: |
| return script_error.message_with_output() |
| results_link = tool.status_server.results_url_for_status(status_id) |
| return "%s\nFull output: %s" % (script_error.message_with_output(), results_link) |
| |
| @classmethod |
| def handle_script_error(cls, tool, state, script_error): |
| status_id = cls._update_status_for_script_error(tool, state, script_error) |
| validator = CommitterValidator(tool.bugs) |
| validator.reject_patch_from_commit_queue(state["patch"].id(), cls._error_message_for_bug(tool, status_id, script_error)) |
| |
| @classmethod |
| def handle_checkout_needs_update(cls, tool, state, options, error): |
| # The only time when we find out that out checkout needs update is |
| # when we were ready to actually pull the trigger and land the patch. |
| # Rather than spinning in the master process, we retry without |
| # building or testing, which is much faster. |
| options.build = False |
| options.test = False |
| options.update = True |
| raise TryAgain() |
| |
| |
| class RietveldUploadQueue(AbstractPatchQueue, StepSequenceErrorHandler): |
| name = "rietveld-upload-queue" |
| |
| def __init__(self): |
| AbstractPatchQueue.__init__(self) |
| |
| # AbstractPatchQueue methods |
| |
| def next_work_item(self): |
| patch_id = self.tool.bugs.queries.fetch_first_patch_from_rietveld_queue() |
| if patch_id: |
| return patch_id |
| self._update_status("Empty queue") |
| |
| def should_proceed_with_work_item(self, patch): |
| self._update_status("Uploading patch", patch) |
| return True |
| |
| def process_work_item(self, patch): |
| try: |
| self.run_webkit_patch(["post-attachment-to-rietveld", "--force-clean", "--non-interactive", "--parent-command=rietveld-upload-queue", patch.id()]) |
| self._did_pass(patch) |
| return True |
| except ScriptError, e: |
| if e.exit_code != QueueEngine.handled_error_code: |
| self._did_fail(patch) |
| raise e |
| |
| @classmethod |
| def _reject_patch(cls, tool, patch_id): |
| tool.bugs.set_flag_on_attachment(patch_id, "in-rietveld", "-") |
| |
| def handle_unexpected_error(self, patch, message): |
| log(message) |
| self._reject_patch(self.tool, patch.id()) |
| |
| # StepSequenceErrorHandler methods |
| |
| @classmethod |
| def handle_script_error(cls, tool, state, script_error): |
| log(script_error.message_with_output()) |
| cls._update_status_for_script_error(tool, state, script_error) |
| cls._reject_patch(tool, state["patch"].id()) |
| |
| |
| class AbstractReviewQueue(AbstractPatchQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler): |
| def __init__(self, options=None): |
| AbstractPatchQueue.__init__(self, options) |
| |
| def review_patch(self, patch): |
| raise NotImplementedError, "subclasses must implement" |
| |
| # PersistentPatchCollectionDelegate methods |
| |
| def collection_name(self): |
| return self.name |
| |
| def fetch_potential_patch_ids(self): |
| return self.tool.bugs.queries.fetch_attachment_ids_from_review_queue() |
| |
| def status_server(self): |
| return self.tool.status_server |
| |
| def is_terminal_status(self, status): |
| return status == "Pass" or status == "Fail" or status.startswith("Error:") |
| |
| # AbstractPatchQueue methods |
| |
| def begin_work_queue(self): |
| AbstractPatchQueue.begin_work_queue(self) |
| self._patches = PersistentPatchCollection(self) |
| |
| def next_work_item(self): |
| patch_id = self._patches.next() |
| if patch_id: |
| return self.tool.bugs.fetch_attachment(patch_id) |
| |
| def should_proceed_with_work_item(self, patch): |
| raise NotImplementedError, "subclasses must implement" |
| |
| def process_work_item(self, patch): |
| try: |
| if not self.review_patch(patch): |
| return False |
| self._did_pass(patch) |
| return True |
| except ScriptError, e: |
| if e.exit_code != QueueEngine.handled_error_code: |
| self._did_fail(patch) |
| raise e |
| |
| def handle_unexpected_error(self, patch, message): |
| log(message) |
| |
| # StepSequenceErrorHandler methods |
| |
| @classmethod |
| def handle_script_error(cls, tool, state, script_error): |
| log(script_error.message_with_output()) |
| |
| |
| class StyleQueue(AbstractReviewQueue): |
| name = "style-queue" |
| def __init__(self): |
| AbstractReviewQueue.__init__(self) |
| |
| def should_proceed_with_work_item(self, patch): |
| self._update_status("Checking style", patch) |
| return True |
| |
| def review_patch(self, patch): |
| self.run_webkit_patch(["check-style", "--force-clean", "--non-interactive", "--parent-command=style-queue", patch.id()]) |
| return True |
| |
| @classmethod |
| def handle_script_error(cls, tool, state, script_error): |
| is_svn_apply = script_error.command_name() == "svn-apply" |
| status_id = cls._update_status_for_script_error(tool, state, script_error, is_error=is_svn_apply) |
| if is_svn_apply: |
| QueueEngine.exit_after_handled_error(script_error) |
| message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (state["patch"].id(), cls.name, script_error.message_with_output(output_limit=3*1024)) |
| tool.bugs.post_comment_to_bug(state["patch"].bug_id(), message, cc=cls.watchers) |
| exit(1) |
