# Copyright (c) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from datetime import datetime
import logging
import unittest

from webkitpy.common.net import bugzilla
from webkitpy.common.net.layouttestresults import LayoutTestResults
from webkitpy.common.system.executive import ScriptError
from webkitpy.common.system.outputcapture import OutputCapture
from webkitpy.layout_tests.models import test_results
from webkitpy.layout_tests.models import test_failures
from webkitpy.thirdparty.mock import Mock
from webkitpy.tool.bot.commitqueuetask import *
from webkitpy.tool.mocktool import MockTool

_log = logging.getLogger(__name__)


class MockCommitQueue(CommitQueueTaskDelegate):
    def __init__(self, error_plan):
        self._error_plan = error_plan
        self._failure_status_id = 0
        self._flaky_tests = []

    def run_command(self, command):
        _log.info("run_webkit_patch: %s" % command)
        if self._error_plan:
            error = self._error_plan.pop(0)
            if error:
                raise error

    def command_passed(self, success_message, patch):
        _log.info("command_passed: success_message='%s' patch='%s'" % (
            success_message, patch.id()))

    def command_failed(self, failure_message, script_error, patch):
        _log.info("command_failed: failure_message='%s' script_error='%s' patch='%s'" % (
            failure_message, script_error, patch.id()))
        self._failure_status_id += 1
        return self._failure_status_id

    def refetch_patch(self, patch):
        return patch

    def test_results(self):
        return LayoutTestResults(test_results=[], did_exceed_test_failure_limit=False)

    def report_flaky_tests(self, patch, flaky_results, results_archive):
        current_flaky_tests = [result.test_name for result in flaky_results]
        self._flaky_tests += current_flaky_tests
        _log.info("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'" % (patch.id(), current_flaky_tests, results_archive.filename))

    def get_reported_flaky_tests(self):
        return self._flaky_tests

    def archive_last_test_results(self, patch):
        _log.info("archive_last_test_results: patch='%s'" % patch.id())
        archive = Mock()
        archive.filename = "mock-archive-%s.zip" % patch.id()
        return archive

    def build_style(self):
        return "release"

    def did_pass_testing_ews(self, patch):
        return False


class FailingTestCommitQueue(MockCommitQueue):
    def __init__(self, error_plan, test_failure_plan):
        MockCommitQueue.__init__(self, error_plan)
        self._test_run_counter = -1  # Special value to indicate tests have never been run.
        self._test_failure_plan = test_failure_plan

    def run_command(self, command):
        if command[0] == "build-and-test":
            self._test_run_counter += 1
        MockCommitQueue.run_command(self, command)

    def _mock_test_result(self, testname):
        return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])

    def test_results(self):
        # Doesn't make sense to ask for the test_results until the tests have run at least once.
        assert(self._test_run_counter >= 0)
        failures_for_run = self._test_failure_plan[self._test_run_counter]
        assert(isinstance(failures_for_run, list))
        results = LayoutTestResults(test_results=map(self._mock_test_result, failures_for_run), did_exceed_test_failure_limit=(len(failures_for_run) >= 10))
        return results


class PatchAnalysisResult(object):
    FAIL = "Fail"
    DEFER = "Defer"
    PASS = "Pass"


class MockSimpleTestPlanCommitQueue(MockCommitQueue):
    def __init__(self, first_test_failures, second_test_failures, clean_test_failures):
        MockCommitQueue.__init__(self, [])
        self._did_run_clean_tests = False
        self._patch_test_results = [first_test_failures, second_test_failures]
        self._clean_test_results = [clean_test_failures]
        self._current_test_results = []

    def run_command(self, command):
        MockCommitQueue.run_command(self, command)
        if command[0] == "build-and-test":
            if "--no-clean" in command:
                self._current_test_results = self._patch_test_results.pop(0)
            else:
                self._current_test_results = self._clean_test_results.pop(0)
                self._did_run_clean_tests = True

            if self._current_test_results:
                raise ScriptError("MOCK test failure")

    def _mock_test_result(self, testname):
        return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])

    def test_results(self):
        assert(isinstance(self._current_test_results, list))
        return LayoutTestResults(test_results=map(self._mock_test_result, self._current_test_results), did_exceed_test_failure_limit=(len(self._current_test_results) >= 10))

    def did_run_clean_tests(self):
        return self._did_run_clean_tests


# We use GoldenScriptError to make sure that the code under test throws the
# correct (i.e., golden) exception.
class GoldenScriptError(ScriptError):
    pass


_lots_of_failing_tests = map(lambda num: "test-%s.html" % num, range(0, 100))


class CommitQueueTaskTest(unittest.TestCase):
    def _run_and_expect_patch_analysis_result(self, commit_queue, expected_analysis_result, expected_reported_flaky_tests=[], expect_clean_tests_to_run=False, expected_failure_status_id=0):
        tool = MockTool(log_executive=True)
        patch = tool.bugs.fetch_attachment(10000)
        task = CommitQueueTask(commit_queue, patch)

        try:
            result = task.run()
            if result:
                analysis_result = PatchAnalysisResult.PASS
            else:
                analysis_result = PatchAnalysisResult.DEFER
        except ScriptError:
            analysis_result = PatchAnalysisResult.FAIL

        self.assertEqual(analysis_result, expected_analysis_result)
        self.assertEqual(frozenset(commit_queue.get_reported_flaky_tests()), frozenset(expected_reported_flaky_tests))
        self.assertEqual(commit_queue.did_run_clean_tests(), expect_clean_tests_to_run)

        # The failure status only means anything if we actually failed.
        if expected_analysis_result == PatchAnalysisResult.FAIL:
            self.assertEqual(task.failure_status_id, expected_failure_status_id)
            self.assertIsInstance(task.results_from_patch_test_run(patch), LayoutTestResults)

    def _run_through_task(self, commit_queue, expected_logs, expected_exception=None, expect_retry=False):
        self.maxDiff = None
        tool = MockTool(log_executive=True)
        patch = tool.bugs.fetch_attachment(10000)
        task = CommitQueueTask(commit_queue, patch)
        success = OutputCapture().assert_outputs(self, task.run, expected_logs=expected_logs, expected_exception=expected_exception)
        if not expected_exception:
            self.assertEqual(success, not expect_retry)
        return task

    def test_success_case(self):
        commit_queue = MockCommitQueue([])
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='10000'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
command_passed: success_message='Applied patch' patch='10000'
run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
command_passed: success_message='ChangeLog validated' patch='10000'
run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
command_passed: success_message='Built patch' patch='10000'
run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive', '--build-style=release']
command_passed: success_message='Passed tests' patch='10000'
run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
command_passed: success_message='Landed patch' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs)

    def test_fast_success_case(self):
        commit_queue = MockCommitQueue([])
        commit_queue.did_pass_testing_ews = lambda patch: True
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='10000'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
command_passed: success_message='Applied patch' patch='10000'
run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
command_passed: success_message='ChangeLog validated' patch='10000'
run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
command_passed: success_message='Built patch' patch='10000'
run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
command_passed: success_message='Landed patch' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs)

    def test_clean_failure(self):
        commit_queue = MockCommitQueue([
            ScriptError("MOCK clean failure"),
        ])
        expected_logs = """run_webkit_patch: ['clean']
command_failed: failure_message='Unable to clean working directory' script_error='MOCK clean failure' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs, expect_retry=True)

    def test_update_failure(self):
        commit_queue = MockCommitQueue([
            None,
            ScriptError("MOCK update failure"),
        ])
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_failed: failure_message='Unable to update working directory' script_error='MOCK update failure' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs, expect_retry=True)

    def test_apply_failure(self):
        commit_queue = MockCommitQueue([
            None,
            None,
            GoldenScriptError("MOCK apply failure"),
        ])
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='10000'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
command_failed: failure_message='Patch does not apply' script_error='MOCK apply failure' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs, GoldenScriptError)

    def test_validate_changelog_failure(self):
        commit_queue = MockCommitQueue([
            None,
            None,
            None,
            GoldenScriptError("MOCK validate failure"),
        ])
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='10000'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
command_passed: success_message='Applied patch' patch='10000'
run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
command_failed: failure_message='ChangeLog did not pass validation' script_error='MOCK validate failure' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs, GoldenScriptError)

    def test_build_failure(self):
        commit_queue = MockCommitQueue([
            None,
            None,
            None,
            None,
            GoldenScriptError("MOCK build failure"),
        ])
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='10000'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
command_passed: success_message='Applied patch' patch='10000'
run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
command_passed: success_message='ChangeLog validated' patch='10000'
run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=release']
command_passed: success_message='Able to build without patch' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs, GoldenScriptError)

    def test_red_build_failure(self):
        commit_queue = MockCommitQueue([
            None,
            None,
            None,
            None,
            ScriptError("MOCK build failure"),
            ScriptError("MOCK clean build failure"),
        ])
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='10000'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
command_passed: success_message='Applied patch' patch='10000'
run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
command_passed: success_message='ChangeLog validated' patch='10000'
run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=release']
command_failed: failure_message='Unable to build without patch' script_error='MOCK clean build failure' patch='10000'
"""
        self._run_through_task(commit_queue, expected_logs, expect_retry=True)

    def test_land_failure(self):
        commit_queue = MockCommitQueue([
            None,
            None,
            None,
            None,
            None,
            None,
            GoldenScriptError("MOCK land failure"),
        ])
        expected_logs = """run_webkit_patch: ['clean']
command_passed: success_message='Cleaned working directory' patch='10000'
run_webkit_patch: ['update']
command_passed: success_message='Updated working directory' patch='10000'
run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
command_passed: success_message='Applied patch' patch='10000'
run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
command_passed: success_message='ChangeLog validated' patch='10000'
run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=release']
command_passed: success_message='Built patch' patch='10000'
run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive', '--build-style=release']
command_passed: success_message='Passed tests' patch='10000'
run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
command_failed: failure_message='Unable to land patch' script_error='MOCK land failure' patch='10000'
"""
        # FIXME: This should really be expect_retry=True for a better user experiance.
        self._run_through_task(commit_queue, expected_logs, GoldenScriptError)

    def test_failed_archive(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1"],
            second_test_failures=[],
            clean_test_failures=[])

        # It's possible for the delegate to fail to archive layout tests,
        # but we shouldn't try to report flaky tests when that happens.
        commit_queue.archive_last_test_results = lambda patch: None

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS)

    def test_double_flaky_test_failure(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1"],
            second_test_failures=["Fail2"],
            clean_test_failures=["Fail1"])

        # The (subtle) point of this test is that report_flaky_tests does not get
        # called for this run.
        # Note also that there is no attempt to run the tests w/o the patch.
        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expected_reported_flaky_tests=["Fail1", "Fail2"])

    def test_test_failure(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1"],
            second_test_failures=["Fail1"],
            clean_test_failures=[])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_failure_status_id=1)

    def test_red_test_failure(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1"],
            second_test_failures=["Fail1"],
            clean_test_failures=["Fail1"])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS, expect_clean_tests_to_run=True)

    def test_first_failure_limit(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=_lots_of_failing_tests,
            second_test_failures=[],
            clean_test_failures=[])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_failure_status_id=1)

    def test_first_failure_limit_with_some_tree_redness(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=_lots_of_failing_tests,
            second_test_failures=["Fail1", "Fail2", "Fail3"],
            clean_test_failures=["Fail1", "Fail2", "Fail3"])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_failure_status_id=1)

    def test_second_failure_limit(self):
        # There need to be some failures in the first set of tests, or it won't even make it to the second test.
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1", "Fail2", "Fail3"],
            second_test_failures=_lots_of_failing_tests,
            clean_test_failures=["Fail1", "Fail2", "Fail3"])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_failure_status_id=2)

    def test_tree_failure_limit_with_patch_that_potentially_fixes_some_redness(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1", "Fail2", "Fail3"],
            second_test_failures=["Fail1", "Fail2", "Fail3"],
            clean_test_failures=_lots_of_failing_tests)

        # Unfortunately there are cases where the clean build will randomly fail enough tests to hit the failure limit.
        # With that in mind, we can't actually know that this patch is good or bad until we see a clean run that doesn't
        # exceed the failure limit.
        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True)

    def test_first_and_second_failure_limit(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=_lots_of_failing_tests,
            second_test_failures=_lots_of_failing_tests,
            clean_test_failures=[])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_failure_status_id=1)

    def test_first_and_clean_failure_limit(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=_lots_of_failing_tests,
            second_test_failures=[],
            clean_test_failures=_lots_of_failing_tests)

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True)

    def test_first_second_and_clean_failure_limit(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=_lots_of_failing_tests,
            second_test_failures=_lots_of_failing_tests,
            clean_test_failures=_lots_of_failing_tests)

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True)

    def test_red_tree_patch_rejection(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1", "Fail2"],
            second_test_failures=["Fail1", "Fail2"],
            clean_test_failures=["Fail1"])

        # failure_status_id should be of the test with patch (1), not the test without patch (2).
        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_failure_status_id=1)

    def test_two_flaky_tests(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1"],
            second_test_failures=["Fail2"],
            clean_test_failures=["Fail1", "Fail2"])

        # FIXME: This should pass, but as of right now, it defers.
        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expected_reported_flaky_tests=["Fail1", "Fail2"])

    def test_one_flaky_test(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1"],
            second_test_failures=[],
            clean_test_failures=[])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS, expected_reported_flaky_tests=["Fail1"])

    def test_very_flaky_patch(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
            second_test_failures=["Fail6", "Fail7", "Fail8", "Fail9", "Fail10"],
            clean_test_failures=[])

        # FIXME: This should actually fail, but right now it defers
        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expected_reported_flaky_tests=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6", "Fail7", "Fail8", "Fail9", "Fail10"])

    def test_very_flaky_patch_with_some_tree_redness(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
            second_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail6", "Fail7", "Fail8", "Fail9", "Fail10"],
            clean_test_failures=["PreExistingFail1", "PreExistingFail2"])

        # FIXME: This should actually fail, but right now it defers
        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6", "Fail7", "Fail8", "Fail9", "Fail10"])

    def test_different_test_failures(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6"],
            second_test_failures=["Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
            clean_test_failures=[])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail6"], expected_failure_status_id=1)

    def test_different_test_failures_with_some_tree_redness(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6"],
            second_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
            clean_test_failures=["PreExistingFail1", "PreExistingFail2"])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail6"], expected_failure_status_id=1)

    def test_different_test_failures_with_some_tree_redness_and_some_fixes(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["PreExistingFail1", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5", "Fail6"],
            second_test_failures=["PreExistingFail1", "Fail1", "Fail2", "Fail3", "Fail4", "Fail5"],
            clean_test_failures=["PreExistingFail1", "PreExistingFail2"])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.FAIL, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail6"], expected_failure_status_id=1)

    def test_mildly_flaky_patch(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1"],
            second_test_failures=["Fail2"],
            clean_test_failures=[])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=False, expected_reported_flaky_tests=["Fail1", "Fail2"])

    def test_mildly_flaky_patch_with_some_tree_redness(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail1"],
            second_test_failures=["PreExistingFail1", "PreExistingFail2", "Fail2"],
            clean_test_failures=["PreExistingFail1", "PreExistingFail2"])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.DEFER, expect_clean_tests_to_run=True, expected_reported_flaky_tests=["Fail1", "Fail2"])

    def test_tree_more_red_than_patch(self):
        commit_queue = MockSimpleTestPlanCommitQueue(
            first_test_failures=["Fail1", "Fail2", "Fail3"],
            second_test_failures=["Fail1", "Fail2", "Fail3"],
            clean_test_failures=["Fail1", "Fail2", "Fail3", "Fail4"])

        self._run_and_expect_patch_analysis_result(commit_queue, PatchAnalysisResult.PASS, expect_clean_tests_to_run=True)

    def _expect_validate(self, patch, is_valid):
        class MockDelegate(object):
            def refetch_patch(self, patch):
                return patch

        task = CommitQueueTask(MockDelegate(), patch)
        self.assertEqual(task.validate(), is_valid)

    def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, committer="fake"):
        bug = bugzilla.Bug(bug_dict, None)
        patch = bugzilla.Attachment(attachment_dict, bug)
        patch._committer = committer
        return patch

    def test_validate(self):
        self._expect_validate(self._mock_patch(), True)
        self._expect_validate(self._mock_patch({'is_obsolete': True}), False)
        self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}), False)
        self._expect_validate(self._mock_patch(committer=None), False)
        self._expect_validate(self._mock_patch({'review': '-'}), False)