blob: 9a72c7c7f30cf8ab039eaf2e5cf0c3323f6fa70c [file] [log] [blame]
# Copyright (C) 2017 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import logging
import unittest
from webkitpy.common.net.generictestresults import BindingsTestResults
from webkitpy.common.net.jsctestresults import JSCTestResults
from webkitpy.common.system.executive import ScriptError
from webkitpy.tool.bot.patchanalysistask import *
from webkitpy.tool.commands.earlywarningsystem import AbstractEarlyWarningSystem
from webkitpy.tool.mocktool import MockTool
_log = logging.getLogger(__name__)
class MockPatchAnalysisTask(PatchAnalysisTask):
def __init__(self, delegate, patch, patches_passed_all_tests):
self._delegate = delegate
self._patch = patch
self._script_error = None
self._results_archive_from_patch_test_run = None
self._results_from_patch_test_run = None
self.error = None
self._patches_passed_all_tests = patches_passed_all_tests
self._test_run_count = 0
self.failure_status_id = 0
def _test(self):
self._test_run_count += 1
if self._patches_passed_all_tests.pop() == True:
return True
self._script_error = ScriptError('Regression test')
return False
def _build_and_test_without_patch(self):
self._test_run_count += 1
return True
def validate(self):
return True
def test_run_count(self):
return self._test_run_count
# This is the delegate to be used with MockPatchAnalysisTask, above.
class MockJSCEarlyWarningSystem(AbstractEarlyWarningSystem):
def __init__(self, first_test_results, second_test_results, clean_test_results):
AbstractEarlyWarningSystem.__init__(self)
self._group = 'jsc'
self._results_in_order = [clean_test_results, second_test_results, first_test_results]
def test_results(self):
return self._results_in_order.pop()
class JSCEarlyWarningSystemTest(unittest.TestCase):
def _results_indicate_all_passed(self, results):
if results == None:
return False
return results.all_passed()
def _create_task(self, first_test_results, second_test_results, clean_test_results):
queue = MockJSCEarlyWarningSystem(first_test_results, second_test_results, clean_test_results)
tool = MockTool(log_executive=True)
patch = tool.bugs.fetch_attachment(10000)
patches_passed_all_tests = map(self._results_indicate_all_passed, [second_test_results, first_test_results])
return MockPatchAnalysisTask(queue, patch, patches_passed_all_tests)
def test_success_case(self):
first_test_results = JSCTestResults(True, [])
second_test_results = JSCTestResults(True, [])
clean_test_results = JSCTestResults(True, [])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 1)
self.assertTrue(return_value)
def test_test_failure(self):
first_test_results = JSCTestResults(True, ['Fail.js'])
second_test_results = JSCTestResults(True, ['Fail.js'])
clean_test_results = JSCTestResults(True, [])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
with self.assertRaises(ScriptError):
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 3)
def test_fix(self):
first_test_results = JSCTestResults(True, [])
second_test_results = JSCTestResults(True, [])
clean_test_results = JSCTestResults(True, ['Fail.js'])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 1)
self.assertTrue(return_value)
def test_ineffective_patch(self):
first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
second_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
clean_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 3)
self.assertTrue(return_value)
def test_partially_effective_patch(self):
first_test_results = JSCTestResults(True, ['failure2.js'])
second_test_results = JSCTestResults(True, ['failure2.js'])
clean_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 3)
self.assertTrue(return_value)
def test_different_test_failures_in_patch_and_tree(self):
first_test_results = JSCTestResults(False, [])
second_test_results = JSCTestResults(False, [])
clean_test_results = JSCTestResults(True, ['failure1.js', 'failure2.js'])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
with self.assertRaises(ScriptError):
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 3)
def test_first_results_could_not_be_read(self):
first_test_results = None
second_test_results = JSCTestResults(True, [])
clean_test_results = JSCTestResults(True, [])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 1)
self.assertFalse(return_value)
def test_second_results_could_not_be_read(self):
first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
second_test_results = None
clean_test_results = JSCTestResults(True, [])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 2)
self.assertFalse(return_value)
def test_clean_results_could_not_be_read(self):
first_test_results = JSCTestResults(True, ['failure2.js'])
second_test_results = JSCTestResults(True, ['failure2.js'])
clean_test_results = None
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 3)
self.assertFalse(return_value)
def test_flaky_results_on_clean_tree_pass(self):
first_test_results = JSCTestResults(True, ['failure2.js'])
second_test_results = JSCTestResults(True, [])
clean_test_results = JSCTestResults(True, [])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertTrue(return_value)
self.assertEqual(task.test_run_count(), 2)
def test_flaky_results_on_clean_tree_pass_v2(self):
first_test_results = JSCTestResults(True, [])
second_test_results = JSCTestResults(True, ['failure2.js'])
clean_test_results = JSCTestResults(True, [])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertTrue(return_value)
self.assertEqual(task.test_run_count(), 1)
def test_flaky_results_on_clean_tree_failure(self):
first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
second_test_results = JSCTestResults(True, ['failure2.js'])
clean_test_results = JSCTestResults(True, [])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
with self.assertRaises(ScriptError):
task._test_patch()
self.assertEqual(task.test_run_count(), 3)
def test_flaky_results_on_red_tree_pass(self):
first_test_results = JSCTestResults(True, ['failure1.js'])
second_test_results = JSCTestResults(True, ['failure1.js', 'failure2.js'])
clean_test_results = JSCTestResults(True, ['failure1.js'])
task = self._create_task(first_test_results, second_test_results, clean_test_results)
return_value = task._test_patch()
self.assertTrue(return_value)
self.assertEqual(task.test_run_count(), 3)
class MockBindingsEarlyWarningSystem(AbstractEarlyWarningSystem):
def __init__(self, first_test_results, clean_test_results):
AbstractEarlyWarningSystem.__init__(self)
self._group = 'bindings'
self._results_in_order = [clean_test_results, first_test_results]
def test_results(self):
return self._results_in_order.pop()
class BindingsEarlyWarningSystemTest(unittest.TestCase):
def _results_indicate_all_passed(self, results):
if results == None:
return False
return results.all_passed()
def _create_task(self, first_test_results, clean_test_results):
queue = MockBindingsEarlyWarningSystem(first_test_results, clean_test_results)
tool = MockTool(log_executive=True)
patch = tool.bugs.fetch_attachment(10000)
patches_passed_all_tests = [self._results_indicate_all_passed(first_test_results)]
return MockPatchAnalysisTask(queue, patch, patches_passed_all_tests)
def test_success_case(self):
first_test_results = BindingsTestResults([])
clean_test_results = BindingsTestResults([])
task = self._create_task(first_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 1)
self.assertTrue(return_value)
def test_test_failure(self):
first_test_results = BindingsTestResults(['TestMapLike.idl'])
clean_test_results = BindingsTestResults([])
task = self._create_task(first_test_results, clean_test_results)
with self.assertRaises(ScriptError):
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 2)
def test_fix(self):
first_test_results = BindingsTestResults([])
clean_test_results = BindingsTestResults(['TestMapLike.idl'])
task = self._create_task(first_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 1)
self.assertTrue(return_value)
def test_ineffective_patch(self):
first_test_results = BindingsTestResults(['TestMapLike.idl'])
clean_test_results = BindingsTestResults(['TestMapLike.idl'])
task = self._create_task(first_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 2)
self.assertTrue(return_value)
def test_partially_effective_patch(self):
first_test_results = BindingsTestResults(['TestMapLike.idl'])
clean_test_results = BindingsTestResults(['TestMapLike.idl', 'TestNode.idl'])
task = self._create_task(first_test_results, clean_test_results)
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 2)
self.assertTrue(return_value)
def test_different_test_failures_in_patch_and_tree(self):
first_test_results = BindingsTestResults(['TestNode.idl'])
clean_test_results = BindingsTestResults(['TestMapLike.idl'])
task = self._create_task(first_test_results, clean_test_results)
with self.assertRaises(ScriptError):
return_value = task._test_patch()
self.assertEqual(task.test_run_count(), 2)