| # Copyright (C) 2018-2021 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND |
| # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR |
| # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import inspect |
| import json |
| import operator |
| import os |
| import shutil |
| import sys |
| import tempfile |
| |
| from buildbot.process import remotetransfer |
| from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY |
| from buildbot.test.fake.remotecommand import Expect, ExpectRemoteRef, ExpectShell |
| from buildbot.test.util.misc import TestReactorMixin |
| from buildbot.test.util.steps import BuildStepMixin |
| from buildbot.util import identifiers as buildbot_identifiers |
| from mock import call, patch |
| from twisted.internet import defer, error, reactor |
| from twisted.python import failure, log |
| from twisted.trial import unittest |
| |
| |
| from steps import (AnalyzeAPITestsResults, AnalyzeCompileWebKitResults, AnalyzeJSCTestsResults, |
| AnalyzeLayoutTestsResults, ApplyPatch, ApplyWatchList, ArchiveBuiltProduct, ArchiveTestResults, |
| CheckOutSource, CheckOutSpecificRevision, CheckPatchRelevance, CheckPatchStatusOnEWSQueues, CheckStyle, |
| CleanBuild, CleanUpGitIndexLock, CleanGitRepo, CleanWorkingDirectory, CompileJSC, CompileJSCWithoutPatch, |
| CompileWebKit, CompileWebKitWithoutPatch, ConfigureBuild, CreateLocalGITCommit, |
| DownloadBuiltProduct, DownloadBuiltProductFromMaster, EWS_BUILD_HOSTNAME, ExtractBuiltProduct, ExtractTestResults, |
| FetchBranches, FindModifiedChangeLogs, FindModifiedLayoutTests, GitResetHard, |
| InstallBuiltProduct, InstallGtkDependencies, InstallWpeDependencies, |
| KillOldProcesses, PrintConfiguration, PushCommitToWebKitRepo, ReRunAPITests, ReRunJavaScriptCoreTests, ReRunWebKitPerlTests, |
| ReRunWebKitTests, RunAPITests, RunAPITestsWithoutPatch, RunBindingsTests, RunBuildWebKitOrgUnitTests, |
| RunBuildbotCheckConfigForBuildWebKit, RunBuildbotCheckConfigForEWS, RunEWSUnitTests, RunResultsdbpyTests, |
| RunJavaScriptCoreTests, RunJSCTestsWithoutPatch, RunWebKit1Tests, RunWebKitPerlTests, RunWebKitPyPython2Tests, |
| RunWebKitPyPython3Tests, RunWebKitTests, RunWebKitTestsInStressMode, RunWebKitTestsInStressGuardmallocMode, |
| RunWebKitTestsWithoutPatch, TestWithFailureCount, ShowIdentifier, |
| Trigger, TransferToS3, UnApplyPatchIfRequired, UpdateWorkingDirectory, UploadBuiltProduct, |
| UploadTestResults, ValidateChangeLogAndReviewer, ValidateCommiterAndReviewer, ValidatePatch, VerifyGitHubIntegrity) |
| |
| # Workaround for https://github.com/buildbot/buildbot/issues/4669 |
| from buildbot.test.fake.fakebuild import FakeBuild |
| FakeBuild.addStepsAfterCurrentStep = lambda FakeBuild, step_factories: None |
| |
| |
| def mock_step(step, logs='', results=SUCCESS, stopped=False, properties=None): |
| step.logs = logs |
| step.results = results |
| step.stopped = stopped |
| return step |
| |
| |
| class ExpectMasterShellCommand(object): |
| def __init__(self, command, workdir=None, env=None, usePTY=0): |
| self.args = command |
| self.usePTY = usePTY |
| self.rc = None |
| self.path = None |
| self.logs = [] |
| |
| if env is not None: |
| self.env = env |
| else: |
| self.env = os.environ |
| if workdir: |
| self.path = os.path.join(os.getcwd(), workdir) |
| |
| @classmethod |
| def log(self, name, value): |
| return ('log', name, value) |
| |
| def __add__(self, other): |
| if isinstance(other, int): |
| self.rc = other |
| elif isinstance(other, tuple) and other[0] == 'log': |
| self.logs.append((other[1], other[2])) |
| return self |
| |
| def __repr__(self): |
| return 'ExpectMasterShellCommand({0})'.format(repr(self.args)) |
| |
| |
| class BuildStepMixinAdditions(BuildStepMixin, TestReactorMixin): |
| def setUpBuildStep(self): |
| self.patch(reactor, 'spawnProcess', lambda *args, **kwargs: self._checkSpawnProcess(*args, **kwargs)) |
| self._expected_local_commands = [] |
| self.setUpTestReactor() |
| |
| self._temp_directory = tempfile.mkdtemp() |
| os.chdir(self._temp_directory) |
| self._expected_uploaded_files = [] |
| |
| super(BuildStepMixinAdditions, self).setUpBuildStep() |
| |
| def tearDownBuildStep(self): |
| shutil.rmtree(self._temp_directory) |
| super(BuildStepMixinAdditions, self).tearDownBuildStep() |
| |
| def fakeBuildFinished(self, text, results): |
| self.build.text = text |
| self.build.results = results |
| |
| def setupStep(self, step, *args, **kwargs): |
| self.previous_steps = kwargs.get('previous_steps') or [] |
| if self.previous_steps: |
| del kwargs['previous_steps'] |
| |
| super(BuildStepMixinAdditions, self).setupStep(step, *args, **kwargs) |
| self.build.terminate = False |
| self.build.stopped = False |
| self.build.executedSteps = self.executedSteps |
| self.build.buildFinished = self.fakeBuildFinished |
| self._expected_added_urls = [] |
| self._expected_sources = None |
| |
| @property |
| def executedSteps(self): |
| return [step for step in self.previous_steps if not step.stopped] |
| |
| def setProperty(self, name, value, source='Unknown'): |
| self.properties.setProperty(name, value, source) |
| |
| def getProperty(self, name): |
| return self.properties.getProperty(name) |
| |
| def expectAddedURLs(self, added_urls): |
| self._expected_added_urls = added_urls |
| |
| def expectUploadedFile(self, path): |
| self._expected_uploaded_files.append(path) |
| |
| def expectLocalCommands(self, *expected_commands): |
| self._expected_local_commands.extend(expected_commands) |
| |
| def expectRemoteCommands(self, *expected_commands): |
| self.expectCommands(*expected_commands) |
| |
| def expectSources(self, expected_sources): |
| self._expected_sources = expected_sources |
| |
| def _checkSpawnProcess(self, processProtocol, executable, args, env, path, usePTY, **kwargs): |
| got = (executable, args, env, path, usePTY) |
| if not self._expected_local_commands: |
| self.fail('got local command {0} when no further commands were expected'.format(got)) |
| local_command = self._expected_local_commands.pop(0) |
| try: |
| self.assertEqual(got, (local_command.args[0], local_command.args, local_command.env, local_command.path, local_command.usePTY)) |
| except AssertionError: |
| log.err() |
| raise |
| for name, value in local_command.logs: |
| if name == 'stdout': |
| processProtocol.outReceived(value) |
| elif name == 'stderr': |
| processProtocol.errReceived(value) |
| if local_command.rc != 0: |
| value = error.ProcessTerminated(exitCode=local_command.rc) |
| else: |
| value = error.ProcessDone(None) |
| processProtocol.processEnded(failure.Failure(value)) |
| |
| def _added_files(self): |
| results = [] |
| for dirpath, dirnames, filenames in os.walk(self._temp_directory): |
| relative_root_path = os.path.relpath(dirpath, start=self._temp_directory) |
| if relative_root_path == '.': |
| relative_root_path = '' |
| for name in filenames: |
| results.append(os.path.join(relative_root_path, name)) |
| return results |
| |
| def runStep(self): |
| def check(result): |
| self.assertEqual(self._expected_local_commands, [], 'assert all expected local commands were run') |
| self.expectAddedURLs(self._expected_added_urls) |
| self.assertEqual(self._added_files(), self._expected_uploaded_files) |
| if self._expected_sources is not None: |
| # Convert to dictionaries because assertEqual() only knows how to diff Python built-in types. |
| actual_sources = sorted([source.asDict() for source in self.build.sources], key=operator.itemgetter('codebase')) |
| expected_sources = sorted([source.asDict() for source in self._expected_sources], key=operator.itemgetter('codebase')) |
| self.assertEqual(actual_sources, expected_sources) |
| deferred_result = super(BuildStepMixinAdditions, self).runStep() |
| deferred_result.addCallback(check) |
| return deferred_result |
| |
| |
| def uploadFileWithContentsOfString(string, timestamp=None): |
| def behavior(command): |
| writer = command.args['writer'] |
| writer.remote_write(string + '\n') |
| writer.remote_close() |
| if timestamp: |
| writer.remote_utime(timestamp) |
| return behavior |
| |
| |
| class TestStepNameShouldBeValidIdentifier(BuildStepMixinAdditions, unittest.TestCase): |
| def test_step_names_are_valid(self): |
| import steps |
| build_step_classes = inspect.getmembers(steps, inspect.isclass) |
| for build_step in build_step_classes: |
| if 'name' in vars(build_step[1]): |
| name = build_step[1].name |
| self.assertFalse(' ' in name, 'step name "{}" contain space.'.format(name)) |
| self.assertTrue(buildbot_identifiers.ident_re.match(name), 'step name "{}" is not a valid buildbot identifier.'.format(name)) |
| |
| |
| class TestCheckStyle(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success_internal(self): |
| self.setupStep(CheckStyle()) |
| self.setProperty('try-codebase', 'internal') |
| self.setProperty('platform', 'mac') |
| self.setProperty('configuration', 'debug') |
| |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/check-webkit-style'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='check-webkit-style') |
| return self.runStep() |
| |
| def test_failure_unknown_try_codebase(self): |
| self.setupStep(CheckStyle()) |
| self.setProperty('try-codebase', 'foo') |
| self.setProperty('platform', 'mac') |
| self.setProperty('configuration', 'debug') |
| |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/check-webkit-style'], |
| ) |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='check-webkit-style (failure)') |
| return self.runStep() |
| |
| def test_failures_with_style_issues(self): |
| self.setupStep(CheckStyle()) |
| self.setProperty('try-codebase', 'internal') |
| self.setProperty('platform', 'mac') |
| self.setProperty('configuration', 'debug') |
| |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/check-webkit-style'], |
| ) |
| + ExpectShell.log('stdio', stdout='''ERROR: Source/WebCore/layout/FloatingContext.cpp:36: Code inside a namespace should not be indented. [whitespace/indent] [4] |
| ERROR: Source/WebCore/layout/FormattingContext.h:94: Weird number of spaces at line-start. Are you using a 4-space indent? [whitespace/indent] [3] |
| ERROR: Source/WebCore/layout/LayoutContext.cpp:52: Place brace on its own line for function definitions. [whitespace/braces] [4] |
| ERROR: Source/WebCore/layout/LayoutContext.cpp:55: Extra space before last semicolon. If this should be an empty statement, use { } instead. [whitespace/semicolon] [5] |
| ERROR: Source/WebCore/layout/LayoutContext.cpp:60: Tab found; better to use spaces [whitespace/tab] [1] |
| ERROR: Source/WebCore/layout/Verification.cpp:88: Missing space before ( in while( [whitespace/parens] [5] |
| Total errors found: 8 in 48 files''') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='8 style errors') |
| return self.runStep() |
| |
| def test_failures_no_style_issues(self): |
| self.setupStep(CheckStyle()) |
| self.setProperty('try-codebase', 'internal') |
| self.setProperty('platform', 'mac') |
| self.setProperty('configuration', 'debug') |
| |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/check-webkit-style'], |
| ) |
| + ExpectShell.log('stdio', stdout='Total errors found: 0 in 6 files') |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='check-webkit-style') |
| return self.runStep() |
| |
| def test_failures_no_changes(self): |
| self.setupStep(CheckStyle()) |
| self.setProperty('try-codebase', 'internal') |
| self.setProperty('platform', 'mac') |
| self.setProperty('configuration', 'debug') |
| |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/check-webkit-style'], |
| ) |
| + ExpectShell.log('stdio', stdout='Total errors found: 0 in 0 files') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='check-webkit-style (failure)') |
| return self.runStep() |
| |
| |
| class TestApplyWatchList(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(ApplyWatchList()) |
| self.setProperty('bug_id', '1234') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/webkit-patch', 'apply-watchlist-local', '1234']) |
| + ExpectShell.log('stdio', stdout='Result of watchlist: cc "" messages ""') |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Applied WatchList') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(ApplyWatchList()) |
| self.setProperty('bug_id', '1234') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/webkit-patch', 'apply-watchlist-local', '1234']) |
| + ExpectShell.log('stdio', stdout='Unexpected failure') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed to apply watchlist') |
| return self.runStep() |
| |
| |
| class TestValidateChangeLogAndReviewer(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(ValidateChangeLogAndReviewer()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=180, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/webkit-patch', 'validate-changelog', '--check-oops', '--non-interactive']) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Validated ChangeLog and Reviewer') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(ValidateChangeLogAndReviewer()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=180, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/webkit-patch', 'validate-changelog', '--check-oops', '--non-interactive']) |
| + ExpectShell.log('stdio', stdout='ChangeLog entry in LayoutTests/ChangeLog contains OOPS!.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='ChangeLog validation failed') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('bugzilla_comment_text'), 'ChangeLog entry in LayoutTests/ChangeLog contains OOPS!.\n') |
| self.assertEqual(self.getProperty('build_finish_summary'), 'ChangeLog validation failed') |
| return rc |
| |
| |
| class TestRunBindingsTests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'bindings_test_results.json' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunBindingsTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=300, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/run-bindings-tests', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed bindings tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunBindingsTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=300, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/run-bindings-tests', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| ) |
| + ExpectShell.log('stdio', stdout='FAIL: (JS) JSTestInterface.cpp') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='bindings-tests (failure)') |
| return self.runStep() |
| |
| |
| class TestRunWebKitPerlTests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self): |
| self.setupStep(RunWebKitPerlTests()) |
| |
| def test_success(self): |
| self.configureStep() |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/test-webkitperl'], |
| timeout=120, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed webkitperl tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.configureStep() |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/test-webkitperl'], |
| timeout=120, |
| ) |
| + ExpectShell.log('stdio', stdout='''Failed tests: 1-3, 5-7, 9, 11-13 |
| Files=40, Tests=630, 4 wallclock secs ( 0.16 usr 0.09 sys + 2.78 cusr 0.64 csys = 3.67 CPU) |
| Result: FAIL |
| Failed 1/40 test programs. 10/630 subtests failed.''') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed webkitperl tests') |
| return self.runStep() |
| |
| |
| class TestWebKitPyPython2Tests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'webkitpy_test_python2_results.json' |
| self.json_with_failure = '''{"failures": [{"name": "webkitpy.port.wpe_unittest.WPEPortTest.test_diff_image"}]}\n''' |
| self.json_with_errros = '''{"failures": [], |
| "errors": [{"name": "webkitpy.style.checkers.cpp_unittest.WebKitStyleTest.test_os_version_checks"}, {"name": "webkitpy.port.win_unittest.WinPortTest.test_diff_image__missing_actual"}]}\n''' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunWebKitPyPython2Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed webkitpy python2 tests') |
| return self.runStep() |
| |
| def test_unexpected_failure(self): |
| self.setupStep(RunWebKitPyPython2Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) |
| + ExpectShell.log('stdio', stdout='''Ran 1744 tests in 5.913s |
| FAILED (failures=1, errors=0)''') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='webkitpy-tests (failure)') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunWebKitPyPython2Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) + |
| ExpectShell.log('json', stdout=self.json_with_failure) + |
| 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 webkitpy python2 test failure: webkitpy.port.wpe_unittest.WPEPortTest.test_diff_image') |
| return self.runStep() |
| |
| def test_errors(self): |
| self.setupStep(RunWebKitPyPython2Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) + |
| ExpectShell.log('json', stdout=self.json_with_errros) + |
| 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 2 webkitpy python2 test failures: webkitpy.style.checkers.cpp_unittest.WebKitStyleTest.test_os_version_checks, webkitpy.port.win_unittest.WinPortTest.test_diff_image__missing_actual') |
| return self.runStep() |
| |
| def test_lot_of_failures(self): |
| self.setupStep(RunWebKitPyPython2Tests()) |
| json_with_failures = json.dumps({"failures": [{"name": 'test{}'.format(i)} for i in range(1, 31)]}) |
| |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) + |
| ExpectShell.log('json', stdout=json_with_failures) + |
| 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 30 webkitpy python2 test failures: test1, test2, test3, test4, test5, test6, test7, test8, test9, test10 ...') |
| return self.runStep() |
| |
| |
| class TestWebKitPyPython3Tests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'webkitpy_test_python3_results.json' |
| self.json_with_failure = '''{"failures": [{"name": "webkitpy.port.wpe_unittest.WPEPortTest.test_diff_image"}]}\n''' |
| self.json_with_errros = '''{"failures": [], |
| "errors": [{"name": "webkitpy.style.checkers.cpp_unittest.WebKitStyleTest.test_os_version_checks"}, {"name": "webkitpy.port.win_unittest.WinPortTest.test_diff_image__missing_actual"}]}\n''' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunWebKitPyPython3Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed webkitpy python3 tests') |
| return self.runStep() |
| |
| def test_unexpected_failure(self): |
| self.setupStep(RunWebKitPyPython3Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) |
| + ExpectShell.log('stdio', stdout='''Ran 1744 tests in 5.913s |
| FAILED (failures=1, errors=0)''') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='webkitpy-tests (failure)') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunWebKitPyPython3Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) + |
| ExpectShell.log('json', stdout=self.json_with_failure) + |
| 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 webkitpy python3 test failure: webkitpy.port.wpe_unittest.WPEPortTest.test_diff_image') |
| return self.runStep() |
| |
| def test_errors(self): |
| self.setupStep(RunWebKitPyPython3Tests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) + |
| ExpectShell.log('json', stdout=self.json_with_errros) + |
| 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 2 webkitpy python3 test failures: webkitpy.style.checkers.cpp_unittest.WebKitStyleTest.test_os_version_checks, webkitpy.port.win_unittest.WinPortTest.test_diff_image__missing_actual') |
| return self.runStep() |
| |
| def test_lot_of_failures(self): |
| self.setupStep(RunWebKitPyPython3Tests()) |
| json_with_failures = json.dumps({"failures": [{"name": 'test{}'.format(i)} for i in range(1, 31)]}) |
| |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/test-webkitpy', '--verbose', '--json-output={0}'.format(self.jsonFileName)], |
| logfiles={'json': self.jsonFileName}, |
| timeout=120, |
| ) + |
| ExpectShell.log('json', stdout=json_with_failures) + |
| 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 30 webkitpy python3 test failures: test1, test2, test3, test4, test5, test6, test7, test8, test9, test10 ...') |
| return self.runStep() |
| |
| |
| class TestRunBuildbotCheckConfigForEWS(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunBuildbotCheckConfigForEWS()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport/ews-build', |
| timeout=120, |
| logEnviron=False, |
| command=['buildbot', 'checkconfig'], |
| env={'LC_CTYPE': 'en_US.UTF-8'} |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed buildbot checkconfig') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunBuildbotCheckConfigForEWS()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport/ews-build', |
| timeout=120, |
| logEnviron=False, |
| command=['buildbot', 'checkconfig'], |
| env={'LC_CTYPE': 'en_US.UTF-8'} |
| ) |
| + ExpectShell.log('stdio', stdout='Configuration Errors: builder(s) iOS-14-Debug-Build-EWS have no schedulers to drive them') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed buildbot checkconfig') |
| return self.runStep() |
| |
| |
| class TestRunBuildbotCheckConfigForBuildWebKit(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunBuildbotCheckConfigForBuildWebKit()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport/build-webkit-org', |
| timeout=120, |
| logEnviron=False, |
| command=['buildbot', 'checkconfig'], |
| env={'LC_CTYPE': 'en_US.UTF-8'} |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed buildbot checkconfig') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunBuildbotCheckConfigForBuildWebKit()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport/build-webkit-org', |
| timeout=120, |
| logEnviron=False, |
| command=['buildbot', 'checkconfig'], |
| env={'LC_CTYPE': 'en_US.UTF-8'} |
| ) |
| + ExpectShell.log('stdio', stdout='Configuration Errors: builder(s) Apple-iOS-14-Release-Build have no schedulers to drive them') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed buildbot checkconfig') |
| return self.runStep() |
| |
| |
| class TestRunEWSUnitTests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunEWSUnitTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'runUnittests.py', 'ews-build'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed EWS unit tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunEWSUnitTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'runUnittests.py', 'ews-build'], |
| ) |
| + ExpectShell.log('stdio', stdout='Unhandled Error. Traceback (most recent call last): Keys in cmd missing from expectation: [logfiles.json]') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed EWS unit tests') |
| return self.runStep() |
| |
| |
| class TestRunResultsdbpyTests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunResultsdbpyTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/libraries/resultsdbpy/resultsdbpy/run-tests', '--verbose', '--no-selenium', '--fast-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed resultsdbpy unit tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunResultsdbpyTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'Tools/Scripts/libraries/resultsdbpy/resultsdbpy/run-tests', '--verbose', '--no-selenium', '--fast-tests'], |
| ) |
| + ExpectShell.log('stdio', stdout='FAILED (errors=5, skipped=224)') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed resultsdbpy unit tests') |
| return self.runStep() |
| |
| |
| class TestRunBuildWebKitOrgUnitTests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunBuildWebKitOrgUnitTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'runUnittests.py', 'build-webkit-org'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed build.webkit.org unit tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunBuildWebKitOrgUnitTests()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='build/Tools/CISupport', |
| timeout=120, |
| logEnviron=False, |
| command=['python3', 'runUnittests.py', 'build-webkit-org'], |
| ) |
| + ExpectShell.log('stdio', stdout='Unhandled Error. Traceback (most recent call last): Keys in cmd missing from expectation: [logfiles.json]') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed build.webkit.org unit tests') |
| return self.runStep() |
| |
| |
| class TestKillOldProcesses(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(KillOldProcesses()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| command=['python3', 'Tools/CISupport/kill-old-processes', 'buildbot'], |
| logEnviron=False, |
| timeout=120, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Killed old processes') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(KillOldProcesses()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| command=['python3', 'Tools/CISupport/kill-old-processes', 'buildbot'], |
| logEnviron=False, |
| timeout=120, |
| ) |
| + ExpectShell.log('stdio', stdout='Unexpected error.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed to kill old processes') |
| return self.runStep() |
| |
| |
| class TestGitResetHard(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(GitResetHard()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| command=['git', 'reset', 'HEAD~10', '--hard'], |
| logEnviron=False, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Performed git reset --hard') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(GitResetHard()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| command=['git', 'reset', 'HEAD~10', '--hard'], |
| logEnviron=False, |
| ) |
| + ExpectShell.log('stdio', stdout='Unexpected error.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Performed git reset --hard (failure)') |
| return self.runStep() |
| |
| |
| class TestCleanBuild(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(CleanBuild()) |
| self.setProperty('fullPlatform', 'ios-11') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| command=['python3', 'Tools/CISupport/clean-build', '--platform=ios-11', '--release'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Deleted WebKitBuild directory') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(CleanBuild()) |
| self.setProperty('fullPlatform', 'ios-simulator-11') |
| self.setProperty('configuration', 'debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| command=['python3', 'Tools/CISupport/clean-build', '--platform=ios-simulator-11', '--debug'], |
| ) |
| + ExpectShell.log('stdio', stdout='Unexpected error.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Deleted WebKitBuild directory (failure)') |
| return self.runStep() |
| |
| |
| class TestCleanUpGitIndexLock(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(CleanUpGitIndexLock()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['rm', '-f', '.git/index.lock'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Deleted .git/index.lock') |
| return self.runStep() |
| |
| def test_success_windows(self): |
| self.setupStep(CleanUpGitIndexLock()) |
| self.setProperty('platform', 'win') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['rm', '-f', '.git/index.lock'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Deleted .git/index.lock') |
| return self.runStep() |
| |
| def test_success_wincairo(self): |
| self.setupStep(CleanUpGitIndexLock()) |
| self.setProperty('platform', 'wincairo') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['del', r'.git\index.lock'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Deleted .git/index.lock') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(CleanUpGitIndexLock()) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=120, |
| logEnviron=False, |
| command=['rm', '-f', '.git/index.lock'], |
| ) |
| + ExpectShell.log('stdio', stdout='Unexpected error.') |
| + 1, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Deleted .git/index.lock (failure)') |
| return self.runStep() |
| |
| |
| class TestInstallGtkDependencies(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(InstallGtkDependencies()) |
| self.setProperty('configuration', 'release') |
| self.assertEqual(InstallGtkDependencies.haltOnFailure, True) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/update-webkitgtk-libs', '--release'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Updated gtk dependencies') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(InstallGtkDependencies()) |
| self.setProperty('configuration', 'release') |
| self.assertEqual(InstallGtkDependencies.haltOnFailure, True) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/update-webkitgtk-libs', '--release'], |
| ) |
| + ExpectShell.log('stdio', stdout='Unexpected error.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Updated gtk dependencies (failure)') |
| return self.runStep() |
| |
| |
| class TestInstallWpeDependencies(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(InstallWpeDependencies()) |
| self.setProperty('configuration', 'release') |
| self.assertEqual(InstallWpeDependencies.haltOnFailure, True) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/update-webkitwpe-libs', '--release'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Updated wpe dependencies') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(InstallWpeDependencies()) |
| self.setProperty('configuration', 'release') |
| self.assertEqual(InstallWpeDependencies.haltOnFailure, True) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/update-webkitwpe-libs', '--release'], |
| ) |
| + ExpectShell.log('stdio', stdout='Unexpected error.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Updated wpe dependencies (failure)') |
| return self.runStep() |
| |
| |
| class TestCompileWebKit(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(CompileWebKit()) |
| self.setProperty('fullPlatform', 'ios-simulator-11') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-webkit', '--release'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Compiled WebKit') |
| return self.runStep() |
| |
| def test_success_gtk(self): |
| self.setupStep(CompileWebKit()) |
| self.setProperty('platform', 'gtk') |
| self.setProperty('fullPlatform', 'gtk') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-webkit', '--release', '--prefix=/app/webkit/WebKitBuild/release/install', '--gtk'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Compiled WebKit') |
| return self.runStep() |
| |
| def test_success_wpe(self): |
| self.setupStep(CompileWebKit()) |
| self.setProperty('platform', 'wpe') |
| self.setProperty('fullPlatform', 'wpe') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-webkit', '--release', '--wpe'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Compiled WebKit') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(CompileWebKit()) |
| self.setProperty('fullPlatform', 'mac-sierra') |
| self.setProperty('configuration', 'debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-webkit', '--debug'], |
| ) |
| + ExpectShell.log('stdio', stdout='1 error generated.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed to compile WebKit') |
| return self.runStep() |
| |
| def test_skip_for_revert_patches_on_commit_queue(self): |
| self.setupStep(CompileWebKit()) |
| self.setProperty('buildername', 'Commit-Queue') |
| self.setProperty('configuration', 'debug') |
| self.setProperty('fast_commit_queue', True) |
| self.expectOutcome(result=SKIPPED, state_string='Skipped compiling WebKit in fast-cq mode') |
| return self.runStep() |
| |
| |
| class TestCompileWebKitWithoutPatch(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(CompileWebKitWithoutPatch()) |
| self.setProperty('fullPlatform', 'ios-simulator-11') |
| self.setProperty('configuration', 'release') |
| self.setProperty('patchFailedToBuild', True) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-webkit', '--release'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Compiled WebKit') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(CompileWebKitWithoutPatch()) |
| self.setProperty('fullPlatform', 'mac-sierra') |
| self.setProperty('configuration', 'debug') |
| self.setProperty('patchFailedTests', True) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-webkit', '--debug'], |
| ) |
| + ExpectShell.log('stdio', stdout='1 error generated.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed to compile WebKit') |
| return self.runStep() |
| |
| def test_skip(self): |
| self.setupStep(CompileWebKitWithoutPatch()) |
| self.setProperty('fullPlatform', 'ios-simulator-11') |
| self.setProperty('configuration', 'release') |
| self.expectHidden(True) |
| self.expectOutcome(result=SKIPPED, state_string='Skipped compiling WebKit') |
| return self.runStep() |
| |
| |
| class TestAnalyzeCompileWebKitResults(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| AnalyzeCompileWebKitResults.send_email_for_build_failure = lambda self: None |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_patch_with_build_failure(self): |
| previous_steps = [ |
| mock_step(CompileWebKit(), results=FAILURE), |
| mock_step(CompileWebKitWithoutPatch(), results=SUCCESS), |
| ] |
| self.setupStep(AnalyzeCompileWebKitResults(), previous_steps=previous_steps) |
| self.setProperty('patch_id', '1234') |
| self.expectOutcome(result=FAILURE, state_string='Patch 1234 does not build (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('bugzilla_comment_text'), None) |
| self.assertEqual(self.getProperty('build_finish_summary'), 'Patch 1234 does not build') |
| return rc |
| |
| def test_patch_with_build_failure_on_commit_queue(self): |
| previous_steps = [ |
| mock_step(CompileWebKit(), results=FAILURE), |
| mock_step(CompileWebKitWithoutPatch(), results=SUCCESS), |
| ] |
| self.setupStep(AnalyzeCompileWebKitResults(), previous_steps=previous_steps) |
| self.setProperty('patch_id', '1234') |
| self.setProperty('buildername', 'commit-queue') |
| self.expectOutcome(result=FAILURE, state_string='Patch 1234 does not build (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('bugzilla_comment_text'), 'Patch 1234 does not build') |
| self.assertEqual(self.getProperty('build_finish_summary'), 'Patch 1234 does not build') |
| return rc |
| |
| def test_patch_with_trunk_failure(self): |
| previous_steps = [ |
| mock_step(CompileWebKit(), results=FAILURE), |
| mock_step(CompileWebKitWithoutPatch(), results=FAILURE), |
| ] |
| self.setupStep(AnalyzeCompileWebKitResults(), previous_steps=previous_steps) |
| self.expectOutcome(result=FAILURE, state_string='Unable to build WebKit without patch, retrying build (failure)') |
| return self.runStep() |
| |
| def test_filter_logs_containing_error(self): |
| logs = 'In file included from WebCore/unified-sources/UnifiedSource263.cpp:4:\nImageBufferIOSurfaceBackend.cpp:108:30: error: definition of implicitly declared destructor' |
| expected_output = 'ImageBufferIOSurfaceBackend.cpp:108:30: error: definition of implicitly declared destructor' |
| output = AnalyzeCompileWebKitResults().filter_logs_containing_error(logs) |
| self.assertEqual(expected_output, output) |
| |
| def test_filter_logs_containing_error_with_too_many_errors(self): |
| logs = 'Error:1\nError:2\nerror:3\nerror:4\nerror:5\nrandom-string\nerror:6\nerror:7\nerror8\nerror:9\nerror:10\nerror:11\nerror:12\nerror:13' |
| expected_output = 'error:3\nerror:4\nerror:5\nerror:6\nerror:7\nerror:9\nerror:10\nerror:11\nerror:12\nerror:13' |
| output = AnalyzeCompileWebKitResults().filter_logs_containing_error(logs) |
| self.assertEqual(expected_output, output) |
| |
| def test_filter_logs_containing_error_with_no_error(self): |
| logs = 'CompileC /Volumes/Data/worker/macOS-Catalina-Release-Build-EWS' |
| expected_output = '' |
| output = AnalyzeCompileWebKitResults().filter_logs_containing_error(logs) |
| self.assertEqual(expected_output, output) |
| |
| |
| class TestCompileJSC(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(CompileJSC()) |
| self.setProperty('fullPlatform', 'jsc-only') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-jsc', '--release'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Compiled JSC') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(CompileJSC()) |
| self.setProperty('fullPlatform', 'jsc-only') |
| self.setProperty('configuration', 'debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-jsc', '--debug'], |
| ) |
| + ExpectShell.log('stdio', stdout='1 error generated.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed to compile JSC') |
| return self.runStep() |
| |
| |
| class TestCompileJSCWithoutPatch(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(CompileJSCWithoutPatch()) |
| self.setProperty('fullPlatform', 'jsc-only') |
| self.setProperty('configuration', 'release') |
| self.setProperty('patchFailedToBuild', 'True') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-jsc', '--release'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Compiled JSC') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(CompileJSCWithoutPatch()) |
| self.setProperty('fullPlatform', 'jsc-only') |
| self.setProperty('configuration', 'debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/build-jsc', '--debug'], |
| ) |
| + ExpectShell.log('stdio', stdout='1 error generated.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Failed to compile JSC') |
| return self.runStep() |
| |
| |
| class TestRunJavaScriptCoreTests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'jsc_results.json' |
| self.jsc_masm_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":false,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":[],"allApiTestsPassed":true}\n''' |
| self.jsc_b3_and_stress_test_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":false,"allAirTestsPassed":true,"allApiTestsPassed":true,"stressTestFailures":["stress/weakset-gc.js"]}\n''' |
| self.jsc_dfg_air_and_stress_test_failure = '''{"allDFGTestsPassed":false,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":false,"allApiTestsPassed":true,"stressTestFailures":["stress/weakset-gc.js"]}\n''' |
| self.jsc_single_stress_test_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":["stress/switch-on-char-llint-rope.js.dfg-eager"],"allApiTestsPassed":true}\n''' |
| self.jsc_multiple_stress_test_failures = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":["stress/switch-on-char-llint-rope.js.dfg-eager","stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate","stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit","stress/switch-on-char-llint-rope.js.ftl-eager","stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit","stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit-b3o1","stress/switch-on-char-llint-rope.js.ftl-no-cjit-b3o0","stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-inline-validate","stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-put-stack-validate","stress/switch-on-char-llint-rope.js.ftl-no-cjit-small-pool","stress/switch-on-char-llint-rope.js.ftl-no-cjit-validate-sampling-profiler","stress/switch-on-char-llint-rope.js.no-cjit-collect-continuously","stress/switch-on-char-llint-rope.js.no-cjit-validate-phases","stress/switch-on-char-llint-rope.js.no-ftl"],"allApiTestsPassed":true}\n''' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self, platform=None, fullPlatform=None, configuration=None): |
| self.setupStep(RunJavaScriptCoreTests()) |
| self.prefix = RunJavaScriptCoreTests.prefix |
| if platform: |
| self.setProperty('platform', platform) |
| if fullPlatform: |
| self.setProperty('fullPlatform', fullPlatform) |
| if configuration: |
| self.setProperty('configuration', configuration) |
| |
| def test_success(self): |
| self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'], |
| logfiles={'json': self.jsonFileName}, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests') |
| return self.runStep() |
| |
| def test_remote_success(self): |
| self.configureStep(platform='jsc-only', fullPlatform='jsc-only', configuration='release') |
| self.setProperty('remotes', 'remote-machines.json') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release', '--remote-config-file=remote-machines.json', '--no-testmasm', '--no-testair', '--no-testb3', '--no-testdfg', '--no-testapi', '--memory-limited', '--verbose', '--jsc-only'], |
| logfiles={'json': self.jsonFileName}, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'], |
| logfiles={'json': self.jsonFileName}, |
| ) |
| + ExpectShell.log('stdio', stdout='9 failures found.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='jscore-tests (failure)') |
| return self.runStep() |
| |
| def test_single_stress_test_failure(self): |
| self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| logfiles={'json': self.jsonFileName}, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.jsc_single_stress_test_failure), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/switch-on-char-llint-rope.js.dfg-eager') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/switch-on-char-llint-rope.js.dfg-eager']) |
| self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), None) |
| return rc |
| |
| def test_lot_of_stress_test_failure(self): |
| self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| logfiles={'json': self.jsonFileName}, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.jsc_multiple_stress_test_failures), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 14 jsc stress test failures: stress/switch-on-char-llint-rope.js.dfg-eager, stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate, stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit, stress/switch-on-char-llint-rope.js.ftl-eager, stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit ...') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ["stress/switch-on-char-llint-rope.js.dfg-eager", "stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate", "stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit", "stress/switch-on-char-llint-rope.js.ftl-eager", "stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit", "stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit-b3o1", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-b3o0", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-inline-validate", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-put-stack-validate", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-small-pool", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-validate-sampling-profiler", "stress/switch-on-char-llint-rope.js.no-cjit-collect-continuously", "stress/switch-on-char-llint-rope.js.no-cjit-validate-phases", "stress/switch-on-char-llint-rope.js.no-ftl"]) |
| self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), None) |
| return rc |
| |
| def test_masm_failure(self): |
| self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| logfiles={'json': self.jsonFileName}, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.jsc_masm_failure), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='JSC test binary failure: testmasm') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), None) |
| self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testmasm']) |
| return rc |
| |
| def test_b3_and_stress_test_failure(self): |
| self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| logfiles={'json': self.jsonFileName}, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.jsc_b3_and_stress_test_failure), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/weakset-gc.js, JSC test binary failure: testb3') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/weakset-gc.js']) |
| self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testb3']) |
| return rc |
| |
| def test_dfg_air_and_stress_test_failure(self): |
| self.configureStep(platform='jsc-only', fullPlatform='jsc-only', configuration='release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| logfiles={'json': self.jsonFileName}, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release', '--memory-limited', '--verbose', '--jsc-only'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.jsc_dfg_air_and_stress_test_failure), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/weakset-gc.js, JSC test binary failures: testair, testdfg') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/weakset-gc.js']) |
| self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testair', 'testdfg']) |
| return rc |
| |
| |
| class TestReRunJavaScriptCoreTests(TestRunJavaScriptCoreTests): |
| def configureStep(self, platform=None, fullPlatform=None, configuration=None): |
| self.setupStep(ReRunJavaScriptCoreTests()) |
| self.prefix = ReRunJavaScriptCoreTests.prefix |
| if platform: |
| self.setProperty('platform', platform) |
| if fullPlatform: |
| self.setProperty('fullPlatform', fullPlatform) |
| if configuration: |
| self.setProperty('configuration', configuration) |
| |
| def test_success(self): |
| self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='release') |
| self.setProperty('jsc_stress_test_failures', ['test1', 'test2']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'], |
| logfiles={'json': self.jsonFileName}, |
| ) + |
| 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Found flaky tests: test1, test2') |
| return self.runStep() |
| |
| def test_remote_success(self): |
| self.configureStep(platform='jsc-only', fullPlatform='jsc-only', configuration='release') |
| self.setProperty('remotes', 'remote-machines.json') |
| self.setProperty('jsc_binary_failures', ['testmasm']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release', '--remote-config-file=remote-machines.json', '--no-testmasm', '--no-testair', '--no-testb3', '--no-testdfg', '--no-testapi', '--memory-limited', '--verbose', '--jsc-only'], |
| logfiles={'json': self.jsonFileName}, |
| ) + |
| 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Found flaky test: testmasm') |
| return self.runStep() |
| |
| |
| class TestRunJSCTestsWithoutPatch(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'jsc_results.json' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunJSCTestsWithoutPatch()) |
| self.setProperty('fullPlatform', 'jsc-only') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'], |
| logfiles={'json': self.jsonFileName}, |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='jscore-tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunJSCTestsWithoutPatch()) |
| self.setProperty('fullPlatform', 'jsc-only') |
| self.setProperty('configuration', 'debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logEnviron=False, |
| command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'], |
| logfiles={'json': self.jsonFileName}, |
| ) |
| + ExpectShell.log('stdio', stdout='9 failures found.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='jscore-tests (failure)') |
| return self.runStep() |
| |
| |
| class TestAnalyzeJSCTestsResults(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self): |
| self.setupStep(AnalyzeJSCTestsResults()) |
| self.setProperty('jsc_stress_test_failures', []) |
| self.setProperty('jsc_binary_failures', []) |
| self.setProperty('jsc_rerun_stress_test_failures', []) |
| self.setProperty('jsc_rerun_binary_failures', []) |
| self.setProperty('jsc_clean_tree_stress_test_failures', []) |
| self.setProperty('jsc_clean_tree_binary_failures', []) |
| AnalyzeJSCTestsResults.send_email_for_flaky_failure = lambda self, test: None |
| AnalyzeJSCTestsResults.send_email_for_pre_existing_failure = lambda self, test: None |
| |
| def test_single_new_stress_failure(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.bytecode-cache']) |
| self.setProperty('jsc_rerun_stress_test_failures', ['stress/force-error.js.bytecode-cache']) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC stress test failure: stress/force-error.js.bytecode-cache (failure)') |
| return self.runStep() |
| |
| def test_single_new_binary_failure(self): |
| self.configureStep() |
| self.setProperty('jsc_binary_failures', ['testmasm']) |
| self.setProperty('jsc_rerun_binary_failures', ['testmasm']) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC binary failure: testmasm (failure)') |
| return self.runStep() |
| |
| def test_multiple_new_stress_failure(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.setProperty('jsc_rerun_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ... (failure)') |
| return self.runStep() |
| |
| def test_multiple_new_binary_failure(self): |
| self.configureStep() |
| self.setProperty('jsc_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi']) |
| self.setProperty('jsc_rerun_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi']) |
| self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testair, testapi, testb3, testdfg, testmasm (failure)') |
| return self.runStep() |
| |
| def test_new_stress_and_binary_failure(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default']) |
| self.setProperty('jsc_binary_failures', ['testmasm']) |
| self.setProperty('jsc_rerun_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default']) |
| self.setProperty('jsc_rerun_binary_failures', ['testmasm']) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC binary failure: testmasm, Found 1 new JSC stress test failure: es6.yaml/es6/Set_iterator_closing.js.default (failure)') |
| return self.runStep() |
| |
| def test_stress_failure_on_clean_tree(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.default']) |
| self.setProperty('jsc_rerun_stress_test_failures', ['stress/force-error.js.default']) |
| self.setProperty('jsc_clean_tree_stress_test_failures', ['stress/force-error.js.default']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests') |
| return self.runStep() |
| |
| def test_binary_failure_on_clean_tree(self): |
| self.configureStep() |
| self.setProperty('jsc_binary_failures', ['testdfg']) |
| self.setProperty('jsc_rerun_binary_failures', ['testdfg']) |
| self.setProperty('jsc_clean_tree_binary_failures', ['testdfg']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests') |
| return self.runStep() |
| |
| def test_stress_and_binary_failure_on_clean_tree(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default']) |
| self.setProperty('jsc_binary_failures', ['testair']) |
| self.setProperty('jsc_rerun_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default']) |
| self.setProperty('jsc_rerun_binary_failures', ['testair']) |
| self.setProperty('jsc_clean_tree_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default']) |
| self.setProperty('jsc_clean_tree_binary_failures', ['testair']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests') |
| return self.runStep() |
| |
| def test_flaky_stress_and_binary_failures(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.default']) |
| self.setProperty('jsc_binary_failures', ['testapi']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests') |
| return self.runStep() |
| |
| def test_flaky_and_consistent_stress_failures(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['test1', 'test2']) |
| self.setProperty('jsc_rerun_stress_test_failures', ['test2']) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC stress test failure: test2 (failure)') |
| return self.runStep() |
| |
| def test_flaky_and_consistent_failures_with_clean_tree_failures(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', ['test1', 'test2']) |
| self.setProperty('jsc_rerun_stress_test_failures', ['test1']) |
| self.setProperty('jsc_clean_tree_stress_test_failures', ['test1', 'test2']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests') |
| return self.runStep() |
| |
| def test_unexpected_infra_issue(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', []) |
| self.setProperty('jsc_rerun_stress_test_failures', []) |
| self.setProperty('jsc_clean_tree_stress_test_failures', []) |
| self.setProperty('clean_tree_run_status', FAILURE) |
| self.expectOutcome(result=RETRY, state_string='Unexpected infrastructure issue, retrying build (retry)') |
| return self.runStep() |
| |
| def test_patch_breaking_jsc_test_suite(self): |
| self.configureStep() |
| self.setProperty('jsc_stress_test_failures', []) |
| self.setProperty('jsc_rerun_stress_test_failures', []) |
| self.setProperty('jsc_clean_tree_stress_test_failures', []) |
| self.setProperty('clean_tree_run_status', SUCCESS) |
| self.expectOutcome(result=FAILURE, state_string='Found unexpected failure with patch (failure)') |
| return self.runStep() |
| |
| |
| class TestRunWebKitTests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'layout-test-results/full_results.json' |
| self.results_json_regressions = '''ADD_RESULTS({"tests":{"imported":{"w3c":{"web-platform-tests":{"IndexedDB":{"interleaved-cursors-large.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}},"wasm":{"jsapi":{"interface.any.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"instance":{"constructor-bad-imports.any.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}},"global":{"constructor.any.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"constructor.any.worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"toString.any.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}},"constructor":{"instantiate-bad-imports.any.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"instantiate-bad-imports.any.worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}},"interface.any.worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}},"blink":{"storage":{"indexeddb":{"blob-valid-before-commit.html":{"report":"REGRESSION","expected":"PASS","actual":"TIMEOUT","has_stderr":true}}}}}},"skipped":23256,"num_regressions":10,"other_crashes":{},"interrupted":true,"num_missing":0,"layout_tests_dir":"/Volumes/Data/worker/iOS-12-Simulator-WK2-Tests-EWS/build/LayoutTests","version":4,"num_passes":32056,"pixel_tests_enabled":false,"date":"06:21AM on July 15, 2019","has_pretty_patch":true,"fixable":23267,"num_flaky":0,"uses_expectations_file":true}); |
| ''' |
| self.results_json_flakes = '''ADD_RESULTS({"tests":{"http":{"tests":{"workers":{"service":{"service-worker-resource-timing.https.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}},"xmlhttprequest":{"post-content-type-document.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS"}}}},"imported":{"blink":{"storage":{"indexeddb":{"blob-valid-before-commit.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}},"fast":{"text":{"international":{"repaint-glyph-bounds.html":{"report":"FLAKY","expected":"PASS","actual":"IMAGE PASS","reftest_type":["=="],"image_diff_percent":0.08}}}}}}},"skipped":13176,"num_regressions":0,"other_crashes":{},"interrupted":false,"num_missing":0,"layout_tests_dir":"/Volumes/Data/worker/iOS-12-Simulator-WK2-Tests-EWS/build/LayoutTests","version":4,"num_passes":42185,"pixel_tests_enabled":false,"date":"06:54AM on July 17, 2019","has_pretty_patch":true,"fixable":55356,"num_flaky":4,"uses_expectations_file":true}); |
| ''' |
| self.results_json_mix_flakes_and_regression = '''ADD_RESULTS({"tests":{"http":{"tests":{"IndexedDB":{"collect-IDB-objects.https.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"}},"xmlhttprequest":{"on-network-timeout-error-during-preflight.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS"}}}},"transitions":{"lengthsize-transition-to-from-auto.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS"}},"imported":{"blink":{"storage":{"indexeddb":{"blob-valid-before-commit.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}}}},"fast":{"text":{"font-weight-fallback.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true,"reftest_type":["=="]}},"scrolling":{"ios":{"reconcile-layer-position-recursive.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}},"skipped":13174,"num_regressions":1,"other_crashes":{},"interrupted":false,"num_missing":0,"layout_tests_dir":"/Volumes/Data/worker/iOS-12-Simulator-WK2-Tests-EWS/build/LayoutTests","version":4,"num_passes":42158,"pixel_tests_enabled":false,"date":"11:28AM on July 16, 2019","has_pretty_patch":true,"fixable":55329,"num_flaky":5,"uses_expectations_file":true}); |
| ''' |
| |
| self.results_json_with_newlines = '''ADD_RESULTS({"tests":{"http":{"tests":{"IndexedDB":{"collect-IDB-objects.https.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"}},"xmlhttprequest":{"on-network-timeout-error-during-preflight.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS"}}}},"transitions":{"lengthsize-trans |
| ition-to-from-auto.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS"}},"imported":{"blink":{"storage":{"indexeddb":{"blob-valid-before-commit.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}}}},"fast":{"text":{"font-weight-fallback.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true,"reftest_type":["=="]}},"scrolling":{"ios":{"reconcile-layer-position-recursive.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}},"skipped":13174,"num_regressions":1,"other_crashes":{},"interrupted":false,"num_missing":0,"layout_tests_dir":"/Volumes/Data/worker/iOS-12-Simulator-WK2-Tests-EWS/build/LayoutTes |
| ts","version":4,"num_passes":42158,"pixel_tests_enabled":false,"date":"11:28AM on July 16, 2019","has_pretty_patch":true,"fixable":55329,"num_flaky":5,"uses_expectations_file":true}); |
| ''' |
| |
| self.results_with_missing_results = '''ADD_RESULTS({"tests":{"http":{"wpt":{"css":{"css-highlight-api":{"highlight-image-expected-mismatched.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"},"highlight-image.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"}}}}}}, "interrupted":false}); |
| ''' |
| |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self): |
| self.setupStep(RunWebKitTests()) |
| self.property_exceed_failure_limit = 'first_results_exceed_failure_limit' |
| self.property_failures = 'first_run_failures' |
| |
| def test_success(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| return self.runStep() |
| |
| def test_warnings(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 0 |
| + ExpectShell.log('stdio', stdout='''Unexpected flakiness: timeouts (2) |
| imported/blink/storage/indexeddb/blob-valid-before-commit.html [ Timeout Pass ] |
| storage/indexeddb/modern/deleteindex-2.html [ Timeout Pass ]'''), |
| ) |
| self.expectOutcome(result=WARNINGS, state_string='2 flakes') |
| return self.runStep() |
| |
| def test_skip_for_revert_patches_on_commit_queue(self): |
| self.configureStep() |
| self.setProperty('buildername', 'Commit-Queue') |
| self.setProperty('fullPlatform', 'mac') |
| self.setProperty('configuration', 'debug') |
| self.setProperty('fast_commit_queue', True) |
| self.expectOutcome(result=SKIPPED, state_string='Skipped layout-tests in fast-cq mode') |
| return self.runStep() |
| |
| def test_skip_for_mac_wk2_passed_patch_on_commit_queue(self): |
| self.configureStep() |
| self.setProperty('patch_id', '1234') |
| self.setProperty('buildername', 'Commit-Queue') |
| self.setProperty('fullPlatform', 'mac') |
| self.setProperty('configuration', 'debug') |
| self.setProperty('passed_mac_wk2', True) |
| self.expectOutcome(result=SKIPPED, state_string='Skipped layout-tests') |
| return self.runStep() |
| |
| def test_parse_results_json_regression(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.results_json_regressions), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.property_exceed_failure_limit), True) |
| self.assertEqual(self.getProperty(self.property_failures), |
| ['imported/blink/storage/indexeddb/blob-valid-before-commit.html', |
| 'imported/w3c/web-platform-tests/IndexedDB/interleaved-cursors-large.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.worker.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.worker.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/global/toString.any.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/instance/constructor-bad-imports.any.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.html', |
| 'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.worker.html']) |
| return rc |
| |
| def test_parse_results_json_flakes(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 0 |
| + ExpectShell.log('json', stdout=self.results_json_flakes), |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.property_exceed_failure_limit), False) |
| self.assertEqual(self.getProperty(self.property_failures), []) |
| return rc |
| |
| def test_parse_results_json_flakes_and_regressions(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.results_json_mix_flakes_and_regression), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.property_exceed_failure_limit), False) |
| self.assertEqual(self.getProperty(self.property_failures), ['fast/scrolling/ios/reconcile-layer-position-recursive.html']) |
| return rc |
| |
| def test_parse_results_json_with_newlines(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.results_json_with_newlines), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.property_exceed_failure_limit), False) |
| self.assertEqual(self.getProperty(self.property_failures), ['fast/scrolling/ios/reconcile-layer-position-recursive.html']) |
| return rc |
| |
| def test_parse_results_json_with_missing_results(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 2 |
| + ExpectShell.log('json', stdout=self.results_with_missing_results), |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty(self.property_exceed_failure_limit), False) |
| self.assertEqual(self.getProperty(self.property_failures), |
| ['http/wpt/css/css-highlight-api/highlight-image-expected-mismatched.html', |
| 'http/wpt/css/css-highlight-api/highlight-image.html']) |
| return rc |
| |
| def test_unexpected_error(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'mac-highsierra') |
| self.setProperty('configuration', 'debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--debug', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + ExpectShell.log('stdio', stdout='Unexpected error.') |
| + 254, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + ExpectShell.log('stdio', stdout='9 failures found.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| return self.runStep() |
| |
| def test_success_wpt_import_bot(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('patch_author', 'webkit-wpt-import-bot@igalia.com') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', 'imported/w3c/web-platform-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| return self.runStep() |
| |
| |
| class TestReRunWebKitTests(TestRunWebKitTests): |
| def configureStep(self): |
| self.setupStep(ReRunWebKitTests()) |
| self.property_exceed_failure_limit = 'second_results_exceed_failure_limit' |
| self.property_failures = 'second_run_failures' |
| ReRunWebKitTests.send_email_for_flaky_failure = lambda self, test: None |
| |
| def test_flaky_failures_in_first_run(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('first_run_failures', ['test1', 'test2']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Found flaky tests: test1, test2') |
| return rc |
| |
| def test_first_run_failed_unexpectedly(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('first_run_failures', []) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Passed layout tests') |
| return rc |
| |
| |
| class TestRunWebKitTestsInStressMode(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'layout-test-results/full_results.json' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self): |
| self.setupStep(RunWebKitTestsInStressMode()) |
| self.property_exceed_failure_limit = 'first_results_exceed_failure_limit' |
| self.property_failures = 'first_run_failures' |
| |
| def test_success(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('modified_tests', ['test1', 'test2']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', |
| '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', |
| '--exit-after-n-failures', '10', '--skip-failing-tests', |
| '--iterations', 100, 'test1', 'test2'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('modified_tests', ['test']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', |
| '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', |
| '--exit-after-n-failures', '10', '--skip-failing-tests', |
| '--iterations', 100, 'test'], |
| ) |
| + ExpectShell.log('stdio', stdout='9 failures found.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Found test failures') |
| return rc |
| |
| |
| class TestRunWebKitTestsInStressGuardmallocMode(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'layout-test-results/full_results.json' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self): |
| self.setupStep(RunWebKitTestsInStressGuardmallocMode()) |
| self.property_exceed_failure_limit = 'first_results_exceed_failure_limit' |
| self.property_failures = 'first_run_failures' |
| |
| def test_success(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('modified_tests', ['test1', 'test2']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', |
| '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', |
| '--exit-after-n-failures', '10', '--skip-failing-tests', '--guard-malloc', |
| '--iterations', 100, 'test1', 'test2'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('modified_tests', ['test']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', |
| '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', |
| '--exit-after-n-failures', '10', '--skip-failing-tests', '--guard-malloc', |
| '--iterations', 100, 'test'], |
| ) |
| + ExpectShell.log('stdio', stdout='9 failures found.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Found test failures') |
| return rc |
| |
| |
| class TestRunWebKitTestsWithoutPatch(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'layout-test-results/full_results.json' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self): |
| self.setupStep(RunWebKitTestsWithoutPatch()) |
| self.property_exceed_failure_limit = 'clean_tree_results_exceed_failure_limit' |
| self.property_failures = 'clean_tree_run_failures' |
| self.setProperty('buildername', 'iOS-13-Simulator-WK2-Tests-EWS') |
| self.setProperty('buildnumber', '123') |
| self.setProperty('workername', 'ews126') |
| |
| def test_success(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', |
| '--no-show-results', |
| '--no-new-test-results', |
| '--clobber-old-results', |
| '--release', |
| '--results-directory', 'layout-test-results', |
| '--debug-rwt-logging', |
| '--exit-after-n-failures', '30', |
| '--skip-failing-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='layout-tests') |
| return self.runStep() |
| |
| def test_success_retry_only_subset(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('first_run_failures', ['test1', 'test2', 'test3']) |
| self.setProperty('second_run_failures', ['test1', 'test3', 'test4']) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', |
| '--no-show-results', |
| '--no-new-test-results', |
| '--clobber-old-results', |
| '--release', |
| '--results-directory', 'layout-test-results', |
| '--debug-rwt-logging', |
| '--exit-after-n-failures', '30', |
| '--skip-failing-tests', |
| 'test1', 'test2', 'test3', 'test4'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='layout-tests') |
| return self.runStep() |
| |
| def test_success_retry_only_subset_limit_exceeded(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('first_run_failures', ['test1', 'test2', 'test3']) |
| self.setProperty('second_results_exceed_failure_limit', True) |
| self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', |
| '--no-show-results', |
| '--no-new-test-results', |
| '--clobber-old-results', |
| '--release', |
| '--results-directory', 'layout-test-results', |
| '--debug-rwt-logging', |
| '--exit-after-n-failures', '30', |
| '--skip-failing-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='layout-tests') |
| return self.runStep() |
| |
| def test_success_retry_only_subset_patch_no_modifies_expectations(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('first_run_failures', ['test1', 'test2', 'test3']) |
| self.setProperty('second_run_failures', ['test1', 'test3', 'test4']) |
| RunWebKitTests._get_patch = lambda x: b'+++ Tools/ChangeLog\n+++ Tools/WebKitTestRunner/Options.cpp\n' |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', |
| '--no-show-results', |
| '--no-new-test-results', |
| '--clobber-old-results', |
| '--release', |
| '--results-directory', 'layout-test-results', |
| '--debug-rwt-logging', |
| '--exit-after-n-failures', '30', |
| '--skip-failing-tests', |
| 'test1', 'test2', 'test3', 'test4'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='layout-tests') |
| return self.runStep() |
| |
| def test_success_retry_only_subset_patch_modifies_expectations(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.setProperty('first_run_failures', ['test1', 'test2', 'test3']) |
| self.setProperty('second_run_failures', ['test1', 'test3', 'test4']) |
| RunWebKitTests._get_patch = lambda x: b'+++ LayoutTests/Changelog\n+++ LayoutTests/platform/gtk/TestExpectations\n' |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', |
| '--no-show-results', |
| '--no-new-test-results', |
| '--clobber-old-results', |
| '--release', |
| '--results-directory', 'layout-test-results', |
| '--debug-rwt-logging', |
| '--exit-after-n-failures', '30', |
| '--skip-failing-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='layout-tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.configureStep() |
| self.setProperty('fullPlatform', 'ios-simulator') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', |
| 'Tools/Scripts/run-webkit-tests', |
| '--no-build', |
| '--no-show-results', |
| '--no-new-test-results', |
| '--clobber-old-results', |
| '--release', |
| '--results-directory', 'layout-test-results', |
| '--debug-rwt-logging', |
| '--exit-after-n-failures', '30', |
| '--skip-failing-tests'], |
| ) |
| + ExpectShell.log('stdio', stdout='9 failures found.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| return self.runStep() |
| |
| |
| class TestRunWebKit1Tests(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| self.jsonFileName = 'layout-test-results/full_results.json' |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(RunWebKit1Tests()) |
| self.setProperty('fullPlatform', 'ios-11') |
| self.setProperty('configuration', 'debug') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--debug', '--dump-render-tree', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + 0, |
| ) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| return self.runStep() |
| |
| def test_failure(self): |
| self.setupStep(RunWebKit1Tests()) |
| self.setProperty('fullPlatform', 'ios-11') |
| self.setProperty('configuration', 'release') |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| logfiles={'json': self.jsonFileName}, |
| logEnviron=False, |
| command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--dump-render-tree', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'], |
| ) |
| + ExpectShell.log('stdio', stdout='9 failures found.') |
| + 2, |
| ) |
| self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') |
| return self.runStep() |
| |
| def test_skip_for_revert_patches_on_commit_queue(self): |
| self.setupStep(RunWebKit1Tests()) |
| self.setProperty('buildername', 'Commit-Queue') |
| self.setProperty('fullPlatform', 'mac') |
| self.setProperty('configuration', 'debug') |
| self.setProperty('fast_commit_queue', True) |
| self.expectOutcome(result=SKIPPED, state_string='Skipped layout-tests in fast-cq mode') |
| return self.runStep() |
| |
| |
| class TestAnalyzeLayoutTestsResults(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def configureStep(self): |
| AnalyzeLayoutTestsResults.send_email_for_flaky_failure = lambda self, test: None |
| AnalyzeLayoutTestsResults.send_email_for_pre_existing_failure = lambda self, test: None |
| self.setupStep(AnalyzeLayoutTestsResults()) |
| self.setProperty('first_results_exceed_failure_limit', False) |
| self.setProperty('second_results_exceed_failure_limit', False) |
| self.setProperty('clean_tree_results_exceed_failure_limit', False) |
| self.setProperty('clean_tree_run_failures', []) |
| |
| def test_failure_introduced_by_patch(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ["jquery/offset.html"]) |
| self.setProperty('second_run_failures', ["jquery/offset.html"]) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 new test failure: jquery/offset.html (failure)') |
| return self.runStep() |
| |
| def test_failure_on_clean_tree(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ["jquery/offset.html"]) |
| self.setProperty('second_run_failures', ["jquery/offset.html"]) |
| self.setProperty('clean_tree_run_failures', ["jquery/offset.html"]) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Found 1 pre-existing test failure: jquery/offset.html') |
| return rc |
| |
| def test_flaky_and_consistent_failures_without_clean_tree_failures(self): |
| self.configureStep() |
| self.setProperty('buildername', 'iOS-13-Simulator-WK2-Tests-EWS') |
| self.setProperty('first_run_failures', ['test1', 'test2']) |
| self.setProperty('second_run_failures', ['test1']) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 new test failure: test1 (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('bugzilla_comment_text'), None) |
| self.assertEqual(self.getProperty('build_finish_summary'), 'Found 1 new test failure: test1') |
| return rc |
| |
| def test_consistent_failure_without_clean_tree_failures_commit_queue(self): |
| self.configureStep() |
| self.setProperty('buildername', 'Commit-Queue') |
| self.setProperty('first_run_failures', ['test1']) |
| self.setProperty('second_run_failures', ['test1']) |
| self.expectOutcome(result=FAILURE, state_string='Found 1 new test failure: test1 (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('bugzilla_comment_text'), 'Found 1 new test failure: test1') |
| self.assertEqual(self.getProperty('build_finish_summary'), 'Found 1 new test failure: test1') |
| return rc |
| |
| def test_flaky_and_inconsistent_failures_without_clean_tree_failures(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ['test1', 'test2']) |
| self.setProperty('second_run_failures', ['test3']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| return self.runStep() |
| |
| def test_flaky_failures_in_first_run(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ['test1', 'test2']) |
| self.setProperty('second_run_failures', []) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), ' Found flaky tests: test1, test2') |
| return rc |
| |
| def test_flaky_and_inconsistent_failures_with_clean_tree_failures(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ['test1', 'test2']) |
| self.setProperty('second_run_failures', ['test3']) |
| self.setProperty('clean_tree_run_failures', ['test1', 'test2', 'test3']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Found 3 pre-existing test failures: test1, test2, test3 Found flaky tests: test1, test2, test3') |
| return rc |
| |
| def test_flaky_and_consistent_failures_with_clean_tree_failures(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ['test1', 'test2']) |
| self.setProperty('second_run_failures', ['test1']) |
| self.setProperty('clean_tree_run_failures', ['test1', 'test2']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| return self.runStep() |
| |
| def test_mildly_flaky_patch_with_some_tree_redness_and_flakiness(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ['test1', 'test2', 'test3']) |
| self.setProperty('second_run_failures', ['test1', 'test2']) |
| self.setProperty('clean_tree_run_failures', ['test1', 'test2', 'test4']) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Found 3 pre-existing test failures: test1, test2, test4 Found flaky test: test3') |
| return rc |
| |
| def test_first_run_exceed_failure_limit(self): |
| self.configureStep() |
| self.setProperty('first_results_exceed_failure_limit', True) |
| self.setProperty('first_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.setProperty('second_run_failures', []) |
| self.expectOutcome(result=RETRY, state_string='Unable to confirm if test failures are introduced by patch, retrying build (retry)') |
| return self.runStep() |
| |
| def test_second_run_exceed_failure_limit(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', []) |
| self.setProperty('second_results_exceed_failure_limit', True) |
| self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.expectOutcome(result=RETRY, state_string='Unable to confirm if test failures are introduced by patch, retrying build (retry)') |
| return self.runStep() |
| |
| def test_clean_tree_exceed_failure_limit(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ['test1']) |
| self.setProperty('second_run_failures', ['test1']) |
| self.setProperty('clean_tree_results_exceed_failure_limit', True) |
| self.setProperty('clean_tree_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.expectOutcome(result=RETRY, state_string='Unable to confirm if test failures are introduced by patch, retrying build (retry)') |
| return self.runStep() |
| |
| def test_clean_tree_exceed_failure_limit_with_triggered_by(self): |
| self.configureStep() |
| self.setProperty('buildername', 'iOS-13-Simulator-WK2-Tests-EWS') |
| self.setProperty('triggered_by', 'ios-13-sim-build-ews') |
| self.setProperty('first_run_failures', ['test1']) |
| self.setProperty('second_run_failures', ['test1']) |
| self.setProperty('clean_tree_results_exceed_failure_limit', True) |
| self.setProperty('clean_tree_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| message = 'Unable to confirm if test failures are introduced by patch, retrying build' |
| self.expectOutcome(result=SUCCESS, state_string=message) |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), message) |
| return rc |
| |
| def test_clean_tree_has_lot_of_failures(self): |
| self.configureStep() |
| self.setProperty('first_results_exceed_failure_limit', True) |
| self.setProperty('first_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.setProperty('second_results_exceed_failure_limit', True) |
| self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.setProperty('clean_tree_run_failures', ['test{}'.format(i) for i in range(0, 27)]) |
| self.expectOutcome(result=RETRY, state_string='Unable to confirm if test failures are introduced by patch, retrying build (retry)') |
| return self.runStep() |
| |
| def test_clean_tree_has_some_failures(self): |
| self.configureStep() |
| self.setProperty('first_results_exceed_failure_limit', True) |
| self.setProperty('first_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.setProperty('second_results_exceed_failure_limit', True) |
| self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 30)]) |
| self.setProperty('clean_tree_run_failures', ['test{}'.format(i) for i in range(0, 10)]) |
| self.expectOutcome(result=FAILURE, state_string='Found 30 new test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ... (failure)') |
| return self.runStep() |
| |
| def test_clean_tree_has_lot_of_failures_and_no_new_failure(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', ['test1']) |
| self.setProperty('second_run_failures', ['test1']) |
| self.setProperty('clean_tree_run_failures', ['test{}'.format(i) for i in range(0, 20)]) |
| self.expectOutcome(result=SUCCESS, state_string='Passed layout tests') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('build_summary'), 'Found 20 pre-existing test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ...') |
| return rc |
| |
| def test_patch_introduces_lot_of_failures(self): |
| self.configureStep() |
| self.setProperty('buildername', 'Commit-Queue') |
| self.setProperty('first_results_exceed_failure_limit', True) |
| self.setProperty('first_run_failures', ['test{}'.format(i) for i in range(0, 300)]) |
| self.setProperty('second_results_exceed_failure_limit', True) |
| self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 300)]) |
| failure_message = 'Found 300 new test failures: test0, test1, test10, test100, test101, test102, test103, test104, test105, test106 ...' |
| self.expectOutcome(result=FAILURE, state_string=failure_message + ' (failure)') |
| rc = self.runStep() |
| self.assertEqual(self.getProperty('bugzilla_comment_text'), failure_message) |
| self.assertEqual(self.getProperty('build_finish_summary'), failure_message) |
| return rc |
| |
| def test_unexpected_infra_issue(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', []) |
| self.setProperty('second_run_failures', []) |
| self.setProperty('clean_tree_run_status', FAILURE) |
| self.expectOutcome(result=RETRY, state_string='Unexpected infrastructure issue, retrying build (retry)') |
| return self.runStep() |
| |
| def test_patch_breaks_layout_tests(self): |
| self.configureStep() |
| self.setProperty('first_run_failures', []) |
| self.setProperty('second_run_failures', []) |
| self.setProperty('clean_tree_run_failures', []) |
| self.setProperty('clean_tree_run_status', SUCCESS) |
| self.expectOutcome(result=FAILURE, state_string='Found unexpected failure with patch (failure)') |
| return self.runStep() |
| |
| |
| class TestCheckOutSpecificRevision(BuildStepMixinAdditions, unittest.TestCase): |
| def setUp(self): |
| self.longMessage = True |
| return self.setUpBuildStep() |
| |
| def tearDown(self): |
| return self.tearDownBuildStep() |
| |
| def test_success(self): |
| self.setupStep(CheckOutSpecificRevision()) |
| self.setProperty('ews_revision', '1a3425cb92dbcbca12a10aa9514f1b77c76dc26') |
| self.expectHidden(False) |
| self.expectRemoteCommands( |
| ExpectShell(workdir='wkdir', |
| timeout=1200, |
| logEnviron=False, |
| command=['git', 'checkout', '1a3425cb92dbcbca12a10aa9514f1b77c76dc26'], |
| ) |
|
