blob: 1601133e1eae5e26800b56cb150ccdfaa0713910 [file] [log] [blame]
#! /usr/bin/env python
import sys
import os
import StringIO
import unittest
import make_passwords_json
import json
# Show DepricationWarnings come from buildbot - it isn't default with Python 2.7 or newer.
# See https://bugs.webkit.org/show_bug.cgi?id=90161 for details.
import warnings
warnings.simplefilter('default')
class BuildBotConfigLoader(object):
def _add_webkitpy_to_sys_path(self):
# When files are passed to the python interpreter on the command line (e.g. python test.py) __file__ is a relative path.
absolute_file_path = os.path.abspath(__file__)
webkit_org_config_dir = os.path.dirname(absolute_file_path)
build_slave_support_dir = os.path.dirname(webkit_org_config_dir)
webkit_tools_dir = os.path.dirname(build_slave_support_dir)
scripts_dir = os.path.join(webkit_tools_dir, 'Scripts')
sys.path.append(scripts_dir)
def _add_dependent_modules_to_sys_modules(self):
self._add_webkitpy_to_sys_path()
from webkitpy.thirdparty.autoinstalled import buildbot
sys.modules['buildbot'] = buildbot
class RunWebKitTestsTest(unittest.TestCase):
def test_nrwt_leaks_parsing(self):
run_webkit_tests = RunWebKitTests() # pylint is confused by the way we import the module ... pylint: disable-msg=E0602
log_text = """
12:44:24.295 77706 13981 total leaks found for a total of 197,936 bytes.
12:44:24.295 77706 1 unique leaks found.
"""
expected_incorrect_lines = [
'13981 total leaks found for a total of 197,936 bytes.',
'1 unique leaks found.',
]
run_webkit_tests._parseRunWebKitTestsOutput(log_text)
self.assertEqual(run_webkit_tests.incorrectLayoutLines, expected_incorrect_lines)
def test_nrwt_missing_results(self):
run_webkit_tests = RunWebKitTests() # pylint is confused by the way we import the module ... pylint: disable-msg=E0602
log_text = """
Expected to fail, but passed: (2)
animations/additive-transform-animations.html
animations/cross-fade-webkit-mask-box-image.html
Unexpected flakiness: text-only failures (2)
fast/events/touch/touch-inside-iframe.html [ Failure Pass ]
http/tests/inspector-enabled/console-clear-arguments-on-frame-navigation.html [ Failure Pass ]
Unexpected flakiness: timeouts (1)
svg/text/foreignObject-repaint.xml [ Timeout Pass ]
Regressions: Unexpected missing results (1)
svg/custom/zero-path-square-cap-rendering2.svg [ Missing ]
Regressions: Unexpected text-only failures (1)
svg/custom/zero-path-square-cap-rendering2.svg [ Failure ]
"""
run_webkit_tests._parseRunWebKitTestsOutput(log_text)
self.assertEqual(set(run_webkit_tests.incorrectLayoutLines),
set(['2 new passes', '3 flakes', '1 missing results', '1 failures']))
class StubStdio(object):
def __init__(self, stdio):
self._stdio = stdio
def getText(self):
return self._stdio
class StubRemoteCommand(object):
def __init__(self, rc, stdio):
self.rc = rc
self.logs = {'stdio': StubStdio(stdio)}
class RunJavaScriptCoreTestsTest(unittest.TestCase):
def assertResults(self, expected_result, expected_text, rc, stdio):
cmd = StubRemoteCommand(rc, stdio)
step = RunJavaScriptCoreTests()
step.commandComplete(cmd)
actual_results = step.evaluateCommand(cmd)
actual_text = step.getText2(cmd, actual_results)
self.assertEqual(expected_result, actual_results)
self.assertEqual(actual_text, expected_text)
def test_no_regressions_old_output(self):
self.assertResults(SUCCESS, ["jscore-test"], 0, """Results for Mozilla tests:
0 regressions found.
0 tests fixed.
OK.""")
def test_no_failure_new_output(self):
self.assertResults(SUCCESS, ["jscore-test"], 0, """Results for JSC stress tests:
0 failures found.
OK.""")
def test_mozilla_failure_old_output(self):
self.assertResults(FAILURE, ["1 JSC test failed"], 1, """Results for Mozilla tests:
1 regression found.
0 tests fixed.""")
def test_mozilla_failures_old_output(self):
self.assertResults(FAILURE, ["2 JSC tests failed"], 1, """Results for Mozilla tests:
2 regressions found.
0 tests fixed.""")
def test_jsc_stress_failure_new_output(self):
self.assertResults(FAILURE, ["1 JSC test failed"], 1, """Results for JSC stress tests:
1 failure found.""")
def test_jsc_stress_failures_new_output(self):
self.assertResults(FAILURE, ["5 JSC tests failed"], 1, """Results for JSC stress tests:
5 failures found.""")
def test_jsc_stress_failures_with_binary_results_output(self):
self.assertResults(FAILURE, ["8 JSC tests failed"], 1, """Results for JSC stress tests:
5 failures found.
Results for JSC test binaries:
3 failures found.""")
def test_jsc_stress_failures_with_binary_result_output(self):
self.assertResults(FAILURE, ["6 JSC tests failed"], 1, """Results for JSC stress tests:
5 failures found.
Results for JSC test binaries:
1 failure found.""")
class RunTest262TestsTest(unittest.TestCase):
def assertResults(self, expected_result, expected_text, rc, stdio):
cmd = StubRemoteCommand(rc, stdio)
step = RunTest262Tests()
step.commandComplete(cmd)
actual_results = step.evaluateCommand(cmd)
actual_text = step.getText2(cmd, actual_results)
self.assertEqual(expected_result, actual_results)
self.assertEqual(actual_text, expected_text)
def test_no_regressions_output(self):
self.assertResults(SUCCESS, ["test262-test"], 0, """
-------------------------Settings------------------------
Test262 Dir: JSTests/test262
JSC: WebKitBuild/Release/jsc
DYLD_FRAMEWORK_PATH: WebKitBuild/Release
Child Processes: 32
Config file: Tools/Scripts/test262/config.yaml
Expectations file: Tools/Scripts/test262/expectations.yaml
--------------------------------------------------------
NEW PASS: test/annexB/built-ins/Date/prototype/getYear/length.js (default)
NEW PASS test/language/expressions/class/fields-after-same-line-static-method-computed-symbol-names.js (default)
Run with --save to save a new expectations file
Saved all the results in Tools/Scripts/test262/results.yaml
Summarizing results...
See summarized results in Tools/Scripts/test262/results-summary.txt
56071 tests ran
0 expected tests failed
0 tests newly fail
2546 tests newly pass
1241 test files skipped
Done in 247 seconds!
""")
def test_failure_output(self):
self.assertResults(FAILURE, ["1 Test262 test failed"], 0, """
-------------------------Settings------------------------
Test262 Dir: JSTests/test262
JSC: WebKitBuild/Release/jsc
DYLD_FRAMEWORK_PATH: WebKitBuild/Release
Child Processes: 32
Config file: Tools/Scripts/test262/config.yaml
Expectations file: Tools/Scripts/test262/expectations.yaml
--------------------------------------------------------
! NEW FAIL: test/annexB/built-ins/Date/prototype/getYear/length.js (default)
NEW PASS test/language/expressions/class/fields-after-same-line-static-method-computed-symbol-names.js (default)
Run with --save to save a new expectations file
Saved all the results in Tools/Scripts/test262/results.yaml
Summarizing results...
See summarized results in Tools/Scripts/test262/results-summary.txt
56071 tests ran
0 expected tests failed
0 tests newly fail
2546 tests newly pass
1241 test files skipped
Done in 247 seconds!
""")
def test_failures_output(self):
self.assertResults(FAILURE, ["2 Test262 tests failed"], 0, """
-------------------------Settings------------------------
Test262 Dir: JSTests/test262
JSC: WebKitBuild/Release/jsc
DYLD_FRAMEWORK_PATH: WebKitBuild/Release
Child Processes: 32
Config file: Tools/Scripts/test262/config.yaml
Expectations file: Tools/Scripts/test262/expectations.yaml
--------------------------------------------------------
NEW PASS test/language/statements/class/fields-after-same-line-static-async-gen-computed-names.js (default)
! NEW FAIL: test/annexB/built-ins/Date/prototype/getYear/length.js (default)
! NEW FAIL: test/annexB/built-ins/Date/prototype/getYear/length.js (strict mode)
NEW PASS test/language/expressions/class/fields-after-same-line-static-method-computed-symbol-names.js (default)
Run with --save to save a new expectations file
Saved all the results in Tools/Scripts/test262/results.yaml
Summarizing results...
See summarized results in Tools/Scripts/test262/results-summary.txt
56071 tests ran
0 expected tests failed
0 tests newly fail
2546 tests newly pass
1241 test files skipped
Done in 247 seconds!
""")
class RunLLINTCLoopTestsTest(unittest.TestCase):
def assertResults(self, expected_result, expected_text, rc, stdio):
cmd = StubRemoteCommand(rc, stdio)
step = RunLLINTCLoopTests()
step.commandComplete(cmd)
actual_results = step.evaluateCommand(cmd)
actual_text = step.getText2(cmd, actual_results)
self.assertEqual(expected_result, actual_results)
self.assertEqual(actual_text, expected_text)
def test_failures(self):
self.assertResults(FAILURE, ['5 regressions found.'], 1, ' 5 regressions found.')
def test_failure(self):
self.assertResults(FAILURE, ['1 regression found.'], 1, ' 1 regression found.')
def test_no_failure(self):
self.assertResults(SUCCESS, ['webkit-jsc-cloop-test'], 0, ' 0 regressions found.')
class Run32bitJSCTestsTest(unittest.TestCase):
def assertResults(self, expected_result, expected_text, rc, stdio):
cmd = StubRemoteCommand(rc, stdio)
step = Run32bitJSCTests()
step.commandComplete(cmd)
actual_results = step.evaluateCommand(cmd)
actual_text = step.getText2(cmd, actual_results)
self.assertEqual(expected_result, actual_results)
self.assertEqual(actual_text, expected_text)
def test_failures(self):
self.assertResults(FAILURE, ['5 regressions found.'], 1, ' 5 failures found.')
def test_failure(self):
self.assertResults(FAILURE, ['1 regression found.'], 1, ' 1 failure found.')
def test_no_failure(self):
self.assertResults(SUCCESS, ['webkit-32bit-jsc-test'], 0, ' 0 failures found.')
class RunAPITestsTest(unittest.TestCase):
def assertFailures(self, expected_failure_count, stdio):
if expected_failure_count:
rc = 1
expected_results = FAILURE
plural_suffix = "" if expected_failure_count == 1 else "s"
expected_text = '%d api test%s failed or timed out' % (expected_failure_count, plural_suffix)
else:
rc = 0
expected_results = SUCCESS
expected_text = 'run-api-tests'
cmd = StubRemoteCommand(rc, stdio)
step = RunAPITests()
step.commandComplete(cmd)
actual_results = step.evaluateCommand(cmd)
actual_failure_count = step.failedTestCount
actual_text = step.getText(cmd, actual_results)[0]
self.assertEqual(expected_results, actual_results)
self.assertEqual(expected_failure_count, actual_failure_count)
self.assertEqual(expected_text, actual_text)
def test_no_failures_or_timeouts(self):
self.assertFailures(0, """...
worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
worker/0 TestWTF.WTF_Variant.Ref Passed
worker/0 TestWTF.WTF_Variant.RefPtr Passed
worker/0 TestWTF.WTF_Variant.RetainPtr Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
worker/0 exiting
Ran 1888 tests of 1888 with 1888 successful
------------------------------
All tests successfully passed!
""")
def test_no_failures_or_timeouts_with_disabled(self):
self.assertFailures(0, """...
worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
worker/0 TestWTF.WTF_Variant.Ref Passed
worker/0 TestWTF.WTF_Variant.RefPtr Passed
worker/0 TestWTF.WTF_Variant.RetainPtr Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
worker/0 exiting
Ran 1881 tests of 1888 with 1881 successful
------------------------------
All tests successfully passed!
""")
def test_one_failure(self):
self.assertFailures(1, """...
worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
worker/0 TestWTF.WTF_Variant.Ref Passed
worker/0 TestWTF.WTF_Variant.RefPtr Passed
worker/0 TestWTF.WTF_Variant.RetainPtr Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
worker/0 exiting
Ran 1888 tests of 1888 with 1887 successful
------------------------------
Test suite failed
Crashed
TestWTF.WTF.StringConcatenate_Unsigned
**FAIL** WTF.StringConcatenate_Unsigned
C:\\cygwin\\home\\buildbot\\slave\\win-release\\build\\Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
Value of: makeString("hello ", static_cast<unsigned short>(42) , " world")
Actual: hello 42 world
Expected: "hello * world"
Which is: 74B00C9C
Testing completed, Exit status: 3
""")
def test_multiple_failures(self):
self.assertFailures(2, """...
worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
worker/0 TestWTF.WTF_Variant.Ref Passed
worker/0 TestWTF.WTF_Variant.RefPtr Passed
worker/0 TestWTF.WTF_Variant.RetainPtr Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
worker/0 exiting
Ran 1888 tests of 1888 with 1886 successful
------------------------------
Test suite failed
Crashed
TestWTF.WTF.StringConcatenate_Unsigned
**FAIL** WTF.StringConcatenate_Unsigned
C:\\cygwin\\home\\buildbot\\slave\\win-release\\build\\Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
Value of: makeString("hello ", static_cast<unsigned short>(42) , " world")
Actual: hello 42 world
Expected: "hello * world"
Which is: 74B00C9C
TestWTF.WTF_Expected.Unexpected
**FAIL** WTF_Expected.Unexpected
C:\cygwin\home\buildbot\slave\win-release\build\Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
Value of: s1
Actual: oops
Expected: s0
Which is: oops
Testing completed, Exit status: 3
""")
def test_one_timeout(self):
self.assertFailures(1, """...
worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
worker/0 TestWTF.WTF_Variant.Ref Passed
worker/0 TestWTF.WTF_Variant.RefPtr Passed
worker/0 TestWTF.WTF_Variant.RetainPtr Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
worker/0 exiting
Ran 1888 tests of 1888 with 1887 successful
------------------------------
Test suite failed
Timeout
TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
Testing completed, Exit status: 3
""")
def test_multiple_timeouts(self):
self.assertFailures(2, """...
worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
worker/0 TestWTF.WTF_Variant.Ref Passed
worker/0 TestWTF.WTF_Variant.RefPtr Passed
worker/0 TestWTF.WTF_Variant.RetainPtr Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
worker/0 exiting
Ran 1888 tests of 1888 with 1886 successful
------------------------------
Test suite failed
Timeout
TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
TestWTF.WTF_Lock.ContendedShortSection
Testing completed, Exit status: 3
""")
def test_multiple_failures_and_timeouts(self):
self.assertFailures(4, """...
worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
worker/0 TestWTF.WTF_Variant.Ref Passed
worker/0 TestWTF.WTF_Variant.RefPtr Passed
worker/0 TestWTF.WTF_Variant.RetainPtr Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
worker/0 exiting
Ran 1888 tests of 1888 with 1884 successful
------------------------------
Test suite failed
Crashed
TestWTF.WTF.StringConcatenate_Unsigned
**FAIL** WTF.StringConcatenate_Unsigned
C:\\cygwin\\home\\buildbot\\slave\\win-release\\build\\Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
Value of: makeString("hello ", static_cast<unsigned short>(42) , " world")
Actual: hello 42 world
Expected: "hello * world"
Which is: 74B00C9C
TestWTF.WTF_Expected.Unexpected
**FAIL** WTF_Expected.Unexpected
C:\cygwin\home\buildbot\slave\win-release\build\Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
Value of: s1
Actual: oops
Expected: s0
Which is: oops
Timeout
TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
TestWTF.WTF_Lock.ContendedShortSection
Testing completed, Exit status: 3
""")
class SVNMirrorTest(unittest.TestCase):
def setUp(self):
self.config = json.load(open('config.json'))
def get_SVNMirrorFromConfig(self, builderName):
SVNMirror = None
for builder in self.config['builders']:
if builder['name'] == builderName:
SVNMirror = builder.pop('SVNMirror', 'https://svn.webkit.org/repository/webkit/')
return SVNMirror
def test_CheckOutSource(self):
# SVN mirror feature isn't unittestable now with source.oldsource.SVN(==source.SVN) , only with source.svn.SVN(==SVN)
# https://bugs.webkit.org/show_bug.cgi?id=85887
if issubclass(CheckOutSource, source.SVN):
return
# Compare CheckOutSource.baseURL with SVNMirror (or with the default URL) in config.json for all builders
for builder in c['builders']:
for buildStepFactory, kwargs in builder['factory'].steps:
if str(buildStepFactory).split('.')[-1] == 'CheckOutSource':
CheckOutSourceInstance = buildStepFactory(**kwargs)
self.assertEqual(CheckOutSourceInstance.baseURL, self.get_SVNMirrorFromConfig(builder['name']))
class BuildStepsConstructorTest(unittest.TestCase):
# "Passing a BuildStep subclass to factory.addStep is deprecated. Please pass a BuildStep instance instead. Support will be dropped in v0.8.7."
# It checks if all builder's all buildsteps can be insantiated after migration.
# https://bugs.webkit.org/show_bug.cgi?id=89001
# http://buildbot.net/buildbot/docs/0.8.6p1/manual/customization.html#writing-buildstep-constructors
@staticmethod
def generateTests():
for builderNumber, builder in enumerate(c['builders']):
for stepNumber, step in enumerate(builder['factory'].steps):
builderName = builder['name'].encode('ascii', 'ignore')
setattr(BuildStepsConstructorTest, 'test_builder%02d_step%02d' % (builderNumber, stepNumber), BuildStepsConstructorTest.createTest(builderName, step))
@staticmethod
def createTest(builderName, step):
def doTest(self):
try:
buildStepFactory, kwargs = step
buildStepFactory(**kwargs)
except TypeError as e:
buildStepName = str(buildStepFactory).split('.')[-1]
self.fail("Error during instantiation %s buildstep for %s builder: %s\n" % (buildStepName, builderName, e))
return doTest
expected_build_steps = {
'Apple Win 10 Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple Win 10 Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'download-built-product', 'extract-built-product', 'jscore-test', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple Win 10 Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple Win 10 Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple-Catalina-Debug-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple-Catalina-Debug-JSC-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
'Apple-Catalina-Debug-Test262-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],
'Apple-Catalina-Debug-WK1-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple-Catalina-Debug-WK2-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple-Catalina-Debug-WK2-WebGL-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple-Catalina-Debug-WK2-GPUProcess-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple-Catalina-LLINT-CLoop-BuildAndTest': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'webkit-jsc-cloop-test'],
'Apple-Catalina-Release-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple-Catalina-Release-JSC-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
'Apple-Catalina-Release-Test262-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'test262-test'],
'Apple-Catalina-Release-WK1-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple-Catalina-Release-WK2-Tests': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple-Catalina-Release-WK2-Perf': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'perf-test'],
'Apple Mojave Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple Mojave Debug WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple Mojave Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple Mojave Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple Mojave Release WK1 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple Mojave Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'lldb-webkit-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple Mojave Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple iOS 13 Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'Apple iOS 13 Simulator Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple iOS 13 Simulator Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple iOS 13 Simulator Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'Apple iOS 13 Simulator Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple iPadOS 13 Simulator Release WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple iPadOS 13 Simulator Debug WK2 (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wait-for-crash-collection', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'trigger-crash-log-submission', 'archive-test-results', 'upload', 'MasterShellCommand'],
'Apple-tvOS-13-Release-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'Apple-tvOS-Simulator-13-Release-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'Apple-watchOS-6-Release-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'Apple-watchOS-Simulator-6-Release-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'JSCOnly Linux ARMv7 Thumb2 Release': ['configure build', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
'JSCOnly Linux ARMv7 Thumb2 SoftFP Release': ['configure build', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
'JSCOnly Linux AArch64 Release': ['configure build', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
'JSCOnly Linux MIPS32el Release': ['configure build', 'svn', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'jscore-test'],
'GTK Linux 64-bit Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'GTK Linux 64-bit Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
'GTK Linux 64-bit Debug (Tests WebDriver)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'webdriver-test'],
'GTK Linux 64-bit Debug (Tests JS)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'test262-test'],
'GTK Linux 64-bit Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'generate-jsc-bundle', 'transfer-to-s3', 'trigger'],
'GTK Linux 64-bit Release (Perf)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'perf-test', 'benchmark-test'],
'GTK Linux 64-bit Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests'],
'GTK Linux 64-bit Release (Tests WebDriver)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'webdriver-test'],
'GTK Linux 64-bit Release (Tests JS)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'test262-test'],
'GTK Linux 64-bit Release Wayland (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests', 'webdriver-test'],
'GTK Linux 64-bit Release Ubuntu LTS (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'GTK Linux 64-bit Release Debian Stable (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'PlayStation-Debug-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'PlayStation-Release-Build': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit'],
'WinCairo 64-bit JSC Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
'WinCairo 64-bit JSC Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'jscore-test'],
'WinCairo 64-bit WKL Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'WinCairo 64-bit WKL Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wincairo-requirements', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
'WinCairo 64-bit WKL Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'WinCairo 64-bit WKL Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'download-built-product', 'extract-built-product', 'wincairo-requirements', 'layout-test', 'run-api-tests', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'archive-test-results', 'upload', 'MasterShellCommand'],
'WPE Linux 64-bit Release (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'WPE Linux 64-bit Release (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests', 'webdriver-test'],
'WPE Linux 64-bit Release (Tests JS)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'test262-test'],
'WPE Linux 64-bit Debug (Build)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'compile-webkit', 'archive-built-product', 'upload', 'transfer-to-s3', 'trigger'],
'WPE Linux 64-bit Debug (Tests)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'layout-test', 'webkitpy-test', 'webkitperl-test', 'bindings-generation-tests', 'builtins-generator-tests', 'dashboard-tests', 'archive-test-results', 'upload', 'MasterShellCommand', 'API tests', 'webdriver-test'],
'WPE Linux 64-bit Debug (Tests JS)': ['configure build', 'svn', 'kill old processes', 'delete WebKitBuild directory', 'delete stale build files', 'jhbuild', 'download-built-product', 'extract-built-product', 'jscore-test', 'test262-test'],
}
class BuildStepsTest(unittest.TestCase):
@staticmethod
def generateTests():
for builder in c['builders']:
builderName = builder['name'].encode('ascii', 'ignore')
setattr(BuildStepsTest, 'test_builder %s' % builderName, BuildStepsTest.createTest(builder))
@staticmethod
def createTest(builder):
def doTest(self):
buildSteps = []
for step in builder['factory'].steps:
buildSteps.append(step[0].name)
self.assertTrue(builder['name'] in expected_build_steps, "Missing expected result for builder: %s\n Actual result is %s" % (builder['name'], buildSteps))
self.assertListEqual(expected_build_steps[builder['name']], buildSteps)
return doTest
def test_unnecessary_expected_results(self):
builders = set()
for builder in c['builders']:
builders.add(builder['name'].encode('ascii', 'ignore'))
for builder in expected_build_steps:
self.assertTrue(builder in builders, "Builder %s doesn't exist, but has unnecessary expected results" % builder)
class RunAndUploadPerfTestsTest(unittest.TestCase):
def assertResults(self, rc, expected_text):
cmd = StubRemoteCommand(rc, expected_text)
step = RunAndUploadPerfTests()
step.commandComplete(cmd)
actual_results = step.evaluateCommand(cmd)
actual_text = str(step.getText2(cmd, actual_results)[0])
self.assertEqual(expected_text, actual_text)
def test_success(self):
self.assertResults(0, "perf-test")
def test_tests_failed(self):
self.assertResults(5, "5 perf tests failed")
def test_build_bad_build(self):
self.assertResults(255, "build not up to date")
def test_build_bad_source_json(self):
self.assertResults(254, "slave config JSON error")
def test_build_bad_marge(self):
self.assertResults(253, "output JSON merge error")
def test_build_bad_failed_uploading(self):
self.assertResults(252, "upload error")
def test_build_bad_preparation(self):
self.assertResults(251, "system dependency error")
def test_buildbot_timeout(self):
self.assertResults(-1, "timeout")
class RunBenchmarkTest(unittest.TestCase):
def assertResults(self, rc, expected_text):
cmd = StubRemoteCommand(rc, expected_text)
step = RunBenchmarkTests()
step.commandComplete(cmd)
actual_results = step.evaluateCommand(cmd)
actual_text = str(step.getText2(cmd, actual_results)[0])
self.assertEqual(expected_text, actual_text)
def test_success(self):
self.assertResults(0, "benchmark-test")
def test_tests_failed(self):
self.assertResults(7, "7 benchmark tests failed")
# FIXME: We should run this file as part of test-webkitpy.
# Unfortunately test-webkitpy currently requires that unittests
# be located in a directory with a valid module name.
# 'build.webkit.org-config' is not a valid module name (due to '.' and '-')
# so for now this is a stand-alone test harness.
if __name__ == '__main__':
BuildBotConfigLoader()._add_dependent_modules_to_sys_modules()
from loadConfig import *
c = {}
loadBuilderConfig(c, test_mode_is_enabled=True)
BuildStepsConstructorTest.generateTests()
BuildStepsTest.generateTests()
unittest.main()