| # Copyright (C) 2018 Igalia S.L. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY |
| # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON |
| # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import unittest |
| |
| from webkitpy.common.config.ports_mock import MockPort |
| from webkitpy.common.host_mock import MockHost |
| from webkitpy.w3c.wpt_runner import WPTRunner, parse_args |
| |
| |
| TEST_EXPECTATIONS_JSON_CONTENT = """{ |
| "passing_test.html": { "expected": "PASS" }, |
| "failing_test.html": { "expected": "FAIL" }, |
| "disabled_test.html": { "disabled": "Test Name #1" }, |
| "custom_test_name.html": { |
| "test_name": "Custom Test Name", |
| "expected": "FAIL" |
| }, |
| "test_with_subtests.html": { |
| "subtests": { |
| "Subtest #1": { "expected": "PASS" }, |
| "Subtest #2": { "expected": "TIMEOUT" } |
| } |
| }, |
| "nested/test_file.html": { "expected": "PASS" }, |
| "nested/nested/test_file.html": { |
| "test_name": "Deeper-nested test case", |
| "subtests": { |
| "Only Subtest": { "expected": "PASS" } |
| } |
| } |
| }""" |
| |
| EXPECTED_TEST_MANIFESTS = { |
| "passing_test.html.ini": |
| """[passing_test.html] |
| expected: PASS |
| """, |
| |
| "failing_test.html.ini": |
| """[failing_test.html] |
| expected: FAIL |
| """, |
| |
| "disabled_test.html.ini": |
| """[disabled_test.html] |
| disabled: Test Name #1 |
| """, |
| |
| "custom_test_name.html.ini": |
| """[Custom Test Name] |
| expected: FAIL |
| """, |
| |
| "test_with_subtests.html.ini": |
| """[test_with_subtests.html] |
| [Subtest #1] |
| expected: PASS |
| [Subtest #2] |
| expected: TIMEOUT |
| """, |
| |
| "nested/test_file.html.ini": |
| """[test_file.html] |
| expected: PASS |
| """, |
| |
| "nested/nested/test_file.html.ini": |
| """[Deeper-nested test case] |
| [Only Subtest] |
| expected: PASS |
| """ |
| } |
| |
| |
| class WPTRunnerTest(unittest.TestCase): |
| |
| class MockTestDownloader(object): |
| @staticmethod |
| def default_options(): |
| return {} |
| |
| def __init__(self, repository_directory, host, options): |
| self._repository_directory = repository_directory |
| self._host = host |
| |
| def clone_tests(self): |
| self._host.filesystem.maybe_make_directory(self._repository_directory, "web-platform-tests") |
| |
| class MockWebDriver(object): |
| @staticmethod |
| def create(port): |
| return WPTRunnerTest.MockWebDriver() |
| |
| def binary_path(self): |
| return "/mock-webdriver/bin/webdriver" |
| |
| def browser_path(self): |
| return "/mock-webdriver/bin/browser" |
| |
| def browser_args(self): |
| return ["webdriver_arg1", "webdriver_arg2"] |
| |
| class MockSpawnWPT(object): |
| def __init__(self, test_case, expected_wpt_checkout=None, expected_wpt_args=None): |
| self._test_case = test_case |
| self._expected_wpt_checkout = expected_wpt_checkout |
| self._expected_wpt_args = expected_wpt_args |
| |
| def __call__(self, script_name, wpt_checkout, wpt_args): |
| self._test_case.assertEquals(script_name, "wptrunner_unittest") |
| self._test_case.assertEquals(wpt_checkout, self._expected_wpt_checkout) |
| self._test_case.assertEquals(wpt_args, self._expected_wpt_args) |
| |
| class TestInstance(object): |
| def __init__(self, options, spawn_wpt_func=None): |
| self.port = MockPort() |
| self.host = MockHost() |
| |
| # In non-test environments, this value is by default set to the WEBKIT_TEST_CHILD_PROCESSES |
| # env value or, if that's not present, to the default number of child processes. Here we |
| # manually set it to 4 for consistency, unless the test case specifies it. |
| if not options.child_processes: |
| options.child_processes = 4 |
| |
| self.runner = WPTRunner(self.port, self.host, "wptrunner_unittest", options, |
| WPTRunnerTest.MockTestDownloader, WPTRunnerTest.MockWebDriver.create, spawn_wpt_func) |
| |
| def prepare_mock_files_for_run(self): |
| self.host.filesystem.maybe_make_directory(self.port.wpt_metadata_directory()) |
| self.host.filesystem.write_text_file( |
| "/mock-checkout/WebPlatformTests/MockPort/TestExpectations.json", "{}") |
| self.host.filesystem.write_text_file( |
| "/mock-checkout/WebPlatformTests/MockPort/TestManifest.ini", "{}") |
| |
| def test_prepare_wpt_checkout(self): |
| # Tests the prepare_wpt_checkout() method with no WPT checkout specified in options. |
| |
| options, _ = parse_args([]) |
| instance = WPTRunnerTest.TestInstance(options) |
| |
| self.assertTrue(instance.runner.prepare_wpt_checkout()) |
| |
| expected_wpt_checkout = "/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests" |
| self.assertEquals(instance.runner._options.wpt_checkout, expected_wpt_checkout) |
| self.assertTrue(instance.host.filesystem.isdir(expected_wpt_checkout)) |
| |
| def test_prepare_wpt_checkout_specified_path(self): |
| # Tests the prepare_wpt_checkout() method with WPT checkout specified in options. |
| |
| specified_wpt_checkout = "/mock-path/web-platform-tests" |
| options, _ = parse_args(["--wpt-checkout", specified_wpt_checkout]) |
| instance = WPTRunnerTest.TestInstance(options) |
| instance.host.filesystem.maybe_make_directory(specified_wpt_checkout) |
| |
| self.assertTrue(instance.runner.prepare_wpt_checkout()) |
| self.assertEquals(instance.runner._options.wpt_checkout, specified_wpt_checkout) |
| |
| def test_generate_metadata_directory(self): |
| # Tests the _generate_metadata_directory() method. TestExpectations.json file is mocked with |
| # TEST_EXPECTATIONS_JSON_CONTENT value as contents. Expected metadata is generated under |
| # /mock-metadata and should correspond to files and files' content in the |
| # EXPECTED_TEST_MANIFESTS dictionary. |
| |
| expectations_json_path = "/mock-checkout/WebPlatformTests/MockPort/TestExpectations.json" |
| metadata_path = "/mock-metadata" |
| |
| options, _ = parse_args([]) |
| instance = WPTRunnerTest.TestInstance(options) |
| |
| instance.host.filesystem.write_text_file( |
| "/mock-checkout/WebPlatformTests/MockPort/TestExpectations.json", |
| TEST_EXPECTATIONS_JSON_CONTENT) |
| |
| # Specify a stale metadata file that should be removed before generation. |
| stale_metadata_file = "/mock-metadata/test/stale_metadata_file.html.ini" |
| instance.host.filesystem.write_text_file( |
| stale_metadata_file, "[This file should be removed during generation]") |
| |
| self.assertTrue(instance.runner._generate_metadata_directory(metadata_path)) |
| |
| # Check that the stale metadata file was indeed removed. |
| self.assertFalse(instance.host.filesystem.exists(stale_metadata_file)) |
| |
| for path, content in EXPECTED_TEST_MANIFESTS.items(): |
| manifest_path = instance.host.filesystem.join(metadata_path, path) |
| self.assertTrue(instance.host.filesystem.isfile(manifest_path)) |
| self.assertEquals(instance.host.filesystem.read_text_file(manifest_path), content) |
| |
| def test_run(self): |
| # Tests the run() method. Files are mocked to the point that helper methods don't fail. |
| # Goal of this test is for the WPT spawn command to match the desired WPT directory and |
| # arguments. No options or arguments are used. |
| |
| spawn_wpt_func = WPTRunnerTest.MockSpawnWPT(self, |
| "/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests", |
| ["run", "--webkit-port=MockPort", "--processes=4", |
| "--metadata=/mock-path/mock-wpt-tests-metadata", |
| "--manifest=/mock-path/mock-wpt-manifest.json", |
| "--include-manifest=/mock-checkout/WebPlatformTests/MockPort/TestManifest.ini", |
| "--webdriver-binary=/mock-webdriver/bin/webdriver", |
| "--binary=/mock-webdriver/bin/browser", |
| "--binary-arg=webdriver_arg1", "--binary-arg=webdriver_arg2", "webkit"]) |
| |
| options, _ = parse_args([]) |
| instance = WPTRunnerTest.TestInstance(options, spawn_wpt_func) |
| instance.prepare_mock_files_for_run() |
| |
| self.assertTrue(instance.runner.run([])) |
| |
| def test_run_with_specified_options(self): |
| # Tests the run() method. Files are mocked to the point that helper methods don't fail. |
| # Goal of this test is for the WPT spawn command to match the desired WPT directory and |
| # arguments. Custom WPT checkout and child process count are specified. Note that the |
| # WPT checkout doesn't have an impact on the resulting WPT argument list, as intended. |
| specified_wpt_checkout = "/mock-path/web-platform-tests" |
| specified_wpt_metadata = "/mock-path/wpt-metadata" |
| specified_child_processes = 16 |
| |
| spawn_wpt_func = WPTRunnerTest.MockSpawnWPT(self, specified_wpt_checkout, |
| ["run", "--webkit-port=MockPort", "--processes=16", |
| "--metadata=/mock-path/wpt-metadata", |
| "--manifest=/mock-path/wpt-metadata/MANIFEST.json", |
| "--include-manifest=/mock-path/wpt-metadata/MockPort/TestManifest.ini", |
| "--webdriver-binary=/mock-webdriver/bin/webdriver", |
| "--binary=/mock-webdriver/bin/browser", |
| "--binary-arg=webdriver_arg1", "--binary-arg=webdriver_arg2", "webkit"]) |
| |
| options, _ = parse_args(["--wpt-checkout", specified_wpt_checkout, |
| "--wpt-metadata", specified_wpt_metadata, |
| "--child-processes", specified_child_processes]) |
| instance = WPTRunnerTest.TestInstance(options, spawn_wpt_func) |
| instance.prepare_mock_files_for_run() |
| |
| # Also create the mock WPT checkout and metadata directories. |
| instance.host.filesystem.maybe_make_directory(specified_wpt_checkout) |
| instance.host.filesystem.maybe_make_directory(specified_wpt_metadata) |
| |
| self.assertTrue(instance.runner.run([])) |
| |
| def test_run_with_args(self): |
| # Tests the run() method. Files are mocked to the point that helper methods don't fail. |
| # Goal of this test is for the WPT spawn command to match the desired WPT directory and |
| # arguments. A custom two-element argument list is used. It's expected to be appended |
| # to the resulting WPT argument list. |
| |
| specified_args = ["test1.html", "test2.html"] |
| |
| spawn_wpt_func = WPTRunnerTest.MockSpawnWPT(self, |
| "/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests", |
| ["run", "--webkit-port=MockPort", "--processes=4", |
| "--metadata=/mock-path/mock-wpt-tests-metadata", |
| "--manifest=/mock-path/mock-wpt-manifest.json", |
| "--include-manifest=/mock-checkout/WebPlatformTests/MockPort/TestManifest.ini", |
| "--webdriver-binary=/mock-webdriver/bin/webdriver", |
| "--binary=/mock-webdriver/bin/browser", |
| "--binary-arg=webdriver_arg1", "--binary-arg=webdriver_arg2", "webkit"] + specified_args) |
| |
| options, _ = parse_args([]) |
| instance = WPTRunnerTest.TestInstance(options, spawn_wpt_func) |
| instance.prepare_mock_files_for_run() |
| |
| self.assertTrue(instance.runner.run(specified_args)) |
