| # Copyright (C) 2010 Google Inc. All rights reserved. |
| # Copyright (C) 2013-2019 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the Google name nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """Abstract base class of Port-specific entry points for the layout tests |
| test infrastructure (the Port and Driver classes).""" |
| |
| import difflib |
| import json |
| import logging |
| import os |
| import optparse |
| import re |
| import sys |
| |
| from collections import OrderedDict |
| from functools import partial |
| |
| from webkitpy.common import find_files |
| from webkitpy.common import read_checksum_from_png |
| from webkitpy.common.checkout.scm.detection import SCMDetector |
| from webkitpy.common.memoized import memoized |
| from webkitpy.common.prettypatch import PrettyPatch |
| from webkitpy.common.system import path, pemfile |
| from webkitpy.common.system.executive import ScriptError |
| from webkitpy.common.version_name_map import PUBLIC_TABLE, INTERNAL_TABLE, VersionNameMap |
| from webkitpy.common.wavediff import WaveDiff |
| from webkitpy.common.webkit_finder import WebKitFinder |
| from webkitpy.common.unicode_compatibility import encode_for, encode_if_necessary, decode_for |
| from webkitpy.layout_tests.models.test_configuration import TestConfiguration |
| from webkitpy.port import config as port_config |
| from webkitpy.port import driver |
| from webkitpy.port import image_diff |
| from webkitpy.port import server_process |
| from webkitpy.port.factory import PortFactory |
| from webkitpy.layout_tests.servers import apache_http_server, http_server, http_server_base |
| from webkitpy.layout_tests.servers import web_platform_test_server |
| from webkitpy.layout_tests.servers import websocket_server |
| |
| _log = logging.getLogger(__name__) |
| |
| |
| class Port(object): |
| """Abstract class for Port-specific hooks for the layout_test package.""" |
| |
| # Subclasses override this. This should indicate the basic implementation |
| # part of the port name, e.g., 'win', 'gtk'; there is probably (?) one unique value per class. |
| |
| # FIXME: We should probably rename this to something like 'implementation_name'. |
| port_name = None |
| |
| # Test names resemble unix relative paths, and use '/' as a directory separator. |
| TEST_PATH_SEPARATOR = '/' |
| |
| ALL_BUILD_TYPES = ('debug', 'release') |
| |
| DEFAULT_ARCHITECTURE = 'x86' |
| DEVICE_TYPE = None |
| DEFAULT_DEVICE_TYPES = [] |
| |
| @classmethod |
| def determine_full_port_name(cls, host, options, port_name): |
| """Return a fully-specified port name that can be used to construct objects.""" |
| # Subclasses will usually override this. |
| options = options or {} |
| assert port_name.startswith(cls.port_name) |
| if getattr(options, 'webkit_test_runner', False) and not '-wk2' in port_name: |
| return port_name + '-wk2' |
| return port_name |
| |
| def __init__(self, host, port_name, options=None, **kwargs): |
| |
| # This value may be different from cls.port_name by having version modifiers |
| # and other fields appended to it (for example, 'mac-wk2' or 'win'). |
| self._name = port_name |
| |
| # These are default values that should be overridden in a subclasses. |
| self._os_version = None |
| |
| # FIXME: This can be removed once default architectures for GTK and EFL EWS bots are set. |
| self.did_override_architecture = False |
| |
| # FIXME: Ideally we'd have a package-wide way to get a |
| # well-formed options object that had all of the necessary |
| # options defined on it. |
| self._options = options or optparse.Values() |
| |
| if self.get_option('architecture'): |
| self.did_override_architecture = True |
| else: |
| self.set_option('architecture', self.DEFAULT_ARCHITECTURE) |
| |
| if self._name and '-wk2' in self._name: |
| self._options.webkit_test_runner = True |
| |
| self.host = host |
| self._executive = host.executive |
| self._filesystem = host.filesystem |
| self._webkit_finder = WebKitFinder(host.filesystem) |
| self._config = port_config.Config(self._executive, self._filesystem, self.port_name) |
| self.pretty_patch = PrettyPatch(self._executive, self.path_from_webkit_base(), self._filesystem) |
| |
| self._helper = None |
| self._http_server = None |
| self._websocket_server = None |
| self._websocket_secure_server = None |
| self._web_platform_test_server = None |
| self._image_differ = None |
| self._server_process_constructor = server_process.ServerProcess # overridable for testing |
| self._test_runner_process_constructor = server_process.ServerProcess |
| |
| if not hasattr(options, 'configuration') or not options.configuration: |
| self.set_option_default('configuration', self.default_configuration()) |
| self._test_configuration = None |
| self._reftest_list = {} |
| self._results_directory = None |
| self._root_was_set = hasattr(options, 'root') and options.root |
| self._jhbuild_wrapper = [] |
| self._layout_tests_dir = hasattr(options, 'layout_tests_dir') and options.layout_tests_dir and self._filesystem.abspath(options.layout_tests_dir) |
| self._w3c_resource_files = None |
| |
| def target_host(self, worker_number=None): |
| return self.host |
| |
| def architecture(self): |
| return self.get_option('architecture') |
| |
| def set_architecture(self, arch): |
| self.did_override_architecture = True |
| self.set_option('architecture', arch) |
| |
| def additional_drt_flag(self): |
| return [] |
| |
| def supports_per_test_timeout(self): |
| return True |
| |
| def default_pixel_tests(self): |
| # FIXME: Disable until they are run by default on build.webkit.org. |
| return False |
| |
| def default_timeout_ms(self): |
| return 30 * 1000 |
| |
| def driver_stop_timeout(self): |
| """ Returns the amount of time in seconds to wait before killing the process in driver.stop().""" |
| # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as |
| # well (for things like ASAN, Valgrind, etc.) |
| return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms() |
| |
| def should_retry_crashes(self): |
| return False |
| |
| def default_child_processes(self, **kwargs): |
| """Return the number of DumpRenderTree instances to use for this port.""" |
| return self._executive.cpu_count() |
| |
| def max_child_processes(self, device_type=None): |
| """Forbid the user from specifying more than this number of child processes""" |
| if device_type: |
| return 0 |
| return float('inf') |
| |
| def supported_device_types(self): |
| # An empty list would indicate a port was incapable of running tests. |
| return [None] |
| |
| def worker_startup_delay_secs(self): |
| # FIXME: If we start workers up too quickly, DumpRenderTree appears |
| # to thrash on something and time out its first few tests. Until |
| # we can figure out what's going on, sleep a bit in between |
| # workers. See https://bugs.webkit.org/show_bug.cgi?id=79147 . |
| return 0.1 |
| |
| def baseline_path(self): |
| """Return the absolute path to the directory to store new baselines in for this port.""" |
| # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir() |
| return self.baseline_version_dir() |
| |
| def baseline_platform_dir(self): |
| """Return the absolute path to the default (version-independent) platform-specific results.""" |
| return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name) |
| |
| def baseline_version_dir(self): |
| """Return the absolute path to the platform-and-version-specific results.""" |
| baseline_search_paths = self.baseline_search_path() |
| return baseline_search_paths[0] |
| |
| def baseline_search_path(self, device_type=None): |
| return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path(device_type=device_type) |
| |
| def default_baseline_search_path(self, device_type=None): |
| """Return a list of absolute paths to directories to search under for |
| baselines. The directories are searched in order.""" |
| search_paths = [] |
| if self.get_option('webkit_test_runner'): |
| search_paths.append(self._wk2_port_name()) |
| search_paths.append(self.name()) |
| if self.name() != self.port_name: |
| search_paths.append(self.port_name) |
| return list(map(self._webkit_baseline_path, search_paths)) |
| |
| @memoized |
| def _compare_baseline(self): |
| factory = PortFactory(self.host) |
| target_port = self.get_option('compare_port') |
| if target_port: |
| return factory.get(target_port).default_baseline_search_path() |
| return [] |
| |
| def check_build(self): |
| """This routine is used to ensure that the build is up to date |
| and all the needed binaries are present.""" |
| # If we're using a pre-built copy of WebKit (--root), we assume it also includes a build of DRT. |
| if not self._root_was_set and self.get_option('build') and not self._build_driver(): |
| return False |
| if self.get_option('install') and not self._check_driver(): |
| return False |
| if not self.check_image_diff(): |
| return self._build_image_diff() |
| return True |
| |
| def check_api_test_build(self, canonicalized_binaries=None): |
| if not canonicalized_binaries: |
| canonicalized_binaries = self.path_to_api_test_binaries().keys() |
| if not self._root_was_set and self.get_option('build') and not self._build_api_tests(wtf_only=(canonicalized_binaries == ['TestWTF'])): |
| return False |
| |
| for binary, path in self.path_to_api_test_binaries().items(): |
| if binary not in canonicalized_binaries: |
| continue |
| if not self._filesystem.exists(path): |
| _log.error('{} was not found at {}'.format(os.path.basename(path), path)) |
| return False |
| return True |
| |
| def environment_for_api_tests(self): |
| return self.setup_environ_for_server() |
| |
| def _check_driver(self): |
| driver_path = self._path_to_driver() |
| if not self._filesystem.exists(driver_path): |
| _log.error("%s was not found at %s" % (self.driver_name(), driver_path)) |
| return False |
| return True |
| |
| def check_sys_deps(self): |
| """If the port needs to do some runtime checks to ensure that the |
| tests can be run successfully, it should override this routine. |
| This step can be skipped with --nocheck-sys-deps. |
| |
| Returns whether the system is properly configured.""" |
| return True |
| |
| def check_image_diff(self, override_step=None, logging=True): |
| """This routine is used to check whether image_diff binary exists.""" |
| image_diff_path = self._path_to_default_image_diff() |
| if not self._filesystem.exists(image_diff_path): |
| if logging: |
| _log.error("ImageDiff was not found at %s" % image_diff_path) |
| return False |
| return True |
| |
| def check_httpd(self): |
| if self._uses_apache(): |
| httpd_path = self._path_to_apache() |
| else: |
| httpd_path = self._path_to_lighttpd() |
| |
| try: |
| server_name = self._filesystem.basename(httpd_path) |
| env = self.setup_environ_for_server(server_name) |
| if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0: |
| _log.error("httpd seems broken. Cannot run http tests.") |
| return False |
| return True |
| except OSError: |
| _log.error("No httpd found. Cannot run http tests.") |
| return False |
| |
| def do_text_results_differ(self, expected_text, actual_text): |
| return expected_text != actual_text |
| |
| def do_audio_results_differ(self, expected_audio, actual_audio): |
| if expected_audio == actual_audio: |
| return False |
| return not WaveDiff(expected_audio, actual_audio).filesAreIdenticalWithinTolerance() |
| |
| def diff_image(self, expected_contents, actual_contents, tolerance=None): |
| """Compare two images and return a tuple of an image diff, a percentage difference (0-100), and an error string. |
| |
| |tolerance| should be a percentage value (0.0 - 100.0). |
| If it is omitted, the port default tolerance value is used. |
| |
| If an error occurs (like ImageDiff isn't found, or crashes, we log an error and return True (for a diff). |
| """ |
| if not actual_contents and not expected_contents: |
| return (None, 0, None) |
| if not actual_contents or not expected_contents: |
| return (True, 0, None) |
| if not self._image_differ: |
| self._image_differ = image_diff.ImageDiffer(self) |
| self.set_option_default('tolerance', 0.1) |
| if tolerance is None: |
| tolerance = self.get_option('tolerance') |
| return self._image_differ.diff_image(expected_contents, actual_contents, tolerance) |
| |
| def diff_text(self, expected_text, actual_text, expected_filename, actual_filename): |
| """Returns a string containing the diff of the two text strings |
| in 'unified diff' format.""" |
| expected_filename = decode_for(encode_if_necessary(expected_filename), str) |
| actual_filename = decode_for(encode_if_necessary(actual_filename), str) |
| diff = difflib.unified_diff(expected_text.splitlines(True), |
| actual_text.splitlines(True), |
| expected_filename, |
| actual_filename) |
| result = "" |
| for line in diff: |
| result += line |
| if not line.endswith('\n'): |
| result += '\n No newline at end of file\n' |
| return result |
| |
| def check_for_leaks(self, process_name, process_id): |
| # Subclasses should check for leaks in the running process |
| # and print any necessary warnings if leaks are found. |
| # FIXME: We should consider moving much of this logic into |
| # Executive and make it platform-specific instead of port-specific. |
| pass |
| |
| def print_leaks_summary(self): |
| # Subclasses can override this to print a summary of leaks found |
| # while running the layout tests. |
| pass |
| |
| def driver_name(self): |
| if self.get_option('driver_name'): |
| return self.get_option('driver_name') |
| if self.get_option('webkit_test_runner'): |
| return 'WebKitTestRunner' |
| return 'DumpRenderTree' |
| |
| def expected_baselines_by_extension(self, test_name): |
| """Returns a dict mapping baseline suffix to relative path for each baseline in |
| a test. For reftests, it returns ".==" or ".!=" instead of the suffix.""" |
| # FIXME: The name similarity between this and expected_baselines() below, is unfortunate. |
| # We should probably rename them both. |
| baseline_dict = {} |
| reference_files = self.reference_files(test_name) |
| if reference_files: |
| # FIXME: How should this handle more than one type of reftest? |
| baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1]) |
| |
| for extension in self.baseline_extensions(): |
| path = self.expected_filename(test_name, extension, return_default=False) |
| baseline_dict[extension] = self.relative_test_filename(path) if path else path |
| |
| return baseline_dict |
| |
| def baseline_extensions(self): |
| """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'.""" |
| return ('.wav', '.webarchive', '.txt', '.png') |
| |
| def _expected_baselines_for_suffixes(self, test_name, suffixes, all_baselines=False, device_type=None): |
| baseline_search_path = self.baseline_search_path(device_type=device_type) + [self.layout_tests_dir()] |
| |
| baselines = [] |
| for platform_dir in baseline_search_path: |
| for suffix in suffixes: |
| baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix |
| if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)): |
| baselines.append((platform_dir, baseline_filename)) |
| |
| if not all_baselines and baselines: |
| return baselines |
| |
| if baselines: |
| return baselines |
| |
| for suffix in suffixes: |
| baselines.append((None, self._filesystem.splitext(test_name)[0] + '-expected' + suffix)) |
| return baselines |
| |
| def expected_baselines(self, test_name, suffix, all_baselines=False, device_type=None): |
| """Given a test name, finds where the baseline results are located. |
| |
| Args: |
| test_name: name of test file (usually a relative path under LayoutTests/) |
| suffix: file suffix of the expected results, including dot; e.g. |
| '.txt' or '.png'. This should not be None, but may be an empty |
| string. |
| all_baselines: If True, return an ordered list of all baseline paths |
| for the given platform. If False, return only the first one. |
| Returns |
| a list of ( platform_dir, results_filename ), where |
| platform_dir - abs path to the top of the results tree (or test |
| tree) |
| results_filename - relative path from top of tree to the results |
| file |
| (port.join() of the two gives you the full path to the file, |
| unless None was returned.) |
| Return values will be in the format appropriate for the current |
| platform (e.g., "\\" for path separators on Windows). If the results |
| file is not found, then None will be returned for the directory, |
| but the expected relative pathname will still be returned. |
| |
| This routine is generic but lives here since it is used in |
| conjunction with the other baseline and filename routines that are |
| platform specific. |
| """ |
| return self._expected_baselines_for_suffixes(test_name, [suffix], all_baselines=all_baselines, device_type=device_type) |
| |
| def expected_filename(self, test_name, suffix, return_default=True, device_type=None): |
| """Given a test name, returns an absolute path to its expected results. |
| |
| If no expected results are found in any of the searched directories, |
| the directory in which the test itself is located will be returned. |
| The return value is in the format appropriate for the platform |
| (e.g., "\\" for path separators on windows). |
| |
| Args: |
| test_name: name of test file (usually a relative path under LayoutTests/) |
| suffix: file suffix of the expected results, including dot; e.g. '.txt' |
| or '.png'. This should not be None, but may be an empty string. |
| platform: the most-specific directory name to use to build the |
| search list of directories; e.g. 'mountainlion-wk2' |
| return_default: if True, returns the path to the generic expectation if nothing |
| else is found; if False, returns None. |
| |
| This routine is generic but is implemented here to live alongside |
| the other baseline and filename manipulation routines. |
| """ |
| platform_dir, baseline_filename = self.expected_baselines(test_name, suffix, device_type=device_type)[0] |
| if platform_dir or return_default: |
| return self._filesystem.join(platform_dir or self.layout_tests_dir(), baseline_filename) |
| return None |
| |
| def expected_checksum(self, test_name, device_type=None): |
| """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test.""" |
| png_path = self.expected_filename(test_name, '.png', device_type=device_type) |
| |
| if self._filesystem.exists(png_path): |
| with self._filesystem.open_binary_file_for_reading(png_path) as filehandle: |
| return read_checksum_from_png.read_checksum(filehandle) |
| |
| return None |
| |
| def expected_image(self, test_name, device_type=None): |
| """Returns the image we expect the test to produce.""" |
| baseline_path = self.expected_filename(test_name, '.png', device_type=device_type) |
| if not self._filesystem.exists(baseline_path): |
| return None |
| return self._filesystem.read_binary_file(baseline_path) |
| |
| def expected_audio(self, test_name, device_type=None): |
| baseline_path = self.expected_filename(test_name, '.wav', device_type=device_type) |
| if not self._filesystem.exists(baseline_path): |
| return None |
| return self._filesystem.read_binary_file(baseline_path) |
| |
| def expected_text(self, test_name, device_type=None): |
| """Returns the text output we expect the test to produce, or None |
| if we don't expect there to be any text output. |
| End-of-line characters are normalized to '\n'.""" |
| # FIXME: DRT output is actually utf-8, but since we don't decode the |
| # output from DRT (instead treating it as a binary string), we read the |
| # baselines as a binary string, too. |
| baseline_path = self.expected_filename(test_name, '.txt', device_type=device_type) |
| if not self._filesystem.exists(baseline_path): |
| baseline_path = self.expected_filename(test_name, '.webarchive', device_type=device_type) |
| if not self._filesystem.exists(baseline_path): |
| return None |
| text = decode_for(self._filesystem.read_binary_file(baseline_path), str) |
| return text.replace("\r\n", "\n") |
| |
| def _get_reftest_list(self, test_name): |
| dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name)) |
| if dirname not in self._reftest_list: |
| self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname) |
| return self._reftest_list[dirname] |
| |
| @staticmethod |
| def _parse_reftest_list(filesystem, test_dirpath): |
| reftest_list_path = filesystem.join(test_dirpath, 'reftest.list') |
| if not filesystem.isfile(reftest_list_path): |
| return None |
| reftest_list_file = filesystem.read_text_file(reftest_list_path) |
| |
| parsed_list = {} |
| for line in reftest_list_file.split('\n'): |
| line = re.sub('#.+$', '', line) |
| split_line = line.split() |
| if len(split_line) < 3: |
| continue |
| expectation_type, test_file, ref_file = split_line |
| parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file))) |
| return parsed_list |
| |
| def reference_files(self, test_name, device_type=None): |
| """Return a list of expectation (== or !=) and filename pairs""" |
| |
| if self.get_option('treat_ref_tests_as_pixel_tests'): |
| return [] |
| |
| result = self._get_reftest_list(test_name) |
| if result: |
| return result.get(self._filesystem.join(self.layout_tests_dir(), test_name), []) |
| |
| result = [] |
| suffixes = [] |
| for part1 in ['', '-mismatch']: |
| for part2 in self._supported_reference_extensions: |
| suffixes.append(part1 + part2) |
| for platform_dir, baseline_filename in self._expected_baselines_for_suffixes(test_name, suffixes, device_type=device_type): |
| if not platform_dir: |
| continue |
| result.append(( |
| '!=' if '-mismatch.' in baseline_filename else '==', |
| self._filesystem.join(platform_dir, baseline_filename), |
| )) |
| return result |
| |
| def potential_test_names_from_expected_file(self, path): |
| """Return potential test names if any from a potential expected file path, relative to LayoutTests directory.""" |
| |
| if not '-expected.' in path: |
| return None |
| |
| if path.startswith('platform' + self._filesystem.sep): |
| steps = path.split(self._filesystem.sep) |
| path = self._filesystem.join(self._filesystem.sep.join(steps[2:])) |
| |
| return [self.host.filesystem.relpath(test, self.layout_tests_dir()) for test in self._filesystem.glob(re.sub('-expected.*', '.*', self._filesystem.join(self.layout_tests_dir(), path))) if self._filesystem.isfile(test)] |
| |
| def tests(self, paths, device_type=None): |
| """Return the list of tests found. Both generic and platform-specific tests matching paths should be returned.""" |
| expanded_paths = self._expanded_paths(paths, device_type=device_type) |
| return self._real_tests(expanded_paths) |
| |
| def _expanded_paths(self, paths, device_type=None): |
| expanded_paths = [] |
| fs = self._filesystem |
| all_platform_dirs = [path for path in fs.glob(fs.join(self.layout_tests_dir(), 'platform', '*')) if fs.isdir(path)] |
| for path in paths: |
| expanded_paths.append(path) |
| if self.test_isdir(path) and not path.startswith('platform') and not fs.isabs(path): |
| for platform_dir in all_platform_dirs: |
| if fs.isdir(fs.join(platform_dir, path)) and platform_dir in self.baseline_search_path(device_type=device_type): |
| expanded_paths.append(self.relative_test_filename(fs.join(platform_dir, path))) |
| |
| return expanded_paths |
| |
| def _real_tests(self, paths): |
| # When collecting test cases, skip these directories |
| skipped_directories = set(['.svn', '_svn', 'resources', 'support', 'script-tests', 'reference', 'reftest']) |
| files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, partial(Port._is_test_file, self), self.test_key) |
| return [self.relative_test_filename(f) for f in files] |
| |
| # When collecting test cases, we include any file with these extensions. |
| _supported_test_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl', '.htm', '.php', '.svg', '.mht', '.xht']) |
| _supported_reference_extensions = set(['.html', '.xml', '.xhtml', '.htm', '.svg', '.xht']) |
| |
| def is_w3c_resource_file(self, filesystem, dirname, filename): |
| path = filesystem.join(dirname, filename) |
| w3c_path = filesystem.join(self.layout_tests_dir(), "imported", "w3c") |
| if not w3c_path in path: |
| return False |
| |
| if not self._w3c_resource_files: |
| filepath = filesystem.join(w3c_path, "resources", "resource-files.json") |
| json_data = filesystem.read_text_file(filepath) |
| self._w3c_resource_files = json.loads(json_data) |
| |
| subpath = path[len(w3c_path) + 1:].replace('\\', '/') |
| if subpath in self._w3c_resource_files["files"]: |
| return True |
| for dirpath in self._w3c_resource_files["directories"]: |
| if dirpath in subpath: |
| return True |
| return False |
| |
| @staticmethod |
| # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well. |
| def is_reference_html_file(filesystem, dirname, filename): |
| if filename.startswith('ref-') or filename.startswith('notref-'): |
| return True |
| filename_wihout_ext, ext = filesystem.splitext(filename) |
| if ext not in Port._supported_reference_extensions: |
| return False |
| for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']: |
| if filename_wihout_ext.endswith(suffix): |
| return True |
| return False |
| |
| @staticmethod |
| def _has_supported_extension(filesystem, filename): |
| """Return true if filename is one of the file extensions we want to run a test on.""" |
| extension = filesystem.splitext(filename)[1] |
| return extension in Port._supported_test_extensions |
| |
| def _is_test_file(self, filesystem, dirname, filename): |
| if not Port._has_supported_extension(filesystem, filename): |
| return False |
| if Port.is_reference_html_file(filesystem, dirname, filename): |
| return False |
| if self.is_w3c_resource_file(filesystem, dirname, filename): |
| return False |
| return True |
| |
| def test_key(self, test_name): |
| """Turns a test name into a list with two sublists, the natural key of the |
| dirname, and the natural key of the basename. |
| |
| This can be used when sorting paths so that files in a directory. |
| directory are kept together rather than being mixed in with files in |
| subdirectories.""" |
| dirname, basename = self.split_test(test_name) |
| return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename)) |
| |
| def _natural_sort_key(self, string_to_split): |
| """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"] |
| |
| This can be used to implement "natural sort" order. See: |
| http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html |
| http://nedbatchelder.com/blog/200712.html#e20071211T054956 |
| """ |
| def tryint(val): |
| try: |
| return int(val) |
| except ValueError: |
| return val |
| |
| return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)] |
| |
| def test_dirs(self): |
| """Returns the list of top-level test directories.""" |
| layout_tests_dir = self.layout_tests_dir() |
| return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)), |
| self._filesystem.listdir(layout_tests_dir)) |
| |
| @memoized |
| def test_isfile(self, test_name): |
| """Return True if the test name refers to a directory of tests.""" |
| # Used by test_expectations.py to apply rules to whole directories. |
| return self._filesystem.isfile(self.abspath_for_test(test_name)) |
| |
| @memoized |
| def test_isdir(self, test_name): |
| """Return True if the test name refers to a directory of tests.""" |
| # Used by test_expectations.py to apply rules to whole directories. |
| return self._filesystem.isdir(self.abspath_for_test(test_name)) |
| |
| @memoized |
| def test_exists(self, test_name): |
| """Return True if the test name refers to an existing test or baseline.""" |
| # Used by test_expectations.py to determine if an entry refers to a |
| # valid test and by printing.py to determine if baselines exist. |
| return self.test_isfile(test_name) or self.test_isdir(test_name) |
| |
| def split_test(self, test_name): |
| """Splits a test name into the 'directory' part and the 'basename' part.""" |
| index = test_name.rfind(self.TEST_PATH_SEPARATOR) |
| if index < 1: |
| return ('', test_name) |
| return (test_name[0:index], test_name[index:]) |
| |
| def normalize_test_name(self, test_name): |
| """Returns a normalized version of the test name or test directory.""" |
| if test_name.endswith(self.TEST_PATH_SEPARATOR): |
| return test_name |
| if self.test_isdir(test_name): |
| return test_name + self.TEST_PATH_SEPARATOR |
| return test_name |
| |
| def driver_cmd_line_for_logging(self): |
| """Prints the DRT command line that will be used.""" |
| driver = self.create_driver(0) |
| return driver.cmd_line(self.get_option('pixel_tests'), []) |
| |
| def update_baseline(self, baseline_path, data): |
| """Updates the baseline for a test. |
| |
| Args: |
| baseline_path: the actual path to use for baseline, not the path to |
| the test. This function is used to update either generic or |
| platform-specific baselines, but we can't infer which here. |
| data: contents of the baseline. |
| """ |
| self._filesystem.write_binary_file(baseline_path, data) |
| |
| # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected). |
| def webkit_base(self): |
| return self._webkit_finder.webkit_base() |
| |
| def path_from_webkit_base(self, *comps): |
| return self._webkit_finder.path_from_webkit_base(*comps) |
| |
| def path_to_script(self, script_name): |
| return self._webkit_finder.path_to_script(script_name) |
| |
| def layout_tests_dir(self): |
| if self._layout_tests_dir: |
| return self._layout_tests_dir |
| return self._webkit_finder.layout_tests_dir() |
| |
| def perf_tests_dir(self): |
| return self._webkit_finder.perf_tests_dir() |
| |
| def skipped_layout_tests(self, test_list, device_type=None): |
| """Returns tests skipped outside of the TestExpectations files.""" |
| return set(self._tests_for_other_platforms(device_type=device_type)).union(self._skipped_tests_for_unsupported_features(test_list)) |
| |
| @memoized |
| def skipped_perf_tests(self): |
| filename = self._filesystem.join(self.perf_tests_dir(), "Skipped") |
| if not self._filesystem.exists(filename): |
| _log.debug("Skipped does not exist: %s" % filename) |
| return [] |
| |
| skipped_file_contents = self._filesystem.read_text_file(filename) |
| tests_to_skip = [] |
| for line_number, line in enumerate(skipped_file_contents.split('\n')): |
| match = re.match(r'^\s*(\[(?P<platforms>[\w ]*?)\])?\s*(?P<test>[\w\-\/\.]+?)?\s*(?P<comment>\#.*)?$', line) |
| if not match: |
| _log.error("Syntax error at line %d in %s: %s" % (line_number + 1, filename, line)) |
| else: |
| platform_names = list(filter(lambda token: token, match.group('platforms').lower().split(' '))) if match.group('platforms') else [] |
| test_name = match.group('test') |
| if test_name and (not platform_names or self.port_name in platform_names or self._name in platform_names): |
| tests_to_skip.append(test_name) |
| |
| return tests_to_skip |
| |
| def skips_perf_test(self, test_name): |
| for test_or_category in self.skipped_perf_tests(): |
| if test_or_category == test_name: |
| return True |
| category = self._filesystem.join(self.perf_tests_dir(), test_or_category) |
| if self._filesystem.isdir(category) and test_name.startswith(test_or_category): |
| return True |
| return False |
| |
| def name(self): |
| """Returns a name that uniquely identifies this particular type of port |
| (e.g., "mac-snowleopard" or "chromium-linux-x86_x64" and can be passed |
| to factory.get() to instantiate the port.""" |
| return self._name |
| |
| def operating_system(self): |
| # Subclasses should override this default implementation. |
| return 'mac' |
| |
| @memoized |
| def version_name(self): |
| """Returns a string indicating the version of a given platform, e.g. |
| 'leopard' or 'xp'. |
| |
| This is used to help identify the exact port when parsing test |
| expectations, determining search paths, and logging information.""" |
| if self._os_version is None: |
| return None |
| result = VersionNameMap.map(self.host.platform).to_name(self._os_version, table=PUBLIC_TABLE) |
| if not result: |
| result = VersionNameMap.map(self.host.platform).to_name(self._os_version, table=INTERNAL_TABLE) |
| return result |
| |
| def get_option(self, name, default_value=None): |
| return getattr(self._options, name, default_value) |
| |
| def set_option(self, name, value): |
| setattr(self._options, name, value) |
| return self.get_option(name) == value |
| |
| def set_option_default(self, name, default_value): |
| return self._options.ensure_value(name, default_value) |
| |
| @memoized |
| def path_to_generic_test_expectations_file(self): |
| return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations') |
| |
| @memoized |
| def path_to_test_expectations_file(self): |
| """Update the test expectations to the passed-in string. |
| |
| This is used by the rebaselining tool. Raises NotImplementedError |
| if the port does not use expectations files.""" |
| |
| # FIXME: We need to remove this when we make rebaselining work with multiple files and just generalize expectations_files(). |
| |
| # test_expectations are always in mac/ not mac-leopard/ by convention, hence we use port_name instead of name(). |
| return self._filesystem.join(self._webkit_baseline_path(self.port_name), 'TestExpectations') |
| |
| def relative_test_filename(self, filename): |
| """Returns a test_name a relative unix-style path for a filename under the LayoutTests |
| directory. Ports may legitimately return abspaths here if no relpath makes sense.""" |
| # Ports that run on windows need to override this method to deal with |
| # filenames with backslashes in them. |
| if filename.startswith(self.layout_tests_dir()): |
| return self.host.filesystem.relpath(filename, self.layout_tests_dir()).replace(self.host.filesystem.sep, self.TEST_PATH_SEPARATOR) |
| else: |
| return self.host.filesystem.abspath(filename) |
| |
| @memoized |
| def abspath_for_test(self, test_name, target_host=None): |
| """Returns the full path to the file for a given test name. This is the |
| inverse of relative_test_filename() if no target_host is specified.""" |
| host = target_host or self.host |
| return host.filesystem.join(host.filesystem.map_base_host_path(self.layout_tests_dir()), test_name.replace(self.TEST_PATH_SEPARATOR, self.host.filesystem.sep)) |
| |
| def jsc_results_directory(self): |
| return self._build_path() |
| |
| def bindings_results_directory(self): |
| return self._build_path() |
| |
| def api_results_directory(self): |
| return self._build_path() |
| |
| def results_directory(self): |
| """Absolute path to the place to store the test results (uses --results-directory).""" |
| if not self._results_directory: |
| option_val = self.get_option('results_directory') or self.default_results_directory() |
| self._results_directory = self._filesystem.abspath(option_val) |
| return self._results_directory |
| |
| def perf_results_directory(self): |
| return self._build_path() |
| |
| def python_unittest_results_directory(self): |
| return self._build_path('python-unittest-results') |
| |
| def default_results_directory(self): |
| """Absolute path to the default place to store the test results.""" |
| # Results are store relative to the built products to make it easy |
| # to have multiple copies of webkit checked out and built. |
| return self._build_path('layout-test-results') |
| |
| def setup_test_run(self, device_type=None): |
| """Perform port-specific work at the beginning of a test run.""" |
| pass |
| |
| def clean_up_test_run(self): |
| """Perform port-specific work at the end of a test run.""" |
| if self._image_differ: |
| self._image_differ.stop() |
| self._image_differ = None |
| |
| # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable. |
| def _value_or_default_from_environ(self, name, default=None): |
| if name in os.environ: |
| return os.environ[name] |
| return default |
| |
| def _copy_value_from_environ_if_set(self, clean_env, name): |
| if name in os.environ: |
| clean_env[name] = os.environ[name] |
| |
| def setup_environ_for_server(self, server_name=None): |
| # We intentionally copy only a subset of os.environ when |
| # launching subprocesses to ensure consistent test results. |
| clean_env = {} |
| # Note: don't set here driver specific variables (related to X11, Wayland, etc.) |
| # Use the driver _setup_environ_for_test() method for that. |
| variables_to_copy = [ |
| # For Linux: |
| 'ALSA_CARD', |
| 'DBUS_SESSION_BUS_ADDRESS', |
| 'LANG', |
| 'LD_LIBRARY_PATH', |
| 'TERM', |
| 'XDG_DATA_DIRS', |
| 'XDG_RUNTIME_DIR', |
| |
| # Darwin: |
| 'DYLD_FRAMEWORK_PATH', |
| 'DYLD_LIBRARY_PATH', |
| '__XPC_DYLD_FRAMEWORK_PATH', |
| '__XPC_DYLD_LIBRARY_PATH', |
| |
| # CYGWIN: |
| 'HOMEDRIVE', |
| 'HOMEPATH', |
| '_NT_SYMBOL_PATH', |
| |
| # Windows: |
| 'COMSPEC', |
| 'SYSTEMDRIVE', |
| 'SYSTEMROOT', |
| 'WEBKIT_LIBRARIES', |
| |
| # Most ports (?): |
| 'HOME', |
| 'PATH', |
| 'WEBKIT_TESTFONTS', |
| 'WEBKIT_OUTPUTDIR', |
| |
| ] |
| for variable in variables_to_copy: |
| self._copy_value_from_environ_if_set(clean_env, variable) |
| |
| for string_variable in self.get_option('additional_env_var', []): |
| [name, value] = string_variable.split('=', 1) |
| clean_env[name] = value |
| |
| return clean_env |
| |
| def _clear_global_caches_and_temporary_files(self): |
| pass |
| |
| @staticmethod |
| def _append_value_colon_separated(env, name, value): |
| assert os.pathsep not in value |
| if name in env and env[name]: |
| env[name] = env[name] + os.pathsep + value |
| else: |
| env[name] = value |
| |
| def show_results_html_file(self, results_filename): |
| """This routine should display the HTML file pointed at by |
| results_filename in a users' browser.""" |
| return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename)) |
| |
| def create_driver(self, worker_number, no_timeout=False): |
| """Return a newly created Driver subclass for starting/stopping the test driver.""" |
| return driver.DriverProxy(self, worker_number, self._driver_class(), pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout) |
| |
| def start_helper(self, pixel_tests=False): |
| """If a port needs to reconfigure graphics settings or do other |
| things to ensure a known test configuration, it should override this |
| method.""" |
| return True |
| |
| def reset_preferences(self): |
| """If a port needs to reset platform-specific persistent preference |
| storage, it should override this method.""" |
| pass |
| |
| def ports_to_forward(self): |
| ports = [] |
| if self._http_server: |
| ports.extend(self._http_server.ports_to_forward()) |
| if self._websocket_server: |
| ports.extend(self._websocket_server.ports_to_forward()) |
| if self._websocket_server: |
| ports.extend(self._websocket_secure_server.ports_to_forward()) |
| if self._web_platform_test_server: |
| ports.extend(self._web_platform_test_server.ports_to_forward()) |
| return ports |
| |
| def start_http_server(self, additional_dirs=None): |
| """Start a web server. Raise an error if it can't start or is already running. |
| |
| Ports can stub this out if they don't need a web server to be running.""" |
| assert not self._http_server, 'Already running an http server.' |
| if not self.check_httpd(): |
| return |
| |
| http_port = self.get_option('http_port') |
| if self._uses_apache(): |
| server = apache_http_server.LayoutTestApacheHttpd(self, self.results_directory(), additional_dirs=additional_dirs, port=http_port) |
| else: |
| server = http_server.Lighttpd(self, self.results_directory(), additional_dirs=additional_dirs, port=http_port) |
| |
| server.start() |
| self._http_server = server |
| |
| def is_http_server_running(self): |
| if self._http_server: |
| return True |
| return http_server_base.is_http_server_running() |
| |
| def is_websocket_server_running(self): |
| if self._websocket_server: |
| return True |
| return websocket_server.is_web_socket_server_running() |
| |
| def is_wpt_server_running(self): |
| if self._web_platform_test_server: |
| return True |
| return web_platform_test_server.is_wpt_server_running(self) |
| |
| def start_websocket_server(self): |
| """Start a web server. Raise an error if it can't start or is already running. |
| |
| Ports can stub this out if they don't need a websocket server to be running.""" |
| assert not self._websocket_server, 'Already running a websocket server.' |
| |
| server = websocket_server.PyWebSocket(self, self.results_directory()) |
| server.start() |
| self._websocket_server = server |
| |
| websocket_server_temporary_directory = self._filesystem.mkdtemp(prefix='webkitpy-websocket-server') |
| self._websocket_server_temporary_directory = websocket_server_temporary_directory |
| |
| pem_file = self._filesystem.join(self.layout_tests_dir(), "http", "conf", "webkit-httpd.pem") |
| pem = pemfile.load(self._filesystem, pem_file) |
| certificate_file = self._filesystem.join(str(websocket_server_temporary_directory), 'webkit-httpd.crt') |
| self._filesystem.write_text_file(certificate_file, pem.certificate) |
| private_key_file = self._filesystem.join(str(websocket_server_temporary_directory), 'webkit-httpd.key') |
| self._filesystem.write_text_file(private_key_file, pem.private_key) |
| |
| secure_server = self._websocket_secure_server = websocket_server.PyWebSocket(self, self.results_directory(), |
| use_tls=True, port=websocket_server.PyWebSocket.DEFAULT_WSS_PORT, private_key=private_key_file, certificate=certificate_file) |
| secure_server.start() |
| self._websocket_secure_server = secure_server |
| |
| def start_web_platform_test_server(self, additional_dirs=None, number_of_servers=None): |
| assert not self._web_platform_test_server, 'Already running a Web Platform Test server.' |
| |
| self._web_platform_test_server = web_platform_test_server.WebPlatformTestServer(self, "wptwk") |
| self._web_platform_test_server.start() |
| |
| def web_platform_test_server_doc_root(self): |
| return web_platform_test_server.doc_root(self) + self.TEST_PATH_SEPARATOR |
| |
| def web_platform_test_server_base_http_url(self): |
| return web_platform_test_server.base_http_url(self) |
| |
| def web_platform_test_server_base_https_url(self): |
| return web_platform_test_server.base_https_url(self) |
| |
| def http_server_supports_ipv6(self): |
| # Cygwin is the only platform to still use Apache 1.3, which only supports IPV4. |
| # Once it moves to Apache 2, we can drop this method altogether. |
| if self.host.platform.is_cygwin(): |
| return False |
| return True |
| |
| def stop_helper(self): |
| """Shut down the test helper if it is running. Do nothing if |
| it isn't, or it isn't available. If a port overrides start_helper() |
| it must override this routine as well.""" |
| pass |
| |
| def stop_http_server(self): |
| """Shut down the http server if it is running. Do nothing if it isn't.""" |
| if self._http_server: |
| self._http_server.stop() |
| self._http_server = None |
| |
| def stop_websocket_server(self): |
| """Shut down the websocket server if it is running. Do nothing if it isn't.""" |
| if self._websocket_server: |
| self._websocket_server.stop() |
| self._websocket_server = None |
| if self._websocket_secure_server: |
| self._websocket_secure_server.stop() |
| self._websocket_secure_server = None |
| if self._websocket_server_temporary_directory: |
| self._filesystem.rmtree(str(self._websocket_server_temporary_directory)) |
| |
| def stop_web_platform_test_server(self): |
| if self._web_platform_test_server: |
| self._web_platform_test_server.stop() |
| self._web_platform_test_server = None |
| |
| def exit_code_from_summarized_results(self, unexpected_results): |
| """Given summarized results, compute the exit code to be returned by new-run-webkit-tests. |
| Bots turn red when this function returns a non-zero value. By default, return the number of regressions |
| to avoid turning bots red by flaky failures, unexpected passes, and missing results""" |
| # Don't turn bots red for flaky failures, unexpected passes, and missing results. |
| return unexpected_results['num_regressions'] |
| |
| # |
| # TEST EXPECTATION-RELATED METHODS |
| # |
| |
| def test_configuration(self): |
| """Returns the current TestConfiguration for the port.""" |
| if not self._test_configuration: |
| self._test_configuration = TestConfiguration(self.version_name(), self.architecture(), self._options.configuration.lower()) |
| return self._test_configuration |
| |
| # FIXME: Belongs on a Platform object. |
| @memoized |
| def all_test_configurations(self): |
| """Returns a list of TestConfiguration instances, representing all available |
| test configurations for this port.""" |
| return self._generate_all_test_configurations() |
| |
| # FIXME: Belongs on a Platform object. |
| def configuration_specifier_macros(self): |
| """Ports may provide a way to abbreviate configuration specifiers to conveniently |
| refer to them as one term or alias specific values to more generic ones. For example: |
| |
| (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake. |
| (lucid) -> linux # Change specific name of the Linux distro to a more generic term. |
| |
| Returns a dictionary, each key representing a macro term ('win', for example), |
| and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7']).""" |
| return {} |
| |
| def all_baseline_variants(self): |
| """Returns a list of platform names sufficient to cover all the baselines. |
| |
| The list should be sorted so that a later platform will reuse |
| an earlier platform's baselines if they are the same (e.g., |
| 'snowleopard' should precede 'leopard').""" |
| raise NotImplementedError |
| |
| def uses_test_expectations_file(self): |
| # This is different from checking test_expectations() is None, because |
| # some ports have Skipped files which are returned as part of test_expectations(). |
| for path in self.default_baseline_search_path(): |
| if self._filesystem.exists(self._filesystem.join(path, 'TestExpectations')): |
| return True |
| return False |
| |
| def warn_if_bug_missing_in_test_expectations(self): |
| return False |
| |
| def expectations_dict(self, device_type=None): |
| """Returns an OrderedDict of name -> expectations strings. |
| The names are expected to be (but not required to be) paths in the filesystem. |
| If the name is a path, the file can be considered updatable for things like rebaselining, |
| so don't use names that are paths if they're not paths. |
| Generally speaking the ordering should be files in the filesystem in cascade order |
| (TestExpectations followed by Skipped, if the port honors both formats), |
| then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options.""" |
| # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict. |
| expectations = OrderedDict() |
| |
| for path in self.expectations_files(device_type=device_type): |
| if self._filesystem.exists(path): |
| expectations[path] = self._filesystem.read_text_file(path) |
| |
| for path in self.get_option('additional_expectations', []): |
| expanded_path = self._filesystem.expanduser(path) |
| if self._filesystem.exists(expanded_path): |
| _log.debug("reading additional_expectations from path '%s'" % path) |
| expectations[path] = self._filesystem.read_text_file(expanded_path) |
| else: |
| _log.warning("additional_expectations path '%s' does not exist" % path) |
| return expectations |
| |
| def _port_specific_expectations_files(self, **kwargs): |
| # Unlike baseline_search_path, we only want to search [WK2-PORT, PORT-VERSION, PORT] and any directories |
| # included via --additional-platform-directory, not the full casade. |
| search_paths = [self.port_name] |
| |
| non_wk2_name = self.name().replace('-wk2', '') |
| if non_wk2_name != self.port_name: |
| search_paths.append(non_wk2_name) |
| |
| if self.get_option('webkit_test_runner'): |
| # Because nearly all of the skipped tests for WebKit 2 are due to cross-platform |
| # issues, all wk2 ports share a skipped list under platform/wk2. |
| search_paths.extend(["wk2", self._wk2_port_name()]) |
| |
| search_paths.extend(self.get_option("additional_platform_directory", [])) |
| |
| return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpectations') for d in search_paths] |
| |
| def expectations_files(self, device_type=None): |
| return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files(device_type=device_type) |
| |
| def repository_paths(self): |
| """Returns a list of (repository_name, repository_path) tuples of its depending code base. |
| By default it returns a list that only contains a ('WebKit', <webkitRepositoryPath>) tuple.""" |
| |
| # We use LayoutTest directory here because webkit_base isn't a part of WebKit repository in Chromium port |
| # where turnk isn't checked out as a whole. |
| repository_paths = [('WebKit', self.layout_tests_dir())] |
| if self.get_option('additional_repository_name') and self.get_option('additional_repository_path'): |
| repository_paths += [(self._options.additional_repository_name, self._options.additional_repository_path)] |
| return repository_paths |
| |
| def allowed_hosts(self): |
| return self.get_option("allowed_host", []) |
| |
| def internal_feature(self): |
| return self.get_option("internal_feature", []) |
| |
| def experimental_feature(self): |
| return self.get_option("experimental_feature", []) |
| |
| def default_configuration(self): |
| return self._config.default_configuration() |
| |
| # |
| # PROTECTED ROUTINES |
| # |
| # The routines below should only be called by routines in this class |
| # or any of its subclasses. |
| # |
| |
| def _uses_apache(self): |
| return True |
| |
| # FIXME: This does not belong on the port object. |
| @memoized |
| def _path_to_apache(self): |
| """Returns the full path to the apache binary. |
| |
| This is needed only by ports that use the apache_http_server module.""" |
| # The Apache binary path can vary depending on OS and distribution |
| # See http://wiki.apache.org/httpd/DistrosDefaultLayout |
| for path in ["/usr/sbin/httpd", "/usr/sbin/apache2", "/app/bin/httpd"]: |
| if self._filesystem.exists(path): |
| return path |
| _log.error("Could not find apache. Not installed or unknown path.") |
| return None |
| |
| def _is_fedora_php_version_7(self): |
| if self._filesystem.exists("/etc/httpd/modules/libphp7.so"): |
| return True |
| return False |
| |
| def _is_darwin_php_version_7(self): |
| if self._filesystem.exists("/usr/libexec/apache2/libphp7.so"): |
| return True |
| return False |
| |
| # FIXME: This belongs on some platform abstraction instead of Port. |
| def _is_redhat_based(self): |
| return self._filesystem.exists('/etc/redhat-release') |
| |
| def _is_debian_based(self): |
| return self._filesystem.exists('/etc/debian_version') |
| |
| def _is_arch_based(self): |
| return self._filesystem.exists('/etc/arch-release') |
| |
| def _is_flatpak(self): |
| return self._filesystem.exists('/.flatpak-info') |
| |
| def _apache_version(self): |
| config = self._executive.run_command([self._path_to_apache(), '-v']) |
| return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config) |
| |
| def _debian_php_version(self): |
| prefix = "/usr/lib/apache2/modules/" |
| for version in ("7.0", "7.1", "7.2", "7.3"): |
| if self._filesystem.exists("%s/libphp%s.so" % (prefix, version)): |
| return "-php%s" % version |
| _log.error("No libphp7.x.so found in %s" % prefix) |
| return "" |
| |
| def _darwin_php_version(self): |
| if self._is_darwin_php_version_7(): |
| return "-php7" |
| return "" |
| |
| def _fedora_php_version(self): |
| if self._is_fedora_php_version_7(): |
| return "-php7" |
| return "" |
| |
| def _win_php_version(self): |
| root = os.environ.get('XAMPP_ROOT', 'C:\\xampp') |
| prefix = self._filesystem.join(root, 'php') |
| for version in ('5', '7'): |
| conf = self._filesystem.join(prefix, "php{}ts.dll".format(version)) |
| if self._filesystem.exists(conf): |
| return "-php{}".format(version) |
| _log.error("No php?ts.dll found in {}".format(prefix)) |
| return "" |
| |
| # We pass sys_platform into this method to make it easy to unit test. |
| def _apache_config_file_name_for_platform(self, sys_platform): |
| if sys_platform in ['cygwin', 'win32']: |
| return 'win-httpd-' + self._apache_version() + self._win_php_version() + '.conf' |
| if sys_platform == 'darwin': |
| return 'apache' + self._apache_version() + self._darwin_php_version() + '-httpd.conf' |
| if sys_platform.startswith('linux'): |
| if self._is_redhat_based(): |
| return 'fedora-httpd-' + self._apache_version() + self._fedora_php_version() + '.conf' |
| if self._is_debian_based(): |
| return 'debian-httpd-' + self._apache_version() + self._debian_php_version() + '.conf' |
| if self._is_arch_based(): |
| return 'archlinux-httpd.conf' |
| if self._is_flatpak(): |
| return 'flatpak-httpd.conf' |
| # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support). |
| return 'apache' + self._apache_version() + '-httpd.conf' |
| |
| def _path_to_apache_config_file(self): |
| """Returns the full path to the apache configuration file. |
| |
| If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its |
| contents will be used instead. |
| |
| This is needed only by ports that use the apache_http_server module.""" |
| config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH') |
| if config_file_from_env: |
| if not self._filesystem.exists(config_file_from_env): |
| raise IOError('%s was not found on the system' % config_file_from_env) |
| return config_file_from_env |
| |
| config_file_name = self._apache_config_file_name_for_platform(sys.platform) |
| return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name) |
| |
| def _build_path(self, *comps): |
| root_directory = self.get_option('_cached_root') or self.get_option('root') |
| if not root_directory: |
| root_directory = self._config.build_directory(self.get_option('configuration')) |
| build_directory = self.get_option('build_directory') |
| if build_directory: |
| root_directory = self._filesystem.join(build_directory, root_directory.split('/')[-1]) |
| |
| # We take advantage of the behavior that self._options is passed by reference to worker |
| # subprocesses to use it as data store to cache the computed root directory path. This |
| # avoids making each worker subprocess compute this path again which is slow because of |
| # the call to config.build_directory(). |
| # |
| # FIXME: This is like decorating this function with @memoized, but more annoying and fragile; |
| # there should be another way to propagate precomputed values to workers without modifying |
| # the options list. |
| self.set_option('_cached_root', root_directory) |
| |
| if sys.platform.startswith('win') or sys.platform == 'cygwin': |
| return self._filesystem.join(root_directory, *comps) |
| |
| return self._filesystem.join(self._filesystem.abspath(root_directory), *comps) |
| |
| def _path_to_driver(self, configuration=None): |
| """Returns the full path to the test driver (DumpRenderTree).""" |
| local_driver_path = self._build_path(self.driver_name()) |
| if sys.platform.startswith('win'): |
| base = os.path.splitext(local_driver_path)[0] |
| local_driver_path = base + ".exe" |
| return local_driver_path |
| |
| def _driver_tempdir(self, target_host=None): |
| host = target_host or self.host |
| return host.filesystem.mkdtemp(prefix='{}s-'.format(self.driver_name())) |
| |
| def _path_to_user_cache_directory(self, suffix=None): |
| return None |
| |
| def _path_to_webcore_library(self): |
| """Returns the full path to a built copy of WebCore.""" |
| return None |
| |
| def _path_to_helper(self): |
| """Returns the full path to the layout_test_helper binary, which |
| is used to help configure the system for the test run, or None |
| if no helper is needed. |
| |
| This is likely only used by start/stop_helper().""" |
| return None |
| |
| def _path_to_default_image_diff(self): |
| """Returns the full path to the default ImageDiff binary, or None if it is not available.""" |
| return self._build_path('ImageDiff') |
| |
| @memoized |
| def _path_to_image_diff(self): |
| """Returns the full path to the image_diff binary, or None if it is not available. |
| |
| This is likely used only by diff_image()""" |
| default_image_diff = self._path_to_default_image_diff() |
| if self._filesystem.exists(default_image_diff): |
| return default_image_diff |
| built_image_diff = self._filesystem.join(self._config.build_directory(self.get_option('configuration'), for_host=True), 'ImageDiff') |
| _log.debug('ImageDiff not found at {}, using {} instead'.format(default_image_diff, built_image_diff)) |
| return built_image_diff |
| |
| API_TEST_BINARY_NAMES = ['TestWTF', 'TestWebKitAPI'] |
| |
| def path_to_api_test_binaries(self): |
| return {binary: self._build_path(binary) for binary in self.API_TEST_BINARY_NAMES} |
| |
| def _path_to_lighttpd(self): |
| """Returns the path to the LigHTTPd binary. |
| |
| This is needed only by ports that use the http_server.py module.""" |
| raise NotImplementedError('Port._path_to_lighttpd') |
| |
| def _path_to_lighttpd_modules(self): |
| """Returns the path to the LigHTTPd modules directory. |
| |
| This is needed only by ports that use the http_server.py module.""" |
| raise NotImplementedError('Port._path_to_lighttpd_modules') |
| |
| def _path_to_lighttpd_php(self): |
| """Returns the path to the LigHTTPd PHP executable. |
| |
| This is needed only by ports that use the http_server.py module.""" |
| raise NotImplementedError('Port._path_to_lighttpd_php') |
| |
| def _webkit_baseline_path(self, platform): |
| """Return the full path to the top of the baseline tree for a |
| given platform.""" |
| return self._filesystem.join(self.layout_tests_dir(), 'platform', platform) |
| |
| # FIXME: Belongs on a Platform object. |
| def _generate_all_test_configurations(self): |
| """Generates a list of TestConfiguration instances, representing configurations |
| for a platform across all OSes, architectures, build and graphics types.""" |
| raise NotImplementedError('Port._generate_test_configurations') |
| |
| def _driver_class(self): |
| """Returns the port's driver implementation.""" |
| return driver.Driver |
| |
| def path_to_crash_logs(self): |
| raise NotImplementedError |
| |
| def _get_crash_log(self, name, pid, stdout, stderr, newer_than, target_host=None): |
| name_str = name or '<unknown process name>' |
| pid_str = str(pid or '<unknown>') |
| stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines() |
| stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines() |
| return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str, |
| '\n'.join(('STDOUT: ' + l) for l in stdout_lines), |
| '\n'.join(('STDERR: ' + l) for l in stderr_lines))) |
| |
| def look_for_new_crash_logs(self, crashed_processes, start_time): |
| pass |
| |
| def look_for_new_samples(self, unresponsive_processes, start_time): |
| pass |
| |
| def sample_process(self, name, pid, target_host=None): |
| pass |
| |
| def should_run_as_pixel_test(self, test_input): |
| if not self._options.pixel_tests: |
| return False |
| if self._options.pixel_test_directories: |
| return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories) |
| return self._should_run_as_pixel_test(test_input) |
| |
| def _should_run_as_pixel_test(self, test_input): |
| # Default behavior is to allow all test to run as pixel tests if --pixel-tests is on and |
| # --pixel-test-directory is not specified. |
| return True |
| |
| def _should_use_flatpak(self): |
| suffix = "" |
| if self.port_name: |
| suffix = self.port_name.upper() |
| return self._filesystem.exists(self.path_from_webkit_base('WebKitBuild', suffix, "FlatpakTree")) |
| |
| def _in_flatpak_sandbox(self): |
| return os.path.exists("/.flatpak-info") |
| |
| def _should_use_jhbuild(self): |
| if self._in_flatpak_sandbox(): |
| return False |
| |
| suffix = "" |
| if self.port_name: |
| suffix = self.port_name.upper() |
| return self._filesystem.exists(self.path_from_webkit_base('WebKitBuild', 'Dependencies%s' % suffix)) |
| |
| # FIXME: Eventually we should standarize port naming, and make this method smart enough |
| # to use for all port configurations (including architectures, graphics types, etc). |
| def _port_flag_for_scripts(self): |
| # This is overrriden by ports which need a flag passed to scripts to distinguish the use of that port. |
| return None |
| |
| # This is modeled after webkitdirs.pm argumentsForConfiguration() from old-run-webkit-tests |
| def _arguments_for_configuration(self): |
| config_args = [] |
| config_args.append(self._config.flag_for_configuration(self.get_option('configuration'))) |
| # FIXME: We may need to add support for passing --32-bit like old-run-webkit-tests had. |
| port_flag = self._port_flag_for_scripts() |
| if port_flag: |
| config_args.append(port_flag) |
| return config_args |
| |
| def _run_script(self, script_name, args=None, include_configuration_arguments=True, decode_output=True, env=None): |
| run_script_command = [self.path_to_script(script_name)] |
| if include_configuration_arguments: |
| run_script_command.extend(self._arguments_for_configuration()) |
| if args: |
| run_script_command.extend(args) |
| output = self._executive.run_command(run_script_command, cwd=self.webkit_base(), decode_output=decode_output, env=env) |
| _log.debug('Output of %s:\n%s' % (run_script_command, encode_for(output, str) if decode_output else output)) |
| return output |
| |
| def _build_driver(self): |
| environment = self.host.copy_current_environment() |
| env = environment.to_dictionary() |
| |
| # FIXME: We build both DumpRenderTree and WebKitTestRunner for WebKitTestRunner runs because |
| # DumpRenderTree includes TestNetscapePlugin. It should be factored out into its own project. |
| try: |
| self._run_script("build-dumprendertree", args=self._build_driver_flags(), env=env) |
| if self.get_option('webkit_test_runner'): |
| self._run_script("build-webkittestrunner", args=self._build_driver_flags(), env=env) |
| except ScriptError as e: |
| _log.error(e.message_with_output(output_limit=None)) |
| return False |
| return True |
| |
| def _build_api_tests(self, wtf_only=False): |
| environment = self.host.copy_current_environment().to_dictionary() |
| try: |
| self._run_script('build-api-tests', args=(['--wtf-only'] if wtf_only else []) + self._build_driver_flags(), env=environment) |
| except ScriptError as e: |
| _log.error(e.message_with_output(output_limit=None)) |
| return False |
| return True |
| |
| def _build_image_diff(self): |
| environment = self.host.copy_current_environment() |
| env = environment.to_dictionary() |
| try: |
| self._run_script("build-imagediff", env=env) |
| except ScriptError as e: |
| _log.error(e.message_with_output(output_limit=None)) |
| return False |
| return True |
| |
| def _build_driver_flags(self): |
| return [] |
| |
| def test_search_path(self, device_type=None): |
| return self.baseline_search_path(device_type=device_type) |
| |
| def _tests_for_other_platforms(self, device_type=None): |
| # By default we will skip any directory under LayoutTests/platform |
| # that isn't in our baseline search path (this mirrors what |
| # old-run-webkit-tests does in findTestsToRun()). |
| # Note this returns LayoutTests/platform/*, not platform/*/*. |
| entries = self._filesystem.glob(self._webkit_baseline_path('*')) |
| dirs_to_skip = [] |
| for entry in entries: |
| if self._filesystem.isdir(entry) and entry not in self.test_search_path(device_type=device_type): |
| basename = self._filesystem.basename(entry) |
| dirs_to_skip.append('platform/%s' % basename) |
| return dirs_to_skip |
| |
| def _skipped_tests_for_unsupported_features(self, test_list): |
| return [] |
| |
| def _wk2_port_name(self): |
| # By current convention, the WebKit2 name is always mac-wk2, win-wk2, not mac-leopard-wk2, etc, |
| return "%s-wk2" % self.port_name |
| |
| def logging_patterns_to_strip(self): |
| return [] |
| |
| def stderr_patterns_to_strip(self): |
| return [] |
| |
| def test_expectations_file_position(self): |
| # By default baseline search path schema is i.e. port-wk2 -> wk2 -> port -> generic, so port expectations file is at second to last position. |
| return 1 |
| |
| def did_spawn_worker(self, worker_number): |
| # This is overridden by ports that need to do work in the parent process after a worker subprocess is spawned, |
| # such as closing file descriptors that were implicitly cloned to the worker. |
| pass |
| |
| def configuration_for_upload(self, host=None): |
| from webkitpy.results.upload import Upload |
| |
| configuration = self.test_configuration() |
| host = self.host or host |
| |
| if self.get_option('guard_malloc'): |
| style = 'guard-malloc' |
| elif self._config.asan: |
| style = 'asan' |
| else: |
| style = configuration.build_type |
| |
| return Upload.create_configuration( |
| platform=host.platform.os_name, |
| version=str(host.platform.os_version), |
| version_name=host.platform.os_version_name(INTERNAL_TABLE) or host.platform.os_version_name(), |
| architecture=configuration.architecture, |
| style=style, |
| sdk=host.platform.build_version(), |
| flavor=self.get_option('result_report_flavor'), |
| ) |
| |
| @memoized |
| def commits_for_upload(self): |
| from webkitpy.results.upload import Upload |
| |
| repos = {} |
| if port_config.apple_additions() and getattr(port_config.apple_additions(), 'repos', False): |
| repos = port_config.apple_additions().repos() |
| |
| up = os.path.dirname |
| repos['webkit'] = up(up(up(up(up(os.path.abspath(__file__)))))) |
| |
| commits = [] |
| for repo_id, path in repos.items(): |
| scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(path) |
| commits.append(Upload.create_commit( |
| repository_id=repo_id, |
| id=scm.native_revision(path), |
| branch=scm.native_branch(path), |
| )) |
| return commits |
