| # Copyright (c) 2014, Canon Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # 3. Neither the name of Canon Inc. nor the names of |
| # its contributors may be used to endorse or promote products derived |
| # from this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR |
| # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """ |
| This script downloads W3C test repositories. |
| """ |
| |
| import json |
| import logging |
| |
| from webkitpy.common.system.filesystem import FileSystem |
| from webkitpy.common.webkit_finder import WebKitFinder |
| from webkitpy.common.checkout.scm.git import Git |
| |
| _log = logging.getLogger(__name__) |
| |
| |
| class TestDownloader(object): |
| |
| @staticmethod |
| def default_options(): |
| options = type('', (), {}) |
| options.fetch = True |
| options.verbose = False |
| options.import_all = False |
| return options |
| |
| @staticmethod |
| def load_test_repositories(filesystem=FileSystem()): |
| webkit_finder = WebKitFinder(filesystem) |
| test_repositories_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'TestRepositories') |
| return json.loads(filesystem.read_text_file(test_repositories_path)) |
| |
| def __init__(self, repository_directory, host, options): |
| self._options = options |
| self._host = host |
| self._filesystem = host.filesystem |
| self._test_suites = [] |
| |
| self.repository_directory = repository_directory |
| |
| self.test_repositories = self.load_test_repositories(self._filesystem) |
| |
| self.paths_to_skip = [] |
| self.paths_to_import = [] |
| for test_repository in self.test_repositories: |
| self.paths_to_skip.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_skip']]) |
| self.paths_to_import.extend([self._filesystem.join(test_repository['name'], path) for path in test_repository['paths_to_import']]) |
| |
| webkit_finder = WebKitFinder(self._filesystem) |
| self.import_expectations_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'import-expectations.json') |
| if not self._filesystem.isfile(self.import_expectations_path): |
| _log.warning('Unable to read import expectation file: %s' % self.import_expectations_path) |
| if not self._options.import_all: |
| self._init_paths_from_expectations() |
| |
| def git(self, test_repository): |
| return Git(test_repository, None, executive=self._host.executive, filesystem=self._filesystem) |
| |
| def checkout_test_repository(self, revision, url, directory): |
| git = None |
| if not self._filesystem.exists(directory): |
| _log.info('Cloning %s into %s...' % (url, directory)) |
| Git.clone(url, directory, self._host.executive) |
| git = self.git(directory) |
| elif self._options.fetch is True: |
| git = self.git(directory) |
| _log.info('Fetching %s...' % url) |
| git.fetch() |
| else: |
| git = self.git(directory) |
| _log.info('Checking out revision ' + revision) |
| git.checkout(revision, not self._options.verbose) |
| |
| def _init_paths_from_expectations(self): |
| import_lines = json.loads(self._filesystem.read_text_file(self.import_expectations_path)) |
| for path, policy in import_lines.items(): |
| if policy == 'skip': |
| self.paths_to_skip.append(path) |
| elif policy == 'import': |
| self.paths_to_import.append(path) |
| else: |
| _log.warning('Problem reading import lines ' + path) |
| |
| def update_import_expectations(self, test_paths): |
| import_lines = json.loads(self._filesystem.read_text_file(self.import_expectations_path)) |
| for path in test_paths: |
| import_lines[path] = 'import' |
| self._filesystem.write_text_file(self.import_expectations_path, json.dumps(import_lines, sort_keys=True, indent=4)) |
| |
| def _add_test_suite_paths(self, test_paths, directory, webkit_path): |
| for name in self._filesystem.listdir(directory): |
| original_path = self._filesystem.join(webkit_path, name) |
| if not name.startswith('.') and not original_path in self.paths_to_skip: |
| test_paths.append(original_path) |
| |
| def _empty_directory(self, directory): |
| if self._filesystem.exists(directory): |
| self._filesystem.rmtree(directory) |
| self._filesystem.maybe_make_directory(directory) |
| |
| def copy_tests(self, destination_directory, test_paths): |
| for test_repository in self.test_repositories: |
| self._empty_directory(self._filesystem.join(destination_directory, test_repository['name'])) |
| |
| copy_paths = [] |
| if test_paths: |
| for path in test_paths: |
| copy_paths.append(path) |
| for path in self.paths_to_import: |
| copy_paths.append(path) |
| else: |
| for test_repository in self.test_repositories: |
| self._add_test_suite_paths(copy_paths, self._filesystem.join(self.repository_directory, test_repository['name']), test_repository['name']) |
| # Handling of tests marked as [ Pass ] in expectations file. |
| for path in self.paths_to_import: |
| if not path in copy_paths: |
| copy_paths.append(path) |
| |
| def longest_path(filesystem, paths): |
| longest_matching_path = "" |
| for path in paths: |
| if path.startswith(longest_matching_path): |
| longest_matching_path = path |
| return longest_matching_path |
| |
| def should_copy_dir(filesystem, directory): |
| relative_path = self._filesystem.relpath(directory, self.repository_directory) |
| if relative_path == ".": |
| return True |
| |
| potential_copy_paths = [copy_directory for copy_directory in copy_paths if relative_path.startswith(copy_directory)] |
| if (not potential_copy_paths): |
| return False |
| |
| potential_skip_paths = [skip_directory for skip_directory in self.paths_to_skip if relative_path.startswith(skip_directory)] |
| if (not potential_skip_paths): |
| return True |
| |
| longest_copy_path = longest_path(filesystem, potential_copy_paths) |
| longest_skip_path = longest_path(filesystem, potential_skip_paths) |
| return longest_copy_path.startswith(longest_skip_path) |
| |
| # Compute directories for which we should copy direct children |
| directories_to_copy = self._filesystem.dirs_under(self.repository_directory, should_copy_dir) |
| |
| def should_copy_file(filesystem, dirname, filename): |
| full_path = filesystem.join(dirname, filename) |
| relative_path = self._filesystem.relpath(full_path, self.repository_directory) |
| if relative_path in copy_paths: |
| return True |
| if relative_path in self.paths_to_skip: |
| return False |
| return dirname in directories_to_copy |
| |
| for source_path in self._filesystem.files_under(self.repository_directory, file_filter=should_copy_file): |
| destination_path = self._filesystem.join(destination_directory, self._filesystem.relpath(source_path, self.repository_directory)) |
| self._filesystem.maybe_make_directory(self._filesystem.dirname(destination_path)) |
| self._filesystem.copyfile(source_path, destination_path) |
| |
| def _git_submodules_description(self, test_repository): |
| directory = self._filesystem.join(self.repository_directory, test_repository['name']) |
| |
| git = self.git(directory) |
| git.init_submodules() |
| |
| submodules = [] |
| submodules_status = [line.strip().split(' ') for line in git.submodules_status().splitlines()] |
| for status in submodules_status: |
| version = status[0] |
| path = status[1].split('/') |
| |
| url = self.git(self._filesystem.join(directory, status[1])).origin_url() |
| if not url.startswith('https://github.com/'): |
| _log.warning('Submodule %s (%s) is not hosted on github' % (status[1], url)) |
| _log.warning('Please ensure that generated URL points to an archive of the module or manually edit its value after the import') |
| url = url[:-4] # to remove .git |
| |
| submodule = {} |
| submodule['path'] = path |
| submodule['url'] = url + '/archive/' + version + '.tar.gz' |
| submodule['url_subpath'] = url.split('/').pop() + '-' + version |
| submodules.append(submodule) |
| |
| git.deinit_submodules() |
| return submodules |
| |
| def generate_git_submodules_description(self, test_repository, filepath): |
| self._filesystem.write_text_file(filepath, json.dumps(self._git_submodules_description(test_repository), sort_keys=True, indent=4)) |
| |
| def generate_gitignore(self, test_repository, destination_directory): |
| rules = [] |
| for submodule in self._git_submodules_description(test_repository): |
| path = list(submodule['path']) |
| path.insert(0, '') |
| rules.append('/'.join(path[:-1]) + '/' + path[-1] + '/') |
| rules.append('/'.join(path[:-1]) + '/.' + path[-1] + '.url') |
| self._filesystem.write_text_file(self._filesystem.join(destination_directory, test_repository['name'], '.gitignore'), '\n'.join(rules)) |
| |
| def clone_tests(self, use_tip_of_tree=False): |
| for test_repository in self.test_repositories: |
| self.checkout_test_repository(test_repository['revision'] if not use_tip_of_tree else 'origin/master', test_repository['url'], self._filesystem.join(self.repository_directory, test_repository['name'])) |
| |
| def download_tests(self, destination_directory, test_paths=[], use_tip_of_tree=False): |
| self.clone_tests(use_tip_of_tree) |
| self.copy_tests(destination_directory, test_paths) |
