| #!/usr/bin/env python |
| # |
| # Copyright (C) 2018 Igalia S.L. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are met: |
| # |
| # 1. Redistributions of source code must retain the above copyright notice, this |
| # list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright notice, |
| # this list of conditions and the following disclaimer in the documentation |
| # and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| |
| import argparse |
| import ConfigParser |
| import json |
| import logging |
| import os |
| import platform |
| import sys |
| import tempfile |
| import time |
| import urllib |
| |
| from webkitpy.benchmark_runner.benchmark_runner import BenchmarkRunner |
| from webkitpy.benchmark_runner.browser_driver.browser_driver_factory import BrowserDriverFactory |
| from webkitpy.benchmark_runner.webdriver_benchmark_runner import WebDriverBenchmarkRunner |
| from webkitpy.benchmark_runner.webserver_benchmark_runner import WebServerBenchmarkRunner |
| from webkitpy.benchmark_runner.run_benchmark import default_browser, default_platform, benchmark_runner_subclasses |
| |
| _log = logging.getLogger(__name__) |
| |
| |
| def parse_args(): |
| parser = argparse.ArgumentParser(description='Automate the browser based performance benchmarks') |
| # browserperfdash specific arguments. |
| parser.add_argument('--config-file', dest='config_file', default=None, required=True, help='Configuration file for sending the results to the performance dashboard server(s).') |
| parser.add_argument('--browser-version', dest='browser_version', default=None, required=True, help='A string that identifies the browser version.') |
| # arguments shared with run-benchmark. |
| parser.add_argument('--build-directory', dest='buildDir', help='Path to the browser executable. e.g. WebKitBuild/Release/') |
| parser.add_argument('--platform', dest='platform', default=default_platform(), choices=BrowserDriverFactory.available_platforms()) |
| parser.add_argument('--browser', dest='browser', default=default_browser(), choices=BrowserDriverFactory.available_browsers()) |
| parser.add_argument('--driver', default=WebServerBenchmarkRunner.name, choices=benchmark_runner_subclasses.keys(), help='Use the specified benchmark driver. Defaults to %s.' % WebServerBenchmarkRunner.name) |
| parser.add_argument('--local-copy', dest='localCopy', help='Path to a local copy of the benchmark. e.g. PerformanceTests/SunSpider/') |
| parser.add_argument('--count', dest='countOverride', type=int, help='Number of times to run the benchmark. e.g. 5') |
| mutual_group = parser.add_mutually_exclusive_group(required=True) |
| mutual_group.add_argument('--plan', dest='plan', help='Benchmark plan to run. e.g. speedometer, jetstream') |
| mutual_group.add_argument('--allplans', action='store_true', help='Run all available benchmark plans sequentially') |
| args = parser.parse_args() |
| return args |
| |
| |
| class BrowserPerfDashRunner(object): |
| name = 'browserperfdash_runner' |
| |
| def __init__(self, args): |
| self._args = args |
| self._plandir = os.path.abspath(BenchmarkRunner.plan_directory()) |
| if not os.path.isdir(self._plandir): |
| raise Exception('Cant find plandir: {plandir}'.format(plandir=self._plandir)) |
| self._parse_config_file(self._args.config_file) |
| # This is the dictionary that will be sent as the HTTP POST request that browserperfdash expects |
| # (as defined in https://github.com/Igalia/browserperfdash/blob/master/docs/design-document.md) |
| # - The bot_* data its obtained from the config file |
| # - the browser_* data is given at startup time via command-line arguments |
| # - The test_* data is generated after each test run. |
| self._result_data = {'bot_id': None, |
| 'bot_password': None, |
| 'browser_id': self._args.browser, |
| 'browser_version': self._args.browser_version, |
| 'test_id': None, |
| 'test_version': None, |
| 'test_data': None} |
| |
| def _parse_config_file(self, config_file): |
| if not os.path.isfile(config_file): |
| raise Exception('Can not open config file for uploading results at: {config_file}'.format(config_file=config_file)) |
| self._config_parser = ConfigParser.RawConfigParser() |
| self._config_parser.read(config_file) |
| |
| def _get_test_version_string(self, plan_name): |
| plan_file_path = os.path.join(self._plandir, '{plan_name}.plan'.format(plan_name=plan_name)) |
| with open(plan_file_path, 'r') as plan_fd: |
| plan_json = json.load(plan_fd) |
| if 'svn_source' in plan_json: |
| return plan_json['svn_source'] |
| elif 'remote_archive' in plan_json: |
| return plan_json['remote_archive'] |
| else: |
| raise Exception('Can not upload results because is not possible to determine test version for test plan: {plan_file_path}'.format(plan_file_path=plan_file_path)) |
| |
| def _get_test_data_json_string(self, temp_result_file): |
| temp_result_file.flush() |
| temp_result_file.seek(0) |
| temp_result_json = json.load(temp_result_file) |
| if 'debugOutput' in temp_result_json: |
| del temp_result_json['debugOutput'] |
| return json.dumps(temp_result_json) |
| |
| def _upload_result(self): |
| upload_failed = False |
| for server in self._config_parser.sections(): |
| self._result_data['bot_id'] = self._config_parser.get(server, 'bot_id') |
| self._result_data['bot_password'] = self._config_parser.get(server, 'bot_password') |
| post_data = urllib.urlencode(self._result_data) |
| post_url = self._config_parser.get(server, 'post_url') |
| try: |
| post_request = urllib.urlopen(post_url, post_data) |
| if post_request.getcode() == 200: |
| _log.info('Sucesfully uploaded results to server {server_name} for test {test_name} and browser {browser_name} version {browser_version}'.format( |
| server_name=server, test_name=self._result_data['test_id'], browser_name=self._result_data['browser_id'], browser_version=self._result_data['browser_version'])) |
| else: |
| upload_failed = True |
| _log.error('The server {server_name} returned an error code: {http_error}'.format(server_name=server, http_error=post_request.getcode())) |
| _log.error('The error text from the server {server_name} was: "{error_text}"'.format(server_name=server, error_text=post_request.read())) |
| except Exception as e: |
| upload_failed = True |
| _log.error('Exception while trying to upload results to server {server_name}'.format(server_name=server)) |
| _log.error(e) |
| return not upload_failed |
| |
| def run(self): |
| failed = [] |
| worked = [] |
| skipped = [] |
| planlist = [] |
| if self._args.plan: |
| if not os.path.isfile(os.path.join(self._plandir, '{plan_name}.plan'.format(plan_name=self._args.plan))): |
| raise Exception('Cant find a file named {plan_name}.plan in directory {plan_directory}'.format(plan_name=self._args.plan, plan_directory=self._plandir)) |
| planlist = [self._args.plan] |
| elif self._args.allplans: |
| planlist = BenchmarkRunner.available_plans() |
| skippedfile = os.path.join(self._plandir, 'Skipped') |
| if not planlist: |
| raise Exception('Cant find any plan in the directory {plan_directory}'.format(plan_directory=self._plandir)) |
| if os.path.isfile(skippedfile): |
| skipped = [line.strip() for line in open(skippedfile) if not line.startswith('#') and len(line) > 1] |
| |
| if len(planlist) < 1: |
| _log.error('No benchmarks plans available to run in directory {plan_directory}'.format(plan_directory=self._plandir)) |
| return 1 |
| |
| _log.info('Starting benchmark for browser {browser} and version {browser_version}'.format(browser=self._args.browser, browser_version=self._args.browser_version)) |
| |
| iteration_count = 0 |
| for plan in sorted(planlist): |
| iteration_count += 1 |
| if plan in skipped: |
| _log.info('Skipping benchmark plan: {plan_name} because is listed on the Skipped file [benchmark {iteration} of {total}]'.format(plan_name=plan, iteration=iteration_count, total=len(planlist))) |
| continue |
| _log.info('Starting benchmark plan: {plan_name} [benchmark {iteration} of {total}]'.format(plan_name=plan, iteration=iteration_count, total=len(planlist))) |
| try: |
| # Run test and save test info |
| with tempfile.NamedTemporaryFile() as temp_result_file: |
| benchmark_runner_class = benchmark_runner_subclasses[self._args.driver] |
| runner = benchmark_runner_class(plan, self._args.localCopy, self._args.countOverride, self._args.buildDir, temp_result_file.name, self._args.platform, self._args.browser) |
| runner.execute() |
| _log.info('Finished benchmark plan: {plan_name}'.format(plan_name=plan)) |
| # Fill test info for upload |
| self._result_data['test_id'] = plan |
| self._result_data['test_version'] = self._get_test_version_string(plan) |
| # Fill obtained test results for upload |
| self._result_data['test_data'] = self._get_test_data_json_string(temp_result_file) |
| |
| # Now upload data to server(s) |
| _log.info('Uploading results for plan: {plan_name} and browser {browser} version {browser_version}'.format(plan_name=plan, browser=self._args.browser, browser_version=self._args.browser_version)) |
| if self._upload_result(): |
| worked.append(plan) |
| else: |
| failed.append(plan) |
| |
| except KeyboardInterrupt: |
| raise |
| except: |
| failed.append(plan) |
| _log.exception('Error running benchmark plan: {plan_name}'.format(plan_name=plan)) |
| |
| if len(worked) > 0: |
| _log.info('The following benchmark plans have been upload succesfully: {list_plan_worked}'.format(list_plan_worked=worked)) |
| |
| if len(failed) > 0: |
| _log.error('The following benchmark plans have failed to run or to upload: {list_plan_failed}'.format(list_plan_failed=failed)) |
| return len(failed) |
| |
| return 0 |
| |
| |
| def format_logger(logger): |
| logger.setLevel(logging.INFO) |
| ch = logging.StreamHandler() |
| formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') |
| ch.setFormatter(formatter) |
| logger.addHandler(ch) |
| |
| |
| def main(): |
| perfdashrunner = BrowserPerfDashRunner(parse_args()) |
| return perfdashrunner.run() |