Tools/Scripts/webkitpy/browserperfdash/browserperfdash_runner.py

#!/usr/bin/env python
#
# Copyright (C) 2018 Igalia S.L.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


import argparse
import ConfigParser
import json
import logging
import os
import platform
import sys
import tempfile
import time
import urllib

from webkitpy.benchmark_runner.benchmark_runner import BenchmarkRunner
from webkitpy.benchmark_runner.browser_driver.browser_driver_factory import BrowserDriverFactory
from webkitpy.benchmark_runner.webdriver_benchmark_runner import WebDriverBenchmarkRunner
from webkitpy.benchmark_runner.webserver_benchmark_runner import WebServerBenchmarkRunner
from webkitpy.benchmark_runner.run_benchmark import default_browser, default_platform, benchmark_runner_subclasses

_log = logging.getLogger(__name__)


def parse_args():
    parser = argparse.ArgumentParser(description='Automate the browser based performance benchmarks')
    # browserperfdash specific arguments.
    parser.add_argument('--config-file', dest='config_file', default=None, required=True, help='Configuration file for sending the results to the performance dashboard server(s).')
    parser.add_argument('--browser-version', dest='browser_version', default=None, required=True, help='A string that identifies the browser version.')
    # arguments shared with run-benchmark.
    parser.add_argument('--build-directory', dest='buildDir', help='Path to the browser executable. e.g. WebKitBuild/Release/')
    parser.add_argument('--platform', dest='platform', default=default_platform(), choices=BrowserDriverFactory.available_platforms())
    parser.add_argument('--browser', dest='browser', default=default_browser(), choices=BrowserDriverFactory.available_browsers())
    parser.add_argument('--driver', default=WebServerBenchmarkRunner.name, choices=benchmark_runner_subclasses.keys(), help='Use the specified benchmark driver. Defaults to %s.' % WebServerBenchmarkRunner.name)
    parser.add_argument('--local-copy', dest='localCopy', help='Path to a local copy of the benchmark. e.g. PerformanceTests/SunSpider/')
    parser.add_argument('--count', dest='countOverride', type=int, help='Number of times to run the benchmark. e.g. 5')
    mutual_group = parser.add_mutually_exclusive_group(required=True)
    mutual_group.add_argument('--plan', dest='plan', help='Benchmark plan to run. e.g. speedometer, jetstream')
    mutual_group.add_argument('--allplans', action='store_true', help='Run all available benchmark plans sequentially')
    args = parser.parse_args()
    return args


class BrowserPerfDashRunner(object):
    name = 'browserperfdash_runner'

    def __init__(self, args):
        self._args = args
        self._plandir = os.path.abspath(BenchmarkRunner.plan_directory())
        if not os.path.isdir(self._plandir):
            raise Exception('Cant find plandir: {plandir}'.format(plandir=self._plandir))
        self._parse_config_file(self._args.config_file)
        # This is the dictionary that will be sent as the HTTP POST request that browserperfdash expects
        # (as defined in https://github.com/Igalia/browserperfdash/blob/master/docs/design-document.md)
        # - The bot_* data its obtained from the config file
        # - the browser_* data is given at startup time via command-line arguments
        # - The test_* data is generated after each test run.
        self._result_data = {'bot_id': None,
                             'bot_password': None,
                             'browser_id': self._args.browser,
                             'browser_version': self._args.browser_version,
                             'test_id': None,
                             'test_version': None,
                             'test_data': None}

    def _parse_config_file(self, config_file):
        if not os.path.isfile(config_file):
            raise Exception('Can not open config file for uploading results at: {config_file}'.format(config_file=config_file))
        self._config_parser = ConfigParser.RawConfigParser()
        self._config_parser.read(config_file)

    def _get_test_version_string(self, plan_name):
        plan_file_path = os.path.join(self._plandir, '{plan_name}.plan'.format(plan_name=plan_name))
        with open(plan_file_path, 'r') as plan_fd:
            plan_json = json.load(plan_fd)
            if 'svn_source' in plan_json:
                return plan_json['svn_source']
            elif 'remote_archive' in plan_json:
                return plan_json['remote_archive']
            else:
                raise Exception('Can not upload results because is not possible to determine test version for test plan: {plan_file_path}'.format(plan_file_path=plan_file_path))

    def _get_test_data_json_string(self, temp_result_file):
        temp_result_file.flush()
        temp_result_file.seek(0)
        temp_result_json = json.load(temp_result_file)
        if 'debugOutput' in temp_result_json:
            del temp_result_json['debugOutput']
        return json.dumps(temp_result_json)

    def _upload_result(self):
        upload_failed = False
        for server in self._config_parser.sections():
            self._result_data['bot_id'] = self._config_parser.get(server, 'bot_id')
            self._result_data['bot_password'] = self._config_parser.get(server, 'bot_password')
            post_data = urllib.urlencode(self._result_data)
            post_url = self._config_parser.get(server, 'post_url')
            try:
                post_request = urllib.urlopen(post_url, post_data)
                if post_request.getcode() == 200:
                    _log.info('Sucesfully uploaded results to server {server_name} for test {test_name} and browser {browser_name} version {browser_version}'.format(
                               server_name=server, test_name=self._result_data['test_id'], browser_name=self._result_data['browser_id'], browser_version=self._result_data['browser_version']))
                else:
                    upload_failed = True
                    _log.error('The server {server_name} returned an error code: {http_error}'.format(server_name=server, http_error=post_request.getcode()))
                    _log.error('The error text from the server {server_name} was: "{error_text}"'.format(server_name=server, error_text=post_request.read()))
            except Exception as e:
                upload_failed = True
                _log.error('Exception while trying to upload results to server {server_name}'.format(server_name=server))
                _log.error(e)
        return not upload_failed

    def run(self):
        failed = []
        worked = []
        skipped = []
        planlist = []
        if self._args.plan:
            if not os.path.isfile(os.path.join(self._plandir, '{plan_name}.plan'.format(plan_name=self._args.plan))):
                raise Exception('Cant find a file named {plan_name}.plan in directory {plan_directory}'.format(plan_name=self._args.plan, plan_directory=self._plandir))
            planlist = [self._args.plan]
        elif self._args.allplans:
            planlist = BenchmarkRunner.available_plans()
            skippedfile = os.path.join(self._plandir, 'Skipped')
            if not planlist:
                raise Exception('Cant find any plan in the directory {plan_directory}'.format(plan_directory=self._plandir))
            if os.path.isfile(skippedfile):
                skipped = [line.strip() for line in open(skippedfile) if not line.startswith('#') and len(line) > 1]

        if len(planlist) < 1:
            _log.error('No benchmarks plans available to run in directory {plan_directory}'.format(plan_directory=self._plandir))
            return 1

        _log.info('Starting benchmark for browser {browser} and version {browser_version}'.format(browser=self._args.browser, browser_version=self._args.browser_version))

        iteration_count = 0
        for plan in sorted(planlist):
            iteration_count += 1
            if plan in skipped:
                _log.info('Skipping benchmark plan: {plan_name} because is listed on the Skipped file [benchmark {iteration} of {total}]'.format(plan_name=plan, iteration=iteration_count, total=len(planlist)))
                continue
            _log.info('Starting benchmark plan: {plan_name} [benchmark {iteration} of {total}]'.format(plan_name=plan, iteration=iteration_count, total=len(planlist)))
            try:
                # Run test and save test info
                with tempfile.NamedTemporaryFile() as temp_result_file:
                    benchmark_runner_class = benchmark_runner_subclasses[self._args.driver]
                    runner = benchmark_runner_class(plan, self._args.localCopy, self._args.countOverride, self._args.buildDir, temp_result_file.name, self._args.platform, self._args.browser, None)
                    runner.execute()
                    _log.info('Finished benchmark plan: {plan_name}'.format(plan_name=plan))
                    # Fill test info for upload
                    self._result_data['test_id'] = plan
                    self._result_data['test_version'] = self._get_test_version_string(plan)
                    # Fill obtained test results for upload
                    self._result_data['test_data'] = self._get_test_data_json_string(temp_result_file)

                # Now upload data to server(s)
                _log.info('Uploading results for plan: {plan_name} and browser {browser} version {browser_version}'.format(plan_name=plan, browser=self._args.browser, browser_version=self._args.browser_version))
                if self._upload_result():
                    worked.append(plan)
                else:
                    failed.append(plan)

            except KeyboardInterrupt:
                raise
            except:
                failed.append(plan)
                _log.exception('Error running benchmark plan: {plan_name}'.format(plan_name=plan))

        if len(worked) > 0:
            _log.info('The following benchmark plans have been upload succesfully: {list_plan_worked}'.format(list_plan_worked=worked))

        if len(failed) > 0:
            _log.error('The following benchmark plans have failed to run or to upload: {list_plan_failed}'.format(list_plan_failed=failed))
            return len(failed)

        return 0


def format_logger(logger):
    logger.setLevel(logging.INFO)
    ch = logging.StreamHandler()
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)


def main():
    perfdashrunner = BrowserPerfDashRunner(parse_args())
    return perfdashrunner.run()