blob: 9cb1f2c66c5f68c53f42dcb4649e8caccc607933 [file] [log] [blame]
#!/usr/bin/env python3
import argparse
import json
import logging
import os
import sys
import time
from webkitpy.benchmark_runner.browser_driver.browser_driver_factory import BrowserDriverFactory
from webkitpy.benchmark_runner.benchmark_runner import BenchmarkRunner
from webkitpy.benchmark_runner.webdriver_benchmark_runner import WebDriverBenchmarkRunner
from webkitpy.benchmark_runner.webserver_benchmark_runner import WebServerBenchmarkRunner
_log = logging.getLogger(__name__)
benchmark_runner_subclasses = {
WebDriverBenchmarkRunner.name: WebDriverBenchmarkRunner,
WebServerBenchmarkRunner.name: WebServerBenchmarkRunner,
}
def default_platform():
if sys.platform.startswith('linux'):
return 'linux'
return 'osx'
def default_browser():
if sys.platform.startswith('linux'):
return 'minibrowser-gtk'
return 'safari'
def default_diagnose_dir():
return '/tmp/run-benchmark-diagnostics-{}/'.format(int(time.time()))
def config_argument_parser():
diagnose_directory = default_diagnose_dir()
parser = argparse.ArgumentParser(description='Run browser based performance benchmarks. To run a single benchmark in the recommended way, use run-benchmark --plan. To see the vailable benchmarks, use run-benchmark --list-plans. This script passes through the __XPC variables in its environment to the Safari process.')
mutual_group = parser.add_mutually_exclusive_group(required=True)
mutual_group.add_argument('--plan', help='Run a specific benchmark plan (e.g. speedometer, jetstream).')
mutual_group.add_argument('--list-plans', action='store_true', help='List all available benchmark plans.')
mutual_group.add_argument('--allplans', action='store_true', help='Run all available benchmark plans in order.')
mutual_group.add_argument('--read-results-json', dest='json_file', help='Instead of running a benchmark, format the output saved in JSON_FILE.')
parser.add_argument('--output-file', default=None, help='Save detailed results to OUTPUT in JSON format. By default, results will not be saved.')
parser.add_argument('--count', type=int, help='Number of times to run the benchmark (e.g. 5).')
parser.add_argument('--driver', default=WebServerBenchmarkRunner.name, choices=list(benchmark_runner_subclasses.keys()), help='Use the specified benchmark driver. Defaults to %s.' % WebServerBenchmarkRunner.name)
parser.add_argument('--browser', default=default_browser(), choices=BrowserDriverFactory.available_browsers(), help='Browser to run the nechmark in. Defaults to %s.' % default_browser())
parser.add_argument('--platform', default=default_platform(), choices=BrowserDriverFactory.available_platforms(), help='Platform that this script is running on. Defaults to %s.' % default_platform())
parser.add_argument('--local-copy', help='Path to a local copy of the benchmark (e.g. PerformanceTests/SunSpider/).')
parser.add_argument('--device-id', default=None, help='Undocumented option for mobile device testing.')
parser.add_argument('--debug', action='store_true', help='Enable debug logging.')
parser.add_argument('--diagnose-directory', dest='diagnose_dir', default=diagnose_directory, help='Directory for storing diagnose information on test failure. Defaults to {}.'.format(diagnose_directory))
parser.add_argument('--no-adjust-unit', dest='scale_unit', action='store_false', help="Don't convert to scientific notation.")
parser.add_argument('--show-iteration-values', dest='show_iteration_values', action='store_true', help="Show the measured value for each iteration in addition to averages.")
parser.add_argument('--generate-profiles', dest='generate_profiles', action='store_true', help="Collect LLVM profiles for PGO, and copy them to the diagnose directory.")
group = parser.add_mutually_exclusive_group()
group.add_argument('--browser-path', help='Specify the path to a non-default copy of the target browser as a path to the .app.')
group.add_argument('--build-directory', dest='build_dir', help='Path to the browser executable (e.g. WebKitBuild/Release/).')
return parser
# FIXME: Remove default arguments when all dependent scripts adopt this change.
def parse_args(parser=None):
if parser is None:
parser = config_argument_parser()
args = parser.parse_args()
if args.debug:
_log.setLevel(logging.DEBUG)
_log.debug('Initializing program with following parameters')
_log.debug('\toutput file name\t: %s' % args.output_file)
_log.debug('\tbuild directory\t: %s' % args.build_dir)
_log.debug('\tplan name\t: %s', args.plan)
if args.generate_profiles and not os.path.isdir(args.diagnose_dir):
_log.error("No diagnose directory to dump profiles to: {}".format(args.diagnose_dir))
exit()
if args.generate_profiles and args.platform is not 'osx':
_log.error("Profile generation is currently only supported on macOS.")
exit()
return args
def run_benchmark_plan(args, plan):
benchmark_runner_class = benchmark_runner_subclasses[args.driver]
runner = benchmark_runner_class(plan,
args.local_copy, args.count, args.build_dir, args.output_file,
args.platform, args.browser, args.browser_path, args.scale_unit,
args.show_iteration_values, args.device_id, args.diagnose_dir,
args.diagnose_dir if args.generate_profiles else None)
runner.execute()
def list_benchmark_plans():
print("Available benchmark plans: ")
for plan in BenchmarkRunner.available_plans():
print("\t%s" % plan)
def start(args):
if args.json_file:
results_json = json.load(open(args.json_file, 'r'))
if 'debugOutput' in results_json:
del results_json['debugOutput']
BenchmarkRunner.show_results(results_json, args.scale_unit, args.show_iteration_values)
return
if args.allplans:
failed = []
skipped = []
planlist = BenchmarkRunner.available_plans()
skippedfile = os.path.join(BenchmarkRunner.plan_directory(), 'Skipped')
if not planlist:
raise Exception('Cant find any .plan file in directory %s' % BenchmarkRunner.plan_directory())
if os.path.isfile(skippedfile):
skipped = [line.strip() for line in open(skippedfile) if not line.startswith('#') and len(line) > 1]
for plan in sorted(planlist):
if plan in skipped:
_log.info('Skipping benchmark plan: %s because is listed on the Skipped file' % plan)
continue
_log.info('Starting benchmark plan: %s' % plan)
try:
run_benchmark_plan(args, plan)
_log.info('Finished benchmark plan: %s' % plan)
except KeyboardInterrupt:
raise
except:
failed.append(plan)
_log.exception('Error running benchmark plan: %s' % plan)
if failed:
_log.error('The following benchmark plans have failed: %s' % failed)
return len(failed)
if args.list_plans:
list_benchmark_plans()
return
run_benchmark_plan(args, args.plan)
def format_logger(logger):
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
def main():
return start(parse_args(config_argument_parser()))