blob: 008924c107187e1feb5030cde40b1e86fc770f6c [file] [log] [blame]
#!/usr/bin/env python
# Copyright (C) 2013 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
import argparse
import subprocess
import sys
import tempfile
def main():
parser = argparse.ArgumentParser(description="Compare layout test results between two configurations of WebKit.")
parser.add_argument("--comparison", action="store", choices=["remote-layer-tree", "accelerated-drawing", "webkit2"], required=True, help="configuration to test")
parser.add_argument("-2", "--webkit2", action="store_true", default=False, help="use WebKit2 always (some configurations may force this)")
parser.add_argument('tests', metavar='TESTS', type=str, help='list of LayoutTest subdirectories to run', default=None, nargs="*")
args = parser.parse_args()
force_webkit2 = args.webkit2
if args.comparison == "remote-layer-tree":
print "Forcing use of WebKit2, as the remote layer tree depends on WebKit2."
force_webkit2 = True
if args.comparison == "webkit2" and force_webkit2:
print "It doesn't make sense to test WebKit1 vs. WebKit2 *and* force WebKit2 on."
sys.exit(1)
configuration_flag = flag_for_comparison(args.comparison)
results_directory = tempfile.mkdtemp(prefix="webkit-comparison-results-", suffix="-" + args.comparison)
# Run first in the default configuration, generating new baselines, then run
# again against those results in the configuration being tested.
run_webkit_tests(tests=args.tests, results_directory=results_directory, use_webkit2=force_webkit2, additional_arguments=["--no-show-results", "--new-baseline"])
run_webkit_tests(tests=args.tests, results_directory=results_directory, additional_arguments=[configuration_flag], use_webkit2=force_webkit2)
def flag_for_comparison(comparison_name):
if comparison_name == "remote-layer-tree":
return "--remote-layer-tree"
if comparison_name == "accessibility-isolated-tree":
return "--accessibility-isolated-tree"
if comparison_name == "accelerated-drawing":
return "--accelerated-drawing"
if comparison_name == "webkit2":
return "-2"
print "Unknown comparison:", comparison_name
sys.exit(1)
def run_webkit_tests(tests, results_directory, use_webkit2, additional_arguments=[]):
if use_webkit2:
use_webkit2_arguments = ["-2"]
else:
use_webkit2_arguments = []
subprocess.call(["run-webkit-tests", "-p", "--force", "--treat-ref-tests-as-pixel-tests", "--additional-platform-directory=" + results_directory, "--run-singly", "--no-retry-failures"] + use_webkit2_arguments + additional_arguments + tests)
if __name__ == '__main__':
main()