Tools/Scripts/compare-webkit-configurations

#!/usr/bin/env python

# Copyright (C) 2013 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.

import argparse
import subprocess
import sys
import tempfile

def main():
    parser = argparse.ArgumentParser(description="Compare layout test results between two configurations of WebKit.")

    parser.add_argument("--comparison", action="store", choices=["remote-layer-tree", "accelerated-drawing", "webkit2"], required=True, help="configuration to test")
    parser.add_argument("-2", "--webkit2", action="store_true", default=False, help="use WebKit2 always (some configurations may force this)")
    parser.add_argument('tests', metavar='TESTS', type=str, help='list of LayoutTest subdirectories to run', default=None, nargs="*")

    args = parser.parse_args()

    force_webkit2 = args.webkit2
    if args.comparison == "remote-layer-tree":
        print "Forcing use of WebKit2, as the remote layer tree depends on WebKit2."
        force_webkit2 = True

    if args.comparison == "webkit2" and force_webkit2:
        print "It doesn't make sense to test WebKit1 vs. WebKit2 *and* force WebKit2 on."
        sys.exit(1)

    configuration_flag = flag_for_comparison(args.comparison)

    results_directory = tempfile.mkdtemp(prefix="webkit-comparison-results-", suffix="-" + args.comparison)

    # Run first in the default configuration, generating new baselines, then run
    # again against those results in the configuration being tested.
    run_webkit_tests(tests=args.tests, results_directory=results_directory, use_webkit2=force_webkit2, additional_arguments=["--no-show-results", "--new-baseline"])
    run_webkit_tests(tests=args.tests, results_directory=results_directory, additional_arguments=[configuration_flag], use_webkit2=force_webkit2)

def flag_for_comparison(comparison_name):
    if comparison_name == "remote-layer-tree":
        return "--remote-layer-tree"
    if comparison_name == "accessibility-isolated-tree":
        return "--accessibility-isolated-tree"
    if comparison_name == "accelerated-drawing":
        return "--accelerated-drawing"
    if comparison_name == "webkit2":
        return "-2"

    print "Unknown comparison:", comparison_name
    sys.exit(1)

def run_webkit_tests(tests, results_directory, use_webkit2, additional_arguments=[]):
    if use_webkit2:
        use_webkit2_arguments = ["-2"]
    else:
        use_webkit2_arguments = []

    subprocess.call(["run-webkit-tests", "-p", "--force", "--treat-ref-tests-as-pixel-tests", "--additional-platform-directory=" + results_directory, "--run-singly", "--no-retry-failures"] + use_webkit2_arguments + additional_arguments + tests)

if __name__ == '__main__':
    main()