blob: 7a834c70c4c70562fd51971ad6e366c42ff269c6 [file] [log] [blame]
# Copyright (C) 2018 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import print_function
import logging
import optparse
import sys
import traceback
from webkitpy.api_tests.manager import Manager
from webkitpy.common.host import Host
from webkitpy.layout_tests.views.metered_stream import MeteredStream
from webkitpy.port import configuration_options, platform_options, base, win
from webkitpy.results.options import upload_options
EXCEPTIONAL_EXIT_STATUS = -1
INTERRUPT_EXIT_STATUS = -2
_log = logging.getLogger(__name__)
def main(argv, stdout, stderr):
options, args = parse_args(argv)
host = Host()
try:
options.webkit_test_runner = True
port = host.port_factory.get(options.platform, options)
except NotImplementedError as e:
print(str(e), file=stderr)
return EXCEPTIONAL_EXIT_STATUS
# Some platforms do not support API tests
does_not_support_api_tests = ['ios-device']
if port.operating_system() in does_not_support_api_tests:
print('{} cannot run API tests'.format(port.operating_system()), file=stderr)
return EXCEPTIONAL_EXIT_STATUS
try:
return run(port, options, args, stderr)
except KeyboardInterrupt:
return INTERRUPT_EXIT_STATUS
except BaseException as e:
if isinstance(e, Exception):
print('\n%s raised: %s' % (e.__class__.__name__, str(e)), file=stderr)
traceback.print_exc(file=stderr)
return EXCEPTIONAL_EXIT_STATUS
def run(port, options, args, logging_stream):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if options.verbose else logging.ERROR if options.quiet else logging.INFO)
try:
stream = MeteredStream(logging_stream, options.verbose, logger=logger, number_of_columns=port.host.platform.terminal_width(), print_timestamps=options.timestamps)
manager = Manager(port, options, stream)
result = manager.run(args, json_output=options.json_output)
_log.debug("Testing completed, Exit status: %d" % result)
return result
finally:
stream.cleanup()
def parse_args(args):
option_group_definitions = []
option_group_definitions.append(('Platform options', platform_options()))
option_group_definitions.append(('Configuration options', configuration_options()))
option_group_definitions.append(('Printing Options', [
optparse.make_option('-q', '--quiet', action='store_true', default=False,
help='Run quietly (errors, warnings, and progress only)'),
optparse.make_option('-v', '--verbose', action='store_true', default=False,
help='Enable verbose printing'),
optparse.make_option('--timestamps', action='store_true', default=False,
help='Print timestamps for each logged line'),
optparse.make_option('--json-output', action='store', default=None,
help='Save test results as JSON to file'),
]))
option_group_definitions.append(('WebKit Options', [
optparse.make_option('-g', '--guard-malloc', action='store_true', default=False,
help='Enable Guard Malloc (OS X only)'),
optparse.make_option('--root', action='store',
help='Path to a directory containing the executables needed to run tests.'),
]))
option_group_definitions.append(('Testing Options', [
optparse.make_option('--wtf-only', action='store_const', const='TestWTF', dest='api_binary',
help='Only build, check and run TestWTF'),
optparse.make_option('--webkit-only', action='store_const', const='TestWebKitAPI', dest='api_binary',
help='Only check and run TestWebKitAPI'),
optparse.make_option('--web-core-only', action='store_const', const='TestWebCore', dest='api_binary',
help='Only check and run TestWebCore.exe (Windows only)'),
optparse.make_option('--webkit-legacy-only', action='store_const', const='TestWebKitLegacy', dest='api_binary',
help='Only check and run TestWebKitLegacy.exe (Windows only)'),
optparse.make_option('-d', '--dump', action='store_true', default=False,
help='Dump all test names without running them'),
optparse.make_option('--build', dest='build', action='store_true', default=True,
help='Check to ensure the build is up-to-date (default).'),
optparse.make_option('--no-build', dest='build', action='store_false',
help="Don't check to see if the build is up-to-date."),
optparse.make_option('--timeout', default=30,
help='Number of seconds to wait before a test times out'),
optparse.make_option('--no-timeout', dest='timeout', action='store_false',
help='Disable timeouts for all tests'),
optparse.make_option('--iterations', type='int', default=1, help='Number of times to run the set of tests (e.g. ABCABCABC)'),
optparse.make_option('--repeat-each', type='int', default=1, help='Number of times to run each test (e.g. AAABBBCCC)'),
# FIXME: Remove the default, API tests should be multiprocess
optparse.make_option('--child-processes', default=1,
help='Number of processes to run in parallel.'),
# FIXME: Default should be false, API tests should not be forced to run singly
optparse.make_option('--run-singly', action='store_true', default=True,
help='Run a separate process for each test'),
optparse.make_option('--force', action='store_true', default=False,
help='Run all tests, even DISABLED tests'),
optparse.make_option('--additional-env-var', type='string', action='append', default=[],
help='Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)'),
]))
option_group_definitions.append(('Upload Options', upload_options()))
option_parser = optparse.OptionParser(
usage='run-api-tests [options] [<test names>...]',
description="""By default, run-api-tests will run all API tests. It also allows the user to specify tests of the \
format <suite>.<test> or <canonicalized binary name>.<suite>.<test>. Note that in the case where a binary is not \
specified, one will be inferred by listing all available tests. Specifying just a binary or just a suite will cause every \
test contained within to be run. The canonicalized binary name is the binary name with any filename extension \
stripped. For Unix ports, these binaries are {} and {}. For Windows ports, they are {} and {}.""".format(
', '.join(base.Port.API_TEST_BINARY_NAMES[:-1]), base.Port.API_TEST_BINARY_NAMES[-1],
', '.join(win.WinPort.API_TEST_BINARY_NAMES[:-1]), win.WinPort.API_TEST_BINARY_NAMES[-1],
))
for group_name, group_options in option_group_definitions:
option_group = optparse.OptionGroup(option_parser, group_name)
option_group.add_options(group_options)
option_parser.add_option_group(option_group)
return option_parser.parse_args(args)
if __name__ == '__main__':
sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))