blob: cf0b92cbf11d3660ade8a293ee2ecf43d767b01e [file] [log] [blame]
#!/usr/bin/env perl
# Copyright (C) 2010-2012, 2014-2015 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
use strict;
use warnings;
use File::Basename;
use File::Spec;
use FindBin;
use Getopt::Long qw(:config pass_through);
use IPC::Open3;
use JSON::PP;
use lib $FindBin::Bin;
use sigtrap qw(die normal-signals);
use webkitdirs;
use VCSUtils;
sub buildTestTool();
sub dumpTestsBySuite(\@);
sub listAllTests();
sub runTest($$);
sub runTestsBySuite(\@);
sub prepareEnvironmentForRunningTestTool();
sub archCommandLineArgumentsForRestrictedEnvironmentVariables();
sub testToolPaths();
sub writeJSONDataIfApplicable();
# Defined in VCSUtils.
sub possiblyColored($$);
# Timeout for individual test, in sec
my $timeout = 30;
my $showHelp = 0;
my $verbose = 0;
my $showLeaks = 0;
my $dumpTests = 0;
my $disableTimeout = 0;
my $build = 1;
my $root;
my $buildDefault = $build ? "build" : "do not build";
my @testsFailed;
my @testsTimedOut;
my $wtfOnly = 0;
my %testToToolMap;
my %jsonData = ();
my $jsonFileName;
my $programName = basename($0);
my $usage = <<EOF;
Usage: $programName [options] [suite or test prefixes]
--help Show this help message
-v|--verbose Verbose output
-d|--dump-tests Dump the names of testcases without running them
--[no-]build Build (or do not build) unit tests prior to running (default: $buildDefault)
--json-output= Create a file at the specified path, listing test failures and timeouts in JSON format.
--root= Path to the pre-built root containing TestWebKitAPI
--show-leaks Show leaks in the output
--no-timeout Disable test timeouts
--wtf-only Only build and run TestWTF
Platform options:
--ios-simulator Run tests in the iOS Simulator
--simulator DEPRECATED alias of --ios-simulator
@{[ sharedCommandLineOptionsUsage(indent => 2, switchWidth => 21) ]}
Examples
The following command will run a single test:
$programName WebKit.AboutBlank
The following command will run all tests in suites that begin with 'WebKit':
$programName WebKit
EOF
my $getOptionsResult = GetOptions(
sharedCommandLineOptions(),
'help' => \$showHelp,
'verbose|v' => \$verbose,
'show-leaks' => \$showLeaks,
'no-timeout' => \$disableTimeout,
'json-output=s' => \$jsonFileName,
'dump|d' => \$dumpTests,
'build!' => \$build,
'root=s' => \$root,
'wtf-only' => \$wtfOnly,
);
if (!$getOptionsResult || $showHelp) {
print STDERR $usage;
exit 1;
}
setConfiguration();
setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));
if (defined($jsonFileName)) {
$jsonFileName = File::Spec->rel2abs($jsonFileName);
}
buildTestTool() if $build && !defined($root);
setPathForRunningWebKitApp(\%ENV);
my $simulatorDevice;
if (willUseIOSSimulatorSDK()) {
$simulatorDevice = findOrCreateSimulatorForIOSDevice(SIMULATOR_DEVICE_SUFFIX_FOR_WEBKIT_DEVELOPMENT);
relaunchIOSSimulator($simulatorDevice);
}
my @testsToRun = listAllTests();
@testsToRun = grep { my $test = $_; grep { $test =~ m/^\Q$_\E/ } @ARGV; } @testsToRun if @ARGV;
if ($dumpTests) {
dumpTestsBySuite(@testsToRun);
exit 0;
}
END { shutDownIOSSimulatorDevice($simulatorDevice) if $simulatorDevice; }
exit runTestsBySuite(@testsToRun);
sub isSupportedPlatform()
{
return isAppleCocoaWebKit() || isAppleWinWebKit();
}
sub dumpTestsBySuite(\@)
{
my ($tests) = @_;
print "Dumping test cases\n";
print "------------------\n";
my $lastSuite = "";
for my $suiteAndTest (sort @$tests) {
my ($suite, $test) = split(/\./, $suiteAndTest);
if ($lastSuite ne $suite) {
$lastSuite = $suite;
print "$suite:\n";
}
print " $test\n";
}
print "------------------\n";
}
sub runTestsBySuite(\@)
{
my ($tests) = @_;
for my $suiteAndTest (sort @$tests) {
my ($suite, $test) = split(/\./, $suiteAndTest);
runTest($suite, $test);
}
if (@testsFailed) {
print "\nTests that failed:\n";
for my $test (@testsFailed) {
print " $test\n";
}
}
if (@testsTimedOut) {
print "\nTests that timed out:\n";
for my $test (@testsTimedOut) {
print " $test\n";
}
}
if (defined($jsonFileName)) {
$jsonData{'failures'} = \@testsFailed;
$jsonData{'timeouts'} = \@testsTimedOut;
}
writeJSONDataIfApplicable();
return @testsFailed > 0 || @testsTimedOut > 0;
}
sub runTest($$)
{
my ($suite, $testName) = @_;
my $test = $suite . "." . $testName;
my $gtestArg = "--gtest_filter=" . $test;
my $result = 0;
my $timedOut = 0;
die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();
local %ENV = %ENV;
prepareEnvironmentForRunningTestTool();
local *DEVNULL;
my ($childIn, $childOut, $childErr);
if ($verbose || $showLeaks) {
$childErr = 0;
} else {
open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
$childErr = ">&DEVNULL";
}
my $pid;
my @commonArguments = ($testToToolMap{$test}, $gtestArg, @ARGV);
if (willUseIOSSimulatorSDK()) {
$pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to run test: $test.";
} elsif (isAppleCocoaWebKit() && architecture()) {
$pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to run test: $test.";
} else {
$pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to run test: $test.";
}
eval {
if ($disableTimeout) {
waitpid($pid, 0);
} else {
local $SIG{ALRM} = sub { die "alarm\n" };
alarm $timeout;
waitpid($pid, 0);
alarm 0;
}
$result = $?;
};
if ($@) {
die unless $@ eq "alarm\n";
kill SIGTERM, $pid or kill SIGKILL, $pid;
$timedOut = 1;
}
my @testOutput = <$childOut>;
@testOutput = grep { !/^LEAK:/ } @testOutput unless $showLeaks;
map { s/\*\*PASS\*\*/possiblyColored("bold green", "PASS")/eg } @testOutput;
map { s/\*\*FAIL\*\*/possiblyColored("bold red", "FAIL")/eg } @testOutput;
if ($result) {
push @testsFailed, $test;
if (!$timedOut && index("@testOutput", $test) == -1) {
print STDOUT possiblyColored("bold red", "UNEXPECTEDLY EXITED"), " $test\n";
}
} elsif ($timedOut) {
push @testsTimedOut, $test;
print STDOUT possiblyColored("bold yellow", "TIMEOUT"), " $test\n";
}
print STDOUT @testOutput;
close($childIn);
close($childOut);
close($childErr) unless ($verbose || $showLeaks);
close(DEVNULL) unless ($verbose || $showLeaks);
if ($timedOut || $result) {
return $timedOut || $result;
}
return 0;
}
sub listAllTests()
{
my @toolOutput;
my $timedOut;
die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();
prepareEnvironmentForRunningTestTool();
local *DEVNULL;
my ($childIn, $childOut, $childErr);
if ($verbose) {
$childErr = ">&STDERR";
} else {
open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
$childErr = ">&DEVNULL";
}
my @tests = ();
foreach (testToolPaths()) {
my $pid;
my $testTool = $_;
my @commonArguments = ($testTool, "--gtest_list_tests");
if (isIOSWebKit()) {
$pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to build list of tests!";
} elsif (isAppleCocoaWebKit() && architecture()) {
$pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to build list of tests!";
} else {
$pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to build list of tests!";
}
close($childIn);
@toolOutput = <$childOut>;
close($childOut);
close($childErr);
waitpid($pid, 0);
my $result = $?;
if ($result) {
print STDERR "Failed to build list of tests!--\n";
exit exitStatus($result);
}
my $suite;
for my $line (@toolOutput) {
$line =~ s/[\r\n]*$//;
if ($line =~ m/\.$/) {
$suite = $line; # "SuiteName."
} else {
$line =~ s/^\s*//; # "TestName"
my $fullName = $suite . $line; # "SuiteName.TestName";
push @tests, $fullName;
$testToToolMap{$fullName} = $testTool;
}
}
}
close(DEVNULL) unless ($verbose);
return @tests;
}
sub buildTestTool()
{
my $originalCwd = getcwd();
chdirWebKit();
my $buildTestTool = "build-api-tests";
print STDERR "Running $buildTestTool\n";
local *DEVNULL;
my ($childIn, $childOut, $childErr);
if ($verbose) {
# When not quiet, let the child use our stdout/stderr.
$childOut = ">&STDOUT";
$childErr = ">&STDERR";
} else {
open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
$childOut = ">&DEVNULL";
$childErr = ">&DEVNULL";
}
my @args = argumentsForConfiguration();
if ($wtfOnly) {
push @args, "--wtf-only";
}
my $pathToBuildTestTool = File::Spec->catfile("Tools", "Scripts", $buildTestTool);
my $buildProcess = open3($childIn, $childOut, $childErr, "perl", $pathToBuildTestTool, @args) or die "Failed to run " . $buildTestTool;
close($childIn);
close($childOut);
close($childErr);
close(DEVNULL) unless ($verbose);
waitpid($buildProcess, 0);
my $buildResult = $?;
if ($buildResult) {
print STDERR "Compiling TestWebKitAPI failed!\n";
exit exitStatus($buildResult);
}
chdir $originalCwd;
}
sub prepareEnvironmentForRunningTestTool()
{
return unless isAppleCocoaWebKit();
if (willUseIOSSimulatorSDK()) {
my %simulatorENV;
{
local %ENV;
setupIOSWebKitEnvironment(productDir());
%simulatorENV = %ENV;
}
# Prefix the environment variables with SIMCTL_CHILD_ per `xcrun simctl help launch`.
foreach my $key (keys %simulatorENV) {
$ENV{"SIMCTL_CHILD_$key"} = $simulatorENV{$key};
}
return;
}
setupMacWebKitEnvironment(productDir());
}
sub testToolPaths()
{
if (!isAppleWinWebKit()) {
my @toolPaths = ();
if (!$wtfOnly) {
push @toolPaths, File::Spec->catfile(productDir(), "TestWebKitAPI");
}
push @toolPaths, File::Spec->catfile(productDir(), "TestWTF");
return @toolPaths;
}
my $binDir = isWin64() ? "bin64" : "bin32";
my $pathWTF = File::Spec->catfile(productDir(), $binDir, "TestWTF");
my $pathWebCore = File::Spec->catfile(productDir(), $binDir, "TestWebCore");
my $pathWebKit = File::Spec->catfile(productDir(), $binDir, "TestWebKitLegacy");
my $suffix;
if (configuration() eq "Debug_All") {
$suffix = "_debug";
} else {
$suffix = "";
}
return ("$pathWTF$suffix.exe", "$pathWebCore$suffix.exe", "$pathWebKit$suffix.exe");
}
sub writeJSONDataIfApplicable()
{
if (defined($jsonFileName)) {
open(my $fileHandler, ">", $jsonFileName) or die;
print $fileHandler "${\encode_json(\%jsonData)}\n";
close($fileHandler);
}
}