blob: c542ce9bec895bc0caa6929cb725cd8104e15aee [file] [log] [blame]
#!/usr/bin/env perl
# Copyright (C) 2005-2017 Apple Inc. All rights reserved.
# Copyright (C) 2007 Eric Seidel <eric@webkit.org>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of Apple Inc. ("Apple") nor the names of
# its contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Script to run the WebKit Open Source Project JavaScriptCore tests (adapted from Mozilla),
# as well as other tests: testapi on Mac and LayoutTests/js.
use strict;
use warnings;
use File::Spec;
use FindBin;
use Getopt::Long qw(:config pass_through);
use JSON::PP;
use lib $FindBin::Bin;
use List::Util qw(min max);
use POSIX;
use webkitdirs;
use Text::ParseWords;
# determine configuration
setConfiguration();
my $configuration = configuration();
# These variables are intentionally left undefined.
my $root;
my $showHelp;
my @extraTests = ();
my $childProcesses;
my $shellRunner;
my $makeRunner;
my $rubyRunner;
my $testWriter;
my $memoryLimited;
my $report;
my $buildbotMaster;
my $builderName;
my $buildNumber;
my $buildbotWorker;
my $buildJSC = 1;
my $copyJSC = 1;
use constant {
ENV_VAR_SAYS_DO_RUN => 4,
ENV_VAR_SAYS_DONT_RUN => 3,
RUN_IF_NO_TESTS_SPECIFIED => 2,
DO_RUN => 1,
DONT_RUN => 0,
};
my $runTestMasm = RUN_IF_NO_TESTS_SPECIFIED;
my $runTestAir = RUN_IF_NO_TESTS_SPECIFIED;
my $runTestB3 = RUN_IF_NO_TESTS_SPECIFIED;
my $runTestDFG = RUN_IF_NO_TESTS_SPECIFIED;
my $runTestAPI = RUN_IF_NO_TESTS_SPECIFIED;
my $runJSCStress = RUN_IF_NO_TESTS_SPECIFIED;
my $runMozillaTests = RUN_IF_NO_TESTS_SPECIFIED;
# $runJITStressTests is special because it is not for enabling a different set of tests.
# Instead it is only meaningful for when we want to disable using JIT test configurations
# on the JSC stress test or mozilla tests.
my $runJITStressTests = 1;
my $runQuickMode = 0;
my $forceCollectContinuously = 0;
my $envVars = "";
my $gmallocPath = undef;
my $gmallocDefaultPath = "/usr/lib/libgmalloc.dylib";
my $createTarball = 0;
my $remoteHost = 0;
my $model = 0;
my $version;
my $versionName;
my $sdk;
my $failFast = 1;
my %jsonData = ();
my %reportData = ();
my @testResults = ();
my $isTestFailed = 0;
my $remoteConfigFile;
my $jsonFileName;
my $verbose = 0;
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM}) {
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM} eq "true") {
$runTestMasm = ENV_VAR_SAYS_DO_RUN;
} elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM} eq "false") {
$runTestMasm = ENV_VAR_SAYS_DONT_RUN;
} else {
print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTMASM environment variable: '"
. $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM} . "'. Should be set to 'true' or 'false'.\n";
}
}
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR}) {
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR} eq "true") {
$runTestAir = ENV_VAR_SAYS_DO_RUN;
} elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR} eq "false") {
$runTestAir = ENV_VAR_SAYS_DONT_RUN;
} else {
print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTAIR environment variable: '"
. $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR} . "'. Should be set to 'true' or 'false'.\n";
}
}
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3}) {
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3} eq "true") {
$runTestB3 = ENV_VAR_SAYS_DO_RUN;
} elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3} eq "false") {
$runTestB3 = ENV_VAR_SAYS_DONT_RUN;
} else {
print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTB3 environment variable: '"
. $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3} . "'. Should be set to 'true' or 'false'.\n";
}
}
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG}) {
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG} eq "true") {
$runTestDFG = ENV_VAR_SAYS_DO_RUN;
} elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG} eq "false") {
$runTestDFG = ENV_VAR_SAYS_DONT_RUN;
} else {
print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTDFG environment variable: '"
. $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG} . "'. Should be set to 'true' or 'false'.\n";
}
}
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI}) {
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} eq "true") {
$runTestAPI = ENV_VAR_SAYS_DO_RUN;
} elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} eq "false") {
$runTestAPI = ENV_VAR_SAYS_DONT_RUN;
} else {
print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTAPI environment variable: '"
. $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} . "'. Should be set to 'true' or 'false'.\n";
}
}
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD}) {
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} eq "true") {
$buildJSC = 1;
} elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} eq "false") {
$buildJSC = 0;
} else {
print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_BUILD environment variable: '"
. $ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} . "'. Should be set to 'true' or 'false'.\n";
}
}
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_COPY}) {
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_COPY} eq "true") {
$copyJSC = 1;
} elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_COPY} eq "false") {
$copyJSC = 0;
} else {
print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_COPY environment variable: '"
. $ENV{RUN_JAVASCRIPTCORE_TESTS_COPY} . "'. Should be set to 'true' or 'false'.\n";
}
}
if ($ENV{RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS}) {
push @extraTests, $ENV{RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS};
}
sub defaultStringForTestState {
my ($state) = @_;
if ($state == ENV_VAR_SAYS_DONT_RUN) {
return "will not run due to environment variable";
} elsif ($state == DONT_RUN) {
return "will not run";
} else {
return "will run";
}
}
# Additional environment parameters
push @ARGV, parse_line('\s+', 0, $ENV{'TEST_JSC_ARGS'}) if ($ENV{'TEST_JSC_ARGS'});
my $programName = basename($0);
my $buildJSCDefault = $buildJSC ? "will check" : "will not check";
my $testmasmDefault = defaultStringForTestState($runTestMasm);
my $testairDefault = defaultStringForTestState($runTestAir);
my $testb3Default = defaultStringForTestState($runTestB3);
my $testDFGDefault = defaultStringForTestState($runTestDFG);
my $testapiDefault = defaultStringForTestState($runTestAPI);
my $jscStressDefault = defaultStringForTestState($runJSCStress);
my $mozillaTestsDefault = defaultStringForTestState($runMozillaTests);
my $jitStressTestsDefault = $runJITStressTests ? "will run" : " will not run";
my $quickModeDefault = $runQuickMode ? "some" : "all";
my $failFastDefault = $failFast ? "fail fast" : "don't fail fast";
my $copyJSCDefault = $copyJSC ? "copy" : "do not copy";
my $filter;
my $usage = <<EOF;
Usage: $programName [options] [options to pass to build system]
--help Show this help message
--root= Path to pre-built root containing jsc
--[no-]ftl-jit Turn the FTL JIT on or off
--[no-]build Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault)
--[no-]testmasm Only run (or don't run) testmasm (default: $testmasmDefault)
--[no-]testair Only run (or don't run) testair (default: $testairDefault)
--[no-]testb3 Only run (or don't run) testb3 (default: $testb3Default)
--[no-]testdfg Only run (or don't run) testdfg (default: $testDFGDefault)
--[no-]testapi Only run (or don't run) testapi (default: $testapiDefault)
--[no-]jsc-stress Only run (or don't run) the JSC stress tests (default: $jscStressDefault)
--[no-]mozilla-tests Only run (or don't run) the Mozilla tests (default: $mozillaTestsDefault)
--[no-]jit-stress-tests Run (or don't run) the JIT stress tests (default: $jitStressTestsDefault)
--[no-]quick Run some (or all) of the regular testing modes (default: $quickModeDefault)
If the runner only runs some it will run the default and no-cjit-validate modes.
Note, this will not change the behavior of tests that specify their own modes.
--[no-]fail-fast Stop this script when a test family reports an error or failure (default: $failFastDefault)
--[no-]force-collectContinuously Enable the collectContinuously mode even if it was disabled on this platform.
--[no-]copy Copy (or don't copy) the JavaScriptCore build product before testing (default: $copyJSCDefault)
--json-output= Create a file at specified path, listing failed stress tests in JSON format.
--tarball Create a tarball of the bundle produced by running the JSC stress tests.
--remote= Run the JSC stress tests on the specified remote host. Implies --tarball.
--remote-config-file= Same as remote, but read config from JSON file.
--model= Specify remote hardware model, this info used for determine what jsc tests should run on remote
--version Specify the version number of the device running tests.
--version-name Specify the version name of the hardware running tests.
--sdk Specific SDK or OS version of the form ##*###
--extra-tests= Path to a file containing extra tests
--child-processes= Specify the number of child processes.
--shell-runner Uses the shell-based test runner instead of the default make-based runner.
In general the shell runner is slower than the make runner.
--make-runner Uses the faster make-based runner.
--ruby-runner Uses the ruby runner for machines without unix shell or make.
--test-writer [writer] Specifies the test script format."
default is to use shell scripts to run the tests"
\"ruby\" to use ruby scripts for systems without a unix shell.
--memory-limited Indicate that we are targeting the test for a memory limited device.
Skip tests tagged with //\@skip if \$memoryLimited
--filter Only run tests whose name matches the given regular expression.
--env-vars Pass a list of environment variables to set before running tests.
Each environment variable should be separated by a space.
e.g. \"foo=bar x=y\" (no quotes).
--gmalloc: Run tests with Guard Malloc enabled (if no path is given: $gmallocDefaultPath is used)
--verbose: Verbose output (specify more than once to increase verbosity).
--report: Results database url to report results to.
--buildbot-master: The url of the buildbot master.
--builder-name: The name of the buildbot builder tests were run on.
--build-number: The buildbot build number tests are associated with.
--buildbot-worker: The buildbot worker tests were run on.
Environment Variables:
- set RUN_JAVASCRIPTCORE_TESTS_TESTMASM to "true" or "false" (no quotes) to determine if we run testmasm by default.
- set RUN_JAVASCRIPTCORE_TESTS_TESTAIR to "true" or "false" (no quotes) to determine if we run testair by default.
- set RUN_JAVASCRIPTCORE_TESTS_TESTB3 to "true" or "false" (no quotes) to determine if we run testb3 by default.
- set RUN_JAVASCRIPTCORE_TESTS_TESTDFG to "true" or "false" (no quotes) to determine if we run testdfg by default.
- set RUN_JAVASCRIPTCORE_TESTS_TESTAPI to "true" or "false" (no quotes) to determine if we run testapi by default.
- set RUN_JAVASCRIPTCORE_TESTS_BUILD to "true" or "false" (no quotes) to set the should-we-build-before-running-tests setting.
- set RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS to the path of a yaml file or a directory of JS files to be run as part of run-javascriptcore-tests.
If one or more --<testname> options are specified, only those tests will run. For example,
the following only runs testmasm and testapi:
\$ run-javascriptcore-tests --testmasm --testapi
Otherwise, all tests will run unless the test is disabled using --no-<testname> or an
environment variable.
EOF
GetOptions(
'root=s' => \$root,
'extra-tests=s' => \@extraTests,
'build!' => \$buildJSC,
'testmasm!' => \$runTestMasm,
'testair!' => \$runTestAir,
'testb3!' => \$runTestB3,
'testdfg!' => \$runTestDFG,
'testapi!' => \$runTestAPI,
'jsc-stress!' => \$runJSCStress,
'mozilla-tests!' => \$runMozillaTests,
'jit-stress-tests!' => \$runJITStressTests,
'quick!' => \$runQuickMode,
'fail-fast!' => \$failFast,
'force-collectContinuously!' => \$forceCollectContinuously,
'copy!' => \$copyJSC,
'json-output=s' => \$jsonFileName,
'tarball!' => \$createTarball,
'remote=s' => \$remoteHost,
'model=s' => \$model,
'version=s' => \$version,
'version-name=s' => \$versionName,
'sdk=s' => \$sdk,
'remote-config-file=s' => \$remoteConfigFile,
'child-processes=s' => \$childProcesses,
'shell-runner' => \$shellRunner,
'make-runner' => \$makeRunner,
'ruby-runner' => \$rubyRunner,
'test-writer=s' => \$testWriter,
'memory-limited' => \$memoryLimited,
'filter=s' => \$filter,
'help' => \$showHelp,
'env-vars=s' => \$envVars,
'gmalloc:s' => \$gmallocPath,
'verbose+' => \$verbose,
'report=s' => \$report,
'buildbot-master=s' => \$buildbotMaster,
'builder-name=s' => \$builderName,
'build-number=s' => \$buildNumber,
'buildbot-worker=s' => \$buildbotWorker,
);
my $specificTestsSpecified = 0;
if ($runTestMasm == DO_RUN
|| $runTestAir == DO_RUN
|| $runTestB3 == DO_RUN
|| $runTestDFG == DO_RUN
|| $runTestAPI == DO_RUN
|| $runJSCStress == DO_RUN
|| $runMozillaTests == DO_RUN) {
$specificTestsSpecified = 1;
}
if ($version) {
$version = splitVersionString($version);
}
sub enableTestOrNot {
my ($state) = @_;
if ($state == RUN_IF_NO_TESTS_SPECIFIED || $state == ENV_VAR_SAYS_DO_RUN) {
return $specificTestsSpecified ? DONT_RUN : DO_RUN;
} elsif ($state == ENV_VAR_SAYS_DONT_RUN) {
return DONT_RUN;
}
return $state;
}
sub configurationForUpload()
{
my $platform;
my $sdk;
my $simulator = 0;
if (index(xcodeSDKPlatformName(), "simulator") != -1) {
$simulator = 1;
}
if (index($model, 'iPhone') != -1 || index($model, 'iPad') != -1) {
$platform = 'ios';
if (!$version) {
$version = iosVersion();
}
} elsif (index($model, 'watch') != -1 || index(xcodeSDKPlatformName(), "watch") != -1) {
$platform = 'watchos';
die "No watchOS version specified" if !$version;
} elsif (index(xcodeSDKPlatformName(), "appletv") != -1) {
$platform = 'tvos';
die "No tvOS version specified" if !$version;
} elsif (isGtk()) {
$platform = 'GTK';
if (!$version) {
chomp($version = `uname -r`);
$version = splitVersionString($version);
}
} elsif (isWPE()) {
$platform = 'WPE';
if (!$version) {
chomp($version = `uname -r`);
$version = splitVersionString($version);
}
} elsif (isAnyWindows()) {
$platform = 'win';
if (!$version) {
$version = winVersion();
}
} elsif (isAppleMacWebKit()) {
$platform = 'mac';
chomp($model = `/usr/sbin/sysctl -n hw.model`);
if (!$version) {
$version = osXVersion();
}
if (!$sdk) {
chomp($sdk = `/usr/bin/sw_vers -buildVersion`);
}
if (!$versionName) {
if ($version->{minor} eq 15) {
$versionName = "Catalina";
} elsif ($version->{minor} eq 14) {
$versionName = "Mojave";
} elsif ($version->{minor} eq 13) {
$versionName = "High Sierra";
}
}
} else {
# FIXME: This will be the kernel version, which is not always exactly what we want, but it will at least
# always give us something to work with
chomp($platform = `uname`);
$platform = lc $platform;
if (!$version) {
chomp($version = `uname -r`);
$version = splitVersionString($version);
}
}
my $result = {
platform => $platform,
architecture => architecture(),
is_simulator => $simulator,
style => lc(configuration()),
version => "$version->{major}.$version->{minor}.$version->{subminor}",
};
if ($model) {
$result->{model} = $model;
}
if ($versionName) {
$result->{version_name} = $versionName;
}
if ($sdk) {
$result->{sdk} = $sdk;
}
return $result;
}
$runTestMasm = enableTestOrNot($runTestMasm);
$runTestAir = enableTestOrNot($runTestAir);
$runTestB3 = enableTestOrNot($runTestB3);
$runTestDFG = enableTestOrNot($runTestDFG);
$runTestAPI = enableTestOrNot($runTestAPI);
$runJSCStress = enableTestOrNot($runJSCStress);
$runMozillaTests = enableTestOrNot($runMozillaTests);
# Assume any arguments left over from GetOptions are assumed to be build arguments
my @buildArgs = @ARGV;
if ($showHelp) {
print STDERR $usage;
exit 1;
}
setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));
configurationForUpload() if (defined($report));
if (defined($jsonFileName)) {
$jsonFileName = File::Spec->rel2abs($jsonFileName);
}
if (!defined($root) && $buildJSC) {
chdirWebKit();
push(@buildArgs, argumentsForConfiguration());
print "Running: build-jsc " . join(" ", @buildArgs) . "\n";
my $buildResult = system "perl", File::Spec->catfile("Tools", "Scripts", "build-jsc"), @buildArgs;
if ($buildResult) {
print STDERR "Compiling jsc failed!\n";
exit exitStatus($buildResult);
}
}
if (defined($gmallocPath)) {
if ($gmallocPath eq "") {
$envVars .= " DYLD_INSERT_LIBRARIES=" . $gmallocDefaultPath;
} else {
$envVars .= " DYLD_INSERT_LIBRARIES=" . $gmallocPath;
}
}
my $productDir = jscProductDir();
$ENV{DYLD_FRAMEWORK_PATH} = $productDir;
$ENV{JSCTEST_timeout} = 120 unless $ENV{JSCTEST_timeout}; # Set a 120 second timeout on all jsc tests (if environment variable not defined already).
$ENV{TZ}="US/Pacific"; # Some tests fail if the time zone is not set to US/Pacific (<https://webkit.org/b/136363>)
setPathForRunningWebKitApp(\%ENV) if isCygwin();
my $startTime = time();
my $endTime = $startTime;
sub testPath {
my ($productDir, $testName) = @_;
$testName .= "_debug" if configuration() eq "Debug_All";
return File::Spec->catfile($productDir, $testName);
}
sub runTest {
my ($testName, $jsonTestStatusName) = @_;
chdirWebKit();
chdir($productDir) or die "Failed to switch directory to '$productDir'\n";
my @command = (testPath($productDir, $testName));
unshift @command, ("xcrun", "-sdk", xcodeSDK(), "sim") if willUseIOSSimulatorSDK();
unshift @command, wrapperPrefixIfNeeded() if shouldUseJhbuild();
if ($envVars ne "") {
foreach my $var (split(/\s+/, $envVars)) {
if ($var =~ /([^=]*)=(.*)/) {
$ENV{$1} = $2;
}
}
}
my $testResult = 0;
my $failureFlag = 0;
my $lastOptimizeLevel;
open(TEST, "-|", "@command 2>&1") or die "Failed to run @command";
while ( my $line = <TEST> ) {
my $subTestName;
print $line if ($verbose);
# e.g.: 'O2: testRotRWithImmShift(uint64-max, uint32-min): OK!''
if ($line =~ /(O\d): (\S+)(\(.*\)): ([A-Za-z]+)!\n/) {
$lastOptimizeLevel = $1;
$subTestName = "$testName.$1.$2$3";
$failureFlag += 1 if ($4 ne "OK");
}
# Assertion in binary
# e.g.: ASSERTION FAILED: things[0] == 2
elsif ($line =~ /ASSERTION FAILED:/) {
$failureFlag += 1;
next;
}
# e.g.: file/path(397) : void (anonymous namespace)::testShuffleSimpleBroadcast()
elsif ($failureFlag && $line =~ /.*\(\d+\) : \S+ .*::(\S+)(\(.*\))/) {
if (defined $lastOptimizeLevel) {
$subTestName = "$testName.$lastOptimizeLevel.$1$2";
} else {
$subTestName = "$testName.$1$2";
}
}
# sub-test from the c-api, of the form:
# e.g.: 'testapi[15422:16904790] parallelPromiseResolveTest(): done.''
elsif ($line =~ /\[\d+:\d+\] (\S+)(): done\.\n/) {
$subTestName = "$testName.objective-c.$1";
}
# End of C-API tests (meaning we just finished the ungrouped main tests)
elsif ($line =~ /Starting C-API tests in C++/) {
$subTestName = "$testName.objective-c.main";
}
# End of testapi
# e.g.: FAIL: Some tests failed.
elsif ($line =~ /FAIL: Some tests failed\./) {
$failureFlag = 0;
next;
}
# Test cases inside API tests
# e.g.: 'testapi[15422:16904790] TEST: "2 + 2": PASSED'
elsif ($line =~ /\[\d+:\d+\] TEST: ".*": ([A-Z]+)\n/) {
$failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS");
next;
}
# e.g.: 'testBitXorAndAndArgs(0, 0,uint32-max): OK!''
elsif ($line =~ /(\S+)(\(.*\)): ([A-Za-z]+)!\n/) {
$subTestName = "$testName.$1$2";
$failureFlag += 1 if ($3 ne "OK");
}
# General testcase statement, filtered to exclude some junk logging
# e.g.: 'PASS: DFG script timed out as expected when no callback is specified.''
elsif ($line =~ /([A-Z]+): (.*)\.*\n/ && !($line =~ /[A-Z]+: \d+\n/ || $line =~ /\[\d+:\d+\] [A-Z]+:/)) {
$subTestName = "$testName.$2";
$failureFlag += 1 if ($1 ne "PASS");
}
# e.g.: 'test description: PASSED'
elsif ($line =~ /.*: ([A-Z]+)\n/) {
$failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS");
next;
} else {
next;
}
if (!defined $reportData{$subTestName}) {
print "$subTestName failed\n" if ($failureFlag && !$verbose);
$reportData{$subTestName} = $failureFlag ? {actual => "FAIL"} : {actual => "PASS"};
}
$failureFlag = 0;
}
$testResult = close(TEST) ? 0 : $?;
my $exitStatus = exitStatus($testResult);
print "$testName completed with rc=$testResult ($exitStatus)\n\n";
if ($testResult) {
$isTestFailed = 1;
push @testResults, $testName;
}
if (defined($jsonFileName)) {
my $testStatus = ($exitStatus == 0)? JSON::PP::true: JSON::PP::false;
$jsonData{$jsonTestStatusName} = $testStatus;
}
if ($testResult && $failFast) {
reportTestFailures();
writeJsonDataIfApplicable();
$endTime = time();
uploadResults();
exit $exitStatus;
}
}
sub reportTestFailures {
my $numJSCtestFailures = @testResults;
if ($numJSCtestFailures) {
print "\n** The following JSC test binaries failures have been introduced:\n";
foreach my $testFailure (@testResults) {
print "\t$testFailure\n";
}
}
print "\n";
print "Results for JSC test binaries:\n";
printThingsFound($numJSCtestFailures, "failure", "failures", "found");
print " OK.\n" if $numJSCtestFailures == 0;
print "\n";
}
if ($runTestMasm) { runTest("testmasm", "allMasmTestsPassed") }
if ($runTestAir) { runTest("testair", "allAirTestsPassed") }
if ($runTestB3) { runTest("testb3", "allB3TestsPassed") }
if ($runTestDFG) { runTest("testdfg", "allDFGTestsPassed") }
if ($runTestAPI) { runTest("testapi", "allApiTestsPassed") }
# Find JavaScriptCore directory
chdirWebKit();
# FIXME: report stress test failures https://bugs.webkit.org/show_bug.cgi?id=204096
runJSCStressTests();
reportTestFailures();
$endTime = time();
uploadResults();
if ($isTestFailed) {
exit(1);
}
sub runJSCStressTests
{
my $jscStressResultsDir = $productDir . "/jsc-stress-results";
my $hasTestsToRun = 0;
my @testList;
if ($runJSCStress) {
@testList = (
"PerformanceTests/SunSpider/tests/sunspider-1.0",
"PerformanceTests/JetStream/cdjs/cdjs-tests.yaml",
"PerformanceTests/ARES-6/Air/airjs-tests.yaml",
"PerformanceTests/ARES-6/Basic/basic-tests.yaml",
"JSTests/executableAllocationFuzz.yaml",
"JSTests/exceptionFuzz.yaml",
"PerformanceTests/SunSpider/no-architecture-specific-optimizations.yaml",
"PerformanceTests/SunSpider/shadow-chicken.yaml",
"PerformanceTests/SunSpider/tests/v8-v6",
"JSTests/stress",
"JSTests/microbenchmarks",
"JSTests/slowMicrobenchmarks.yaml",
"PerformanceTests/SunSpider/profiler-test.yaml",
"LayoutTests/jsc-layout-tests.yaml",
"JSTests/typeProfiler.yaml",
"JSTests/controlFlowProfiler.yaml",
"JSTests/es6.yaml",
"JSTests/modules.yaml",
"JSTests/complex.yaml",
"JSTests/ChakraCore.yaml",
"JSTests/wasm.yaml");
$hasTestsToRun = 1;
}
if ($runMozillaTests) {
push(@testList, "JSTests/mozilla/mozilla-tests.yaml");
$hasTestsToRun = 1;
}
# Set LANG environment variable so the stress tests will work with newer ruby (<rdar://problem/15010705>)
$ENV{LANG}="en_US.UTF-8";
my @jscStressDriverCmd = (
"/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests",
"-j", jscPath($productDir), "-o", $jscStressResultsDir);
push(@jscStressDriverCmd, @testList);
if (isWindows() && !isCygwin()) {
shift @jscStressDriverCmd; # Remove /usr/bin/env
}
if (configuration() eq "Debug") {
push(@jscStressDriverCmd, "--debug");
}
if (!$copyJSC) {
push(@jscStressDriverCmd, "--no-copy");
}
if ($forceCollectContinuously) {
push(@jscStressDriverCmd, "--force-collectContinuously");
}
if ($envVars ne "") {
push(@jscStressDriverCmd, "--env-vars");
push(@jscStressDriverCmd, $envVars);
}
if ($runQuickMode) {
push(@jscStressDriverCmd, "--quick");
}
if (!$runJITStressTests) {
push(@jscStressDriverCmd, "--no-jit");
}
if ($createTarball) {
push(@jscStressDriverCmd, "--tarball");
}
if ($remoteHost) {
push(@jscStressDriverCmd, "--remote");
push(@jscStressDriverCmd, $remoteHost);
}
if ($remoteConfigFile) {
push(@jscStressDriverCmd, "--remote-config-file");
push(@jscStressDriverCmd, $remoteConfigFile);
}
if ($model) {
push(@jscStressDriverCmd, "--model");
push(@jscStressDriverCmd, $model);
}
if ($childProcesses) {
push(@jscStressDriverCmd, "--child-processes");
push(@jscStressDriverCmd, $childProcesses);
}
if ($shellRunner) {
push(@jscStressDriverCmd, "--shell-runner");
}
if ($makeRunner) {
push(@jscStressDriverCmd, "--make-runner");
}
if ($rubyRunner) {
push(@jscStressDriverCmd, "--ruby-runner");
}
if ($testWriter) {
push(@jscStressDriverCmd, "--test-writer");
push(@jscStressDriverCmd, $testWriter);
}
if ($memoryLimited) {
push(@jscStressDriverCmd, "--memory-limited");
}
if ($filter) {
push(@jscStressDriverCmd, "--filter");
push(@jscStressDriverCmd, $filter);
}
push(@jscStressDriverCmd, ("--verbose") x $verbose) if ($verbose > 0);
if (isPlayStation()) {
push(@jscStressDriverCmd, "--os=playstation");
push(@jscStressDriverCmd, "--no-copy");
}
unshift @jscStressDriverCmd, wrapperPrefixIfNeeded() if shouldUseJhbuild();
# End option processing, the rest of the arguments are tests
push(@jscStressDriverCmd, "--");
for my $testSuite (@extraTests) {
push(@jscStressDriverCmd, $testSuite);
$hasTestsToRun = 1;
}
if (defined($ENV{"EXTRA_JSC_TESTS"})) {
push(@jscStressDriverCmd, $ENV{"EXTRA_JSC_TESTS"});
$hasTestsToRun = 1;
}
if (!$hasTestsToRun) {
exit(0);
}
print "Running: " . join(" ", @jscStressDriverCmd) . "\n";
my $result = system(@jscStressDriverCmd);
exit exitStatus($result) if $result;
my @jscStressFailList = readAllLines($jscStressResultsDir . "/failed");
@jscStressFailList = sort @jscStressFailList;
my $numJSCStressFailures = @jscStressFailList;
if ($numJSCStressFailures) {
$isTestFailed = 1;
print "\n** The following JSC stress test failures have been introduced:\n";
foreach my $testFailure (@jscStressFailList) {
print "\t$testFailure\n";
}
}
print "\n";
print "Results for JSC stress tests:\n";
printThingsFound($numJSCStressFailures, "failure", "failures", "found");
print " OK.\n" if $numJSCStressFailures == 0;
print "\n";
if (defined($jsonFileName)) {
$jsonData{'stressTestFailures'} = \@jscStressFailList;
}
writeJsonDataIfApplicable();
}
sub readAllLines
{
my ($filename) = @_;
my @array = ();
eval {
open FILE, $filename or die;
while (<FILE>) {
chomp;
push @array, $_;
}
close FILE;
};
return @array;
}
sub printThingsFound
{
my ($number, $label, $pluralLabel, $verb) = @_;
print " $number ";
if ($number == 1) {
print $label;
} else {
print $pluralLabel;
}
print " $verb.\n";
}
sub writeJsonDataIfApplicable
{
if (defined($jsonFileName)) {
open(my $fileHandler, ">", $jsonFileName) or die;
print $fileHandler "${\encode_json(\%jsonData)}\n";
close($fileHandler);
}
}
sub uploadResults
{
if (not defined $report) {
print "Skipping upload to results database since no report URL was specified\n";
return 0;
}
my %upload = (
version => 0,
suite => 'javascriptcore-tests',
# FIXME: Integrate branches, https://bugs.webkit.org/show_bug.cgi?id=204094
# FIXME: Integrate the Safari repository, https://bugs.webkit.org/show_bug.cgi?id=204095
commits => [{
repository_id => 'webkit',
id => determineCurrentSVNRevision(),
}],
configuration => configurationForUpload(),
test_results => {
run_stats => {
start_time => $startTime,
end_time => $endTime,
tests_skipped => 0,
},
results => \%reportData,
},
timestamp => $startTime,
);
if (defined $buildbotMaster and defined $builderName and defined $buildNumber and defined $buildbotWorker) {
$upload{test_results}{details} = {
'buildbot-master' => $buildbotMaster,
'builder-name' => $builderName,
'build-number' => $buildNumber,
'buildbot-worker' => $buildbotWorker,
};
} else {
print " No buildbot details provided, test run will not be linked to CI system\n";
}
if (defined $ENV{'RESULTS_SERVER_API_KEY'}) {
$upload{'api_key'} = $ENV{'RESULTS_SERVER_API_KEY'};
}
print "Uploading results to $report\n";
open(HANDLE, "|-", "curl -X POST $report/api/upload -H 'Content-Type: application/json' -f -d '\@-'") or die "Failed to open curl";
# Json conforming to https://results.webkit.org/documentation#API-Uploads.
my $encodedUpload = encode_json(\%upload);
print HANDLE "$encodedUpload\n\0";
my $success = close HANDLE;
if ($success) {
print "Upload successful!\n";
return 0;
}
print "Upload to $report failed\n";
return 1;
}