| #!/usr/bin/perl -w |
| |
| # Copyright (C) 2005, 2013-2015 Apple Inc. All rights reserved. |
| # Copyright (C) 2007 Eric Seidel <eric@webkit.org> |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # 3. Neither the name of Apple Inc. ("Apple") nor the names of |
| # its contributors may be used to endorse or promote products derived |
| # from this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY |
| # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| # Script to run the WebKit Open Source Project JavaScriptCore tests (adapted from Mozilla), |
| # as well as other tests: testapi on Mac and LayoutTests/js. |
| |
| use strict; |
| use FindBin; |
| use Getopt::Long qw(:config pass_through); |
| use List::Util qw(min max); |
| use lib $FindBin::Bin; |
| use webkitdirs; |
| use POSIX; |
| |
| # determine configuration |
| setConfiguration(); |
| my $configuration = configuration(); |
| |
| # These variables are intentionally left undefined. |
| my $root; |
| my $showHelp; |
| my $extraTests; |
| my $jsDriverArgs; |
| my $childProcesses; |
| my $shellRunner; |
| my $makeRunner; |
| |
| my $buildJSC = 1; |
| |
| my $runTestAPI = isAppleMacWebKit() || isAppleWinWebKit() || isWinCairo(); |
| |
| my $runJSCStress = 1; |
| |
| my $enableFTL = isAppleMacWebKit(); |
| my $createTarball = 0; |
| my $remoteHost = 0; |
| my $remoteConfigFile; |
| |
| my $programName = basename($0); |
| my $buildJSCDefault = $buildJSC ? "will check" : "will not check"; |
| my $testapiDefault = $runTestAPI ? "will run" : "will not run"; |
| my $jscStressDefault = $runJSCStress ? "will run" : " will not run"; |
| my $usage = <<EOF; |
| Usage: $programName [options] [options to pass to build system] |
| --help Show this help message |
| --root= Path to pre-built root containing jsc |
| --[no-]ftl-jit Turn the FTL JIT on or off |
| --[no-]build Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault) |
| --[no-]testapi Run (or don't run) testapi (default: $testapiDefault) |
| --[no-]jsc-stress Run (or don't run) the JSC stress tests (default: $jscStressDefault) |
| |
| Mozilla tests only options (--no-jsc-stress): |
| ---------------------------------------------- |
| --jsDriver-args= A string of arguments to pass to jsDriver.pl |
| |
| JSC stress tests only options (--jsc-stress): |
| ---------------------------------------------- |
| --tarball Create a tarball of the bundle produced by running the JSC stress tests. |
| --remote= Run the JSC stress tests on the specified remote host. Implies --tarball. |
| --remote-config-file= Same as remote, but read config from JSON file. |
| --extra-tests= Path to a file containing extra tests |
| --child-processes= Specify the number of child processes. |
| --shell-runner Uses the shell-based test runner instead of the default make-based runner. |
| In general the shell runner is slower than the make runner. |
| --make-runner Uses the faster make-based runner. |
| |
| EOF |
| |
| GetOptions( |
| 'j|jsDriver-args=s' => \$jsDriverArgs, |
| 'root=s' => \$root, |
| 'extra-tests=s' => \$extraTests, |
| 'build!' => \$buildJSC, |
| 'ftl-jit!' => \$enableFTL, |
| 'testapi!' => \$runTestAPI, |
| 'jsc-stress!' => \$runJSCStress, |
| 'tarball!' => \$createTarball, |
| 'remote=s' => \$remoteHost, |
| 'remote-config-file=s' => \$remoteConfigFile, |
| 'child-processes=s' => \$childProcesses, |
| 'shell-runner' => \$shellRunner, |
| 'make-runner' => \$makeRunner, |
| 'help' => \$showHelp |
| ); |
| |
| # Assume any arguments left over from GetOptions are assumed to be build arguments |
| my @buildArgs = @ARGV; |
| |
| # The --ftl-jit argument gets passed as a build argument. |
| if ($enableFTL) { |
| push(@buildArgs, '--ftl-jit'); |
| } |
| |
| if ($showHelp) { |
| print STDERR $usage; |
| exit 1; |
| } |
| |
| setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root)); |
| |
| if (!defined($root) && $buildJSC) { |
| chdirWebKit(); |
| |
| push(@buildArgs, argumentsForConfiguration()); |
| |
| print "Running: build-jsc " . join(" ", @buildArgs) . "\n"; |
| my $buildResult = system "perl", "Tools/Scripts/build-jsc", @buildArgs; |
| if ($buildResult) { |
| print STDERR "Compiling jsc failed!\n"; |
| exit exitStatus($buildResult); |
| } |
| } |
| |
| |
| my $productDir = jscProductDir(); |
| $ENV{DYLD_FRAMEWORK_PATH} = $productDir; |
| $ENV{JSC_timeout} = 60 unless $ENV{JSC_timeout}; # Set a 60 second timeout on all jsc tests (if environment variable not defined already). |
| $ENV{TZ}="US/Pacific"; # Some tests fail if the time zone is not set to US/Pacific (<https://webkit.org/b/136363>) |
| setPathForRunningWebKitApp(\%ENV) if isCygwin(); |
| |
| sub testapiPath($) |
| { |
| my ($productDir) = @_; |
| my $jscName = "testapi"; |
| $jscName .= "_debug" if configuration() eq "Debug_All"; |
| return "$productDir/$jscName"; |
| } |
| |
| #run api tests |
| if ($runTestAPI) { |
| chdirWebKit(); |
| chdir($productDir) or die "Failed to switch directory to '$productDir'\n"; |
| my @command = (testapiPath($productDir)); |
| unshift @command, ("xcrun", "-sdk", xcodeSDK(), "sim") if willUseIOSSimulatorSDKWhenBuilding(); |
| |
| # Use an "indirect object" so that system() won't get confused if the path |
| # contains spaces (see perldoc -f exec). |
| my $testapiResult = system { $command[0] } @command; |
| my $exitStatus = exitStatus($testapiResult); |
| print "testAPI completed with rc=$testapiResult ($exitStatus)\n"; |
| exit exitStatus($testapiResult) if $testapiResult; |
| } |
| |
| # Find JavaScriptCore directory |
| chdirWebKit(); |
| |
| if ($runJSCStress) { |
| runJSCStressTests(); |
| } else { |
| runMozillaTests(); |
| } |
| |
| sub runMozillaTests |
| { |
| my %mozillaFailures; |
| my %newMozillaFailures; |
| chdir("Source/JavaScriptCore"); |
| chdir "tests/mozilla" or die "Failed to switch directory to 'tests/mozilla'\n"; |
| |
| # determine the list of skipped tests for Mozilla tests if --jsDriver-args command line argument isn't set |
| $jsDriverArgs ||= defaultJsDriverArgsForMozillaTests(); |
| |
| # Arguments passed to --jsDriver-args (if any) are passed to jsDriver.pl |
| my @jsArgs = split(" ", $jsDriverArgs); |
| |
| my @jsMozillaDriverCmd = ("perl", "jsDriver.pl", "-e", "squirrelfish", "-s", jscPath($productDir), "-f", "actual.html", @jsArgs); |
| if (isGtk() || isEfl()) { |
| my @jhbuildPrefix = sourceDir() . "/Tools/jhbuild/jhbuild-wrapper"; |
| |
| if (isEfl()) { |
| push(@jhbuildPrefix, '--efl'); |
| } elsif (isGtk()) { |
| push(@jhbuildPrefix, '--gtk'); |
| } |
| push(@jhbuildPrefix, 'run'); |
| |
| unshift(@jsMozillaDriverCmd, @jhbuildPrefix); |
| } elsif (isIOSWebKit() && willUseIOSSimulatorSDKWhenBuilding()) { |
| push @jsMozillaDriverCmd, ("--sdk", xcodeSDK()); |
| } |
| print "Running: " . join(" ", @jsMozillaDriverCmd) . "\n"; |
| my $result = system(@jsMozillaDriverCmd); |
| exit exitStatus($result) if $result; |
| |
| open EXPECTED, "expected.html" or die "Failed to open 'expected.html'\n"; |
| while (<EXPECTED>) { |
| last if /failures reported\.$/; |
| } |
| while (<EXPECTED>) { |
| chomp; |
| $mozillaFailures{$_} = 1; |
| } |
| close EXPECTED; |
| |
| open ACTUAL, "actual.html" or die "Failed to open 'actual.html'"; |
| while (<ACTUAL>) { |
| last if /failures reported\.$/; |
| } |
| while (<ACTUAL>) { |
| chomp; |
| if ($mozillaFailures{$_}) { |
| delete $mozillaFailures{$_}; |
| } else { |
| $newMozillaFailures{$_} = 1; |
| } |
| } |
| close ACTUAL; |
| |
| my $numNewMozillaFailures = keys %newMozillaFailures; |
| if ($numNewMozillaFailures) { |
| print "\n** The following Mozilla test failures have been introduced:\n"; |
| foreach my $mozillaFailure (sort keys %newMozillaFailures) { |
| print "\t$mozillaFailure\n"; |
| } |
| } |
| |
| my $numOldMozillaFailures = keys %mozillaFailures; |
| if ($numOldMozillaFailures) { |
| print "\nYou fixed the following test"; |
| print "s" if $numOldMozillaFailures != 1; |
| print ":\n"; |
| foreach my $mozillaFailure (sort keys %mozillaFailures) { |
| print "\t$mozillaFailure\n"; |
| } |
| } |
| |
| print "\nResults for Mozilla tests:\n"; |
| printThingsFound($numNewMozillaFailures, "regression", "regressions", "found"); |
| printThingsFound($numOldMozillaFailures, "test", "tests", "fixed"); |
| print " OK.\n" if $numNewMozillaFailures == 0; |
| print "\n"; |
| |
| exit(1) if $numNewMozillaFailures; |
| } |
| |
| sub runJSCStressTests |
| { |
| my $jscStressResultsDir = $productDir . "/jsc-stress-results"; |
| |
| # Set LANG environment variable so the stress tests will work with newer ruby (<rdar://problem/15010705>) |
| $ENV{LANG}="en_US.UTF-8"; |
| my @jscStressDriverCmd = ( |
| "/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests", |
| "-j", jscPath($productDir), "-o", $jscStressResultsDir, |
| "PerformanceTests/SunSpider/tests/sunspider-1.0", |
| "Source/JavaScriptCore/tests/executableAllocationFuzz.yaml", |
| "Source/JavaScriptCore/tests/exceptionFuzz.yaml", |
| "PerformanceTests/SunSpider/no-architecture-specific-optimizations.yaml", |
| "PerformanceTests/SunSpider/tests/v8-v6", |
| "Source/JavaScriptCore/tests/mozilla/mozilla-tests.yaml", |
| "Source/JavaScriptCore/tests/stress", |
| "LayoutTests/js/regress/script-tests", |
| "PerformanceTests/SunSpider/profiler-test.yaml", |
| "LayoutTests/jsc-layout-tests.yaml", |
| "Source/JavaScriptCore/tests/typeProfiler.yaml", |
| "Source/JavaScriptCore/tests/controlFlowProfiler.yaml" |
| ); |
| if (isWindows() && !isCygwin()) { |
| shift @jscStressDriverCmd; # Remove /usr/bin/env |
| } |
| if ($enableFTL) { |
| push(@jscStressDriverCmd, "--ftl-jit"); |
| } |
| if ($createTarball) { |
| push(@jscStressDriverCmd, "--tarball"); |
| } |
| |
| if ($remoteHost) { |
| push(@jscStressDriverCmd, "--remote"); |
| push(@jscStressDriverCmd, $remoteHost); |
| } |
| |
| if ($remoteConfigFile) { |
| push(@jscStressDriverCmd, "--remote-config-file"); |
| push(@jscStressDriverCmd, $remoteConfigFile); |
| } |
| |
| if ($childProcesses) { |
| push(@jscStressDriverCmd, "--child-processes"); |
| push(@jscStressDriverCmd, $childProcesses); |
| } |
| |
| if ($shellRunner) { |
| push(@jscStressDriverCmd, "--shell-runner"); |
| } |
| |
| if ($makeRunner) { |
| push(@jscStressDriverCmd, "--make-runner"); |
| } |
| |
| # End option processing, the rest of the arguments are tests |
| push(@jscStressDriverCmd, "--"); |
| |
| if (defined($extraTests)) { |
| push(@jscStressDriverCmd, $extraTests); |
| } |
| if (defined($ENV{"EXTRA_JSC_TESTS"})) { |
| push(@jscStressDriverCmd, $ENV{"EXTRA_JSC_TESTS"}); |
| } |
| print "Running: " . join(" ", @jscStressDriverCmd) . "\n"; |
| my $result = system(@jscStressDriverCmd); |
| exit exitStatus($result) if $result; |
| |
| my @jscStressFailList = readAllLines($jscStressResultsDir . "/failed"); |
| @jscStressFailList = sort @jscStressFailList; |
| my $numJSCStressFailures = @jscStressFailList; |
| |
| if ($numJSCStressFailures) { |
| print "\n** The following JSC stress test failures have been introduced:\n"; |
| foreach my $testFailure (@jscStressFailList) { |
| print "\t$testFailure"; |
| } |
| } |
| print "\n"; |
| |
| print "Results for JSC stress tests:\n"; |
| printThingsFound($numJSCStressFailures, "failure", "failures", "found"); |
| print " OK.\n" if $numJSCStressFailures == 0; |
| |
| print "\n"; |
| |
| exit(1) if $numJSCStressFailures; |
| } |
| |
| sub readAllLines |
| { |
| my ($filename) = @_; |
| my @array = (); |
| eval { |
| open FILE, $filename or die; |
| while (<FILE>) { |
| push @array, $_; |
| } |
| close FILE; |
| }; |
| return @array; |
| } |
| |
| sub printThingsFound |
| { |
| my ($number, $label, $pluralLabel, $verb) = @_; |
| print " $number "; |
| if ($number == 1) { |
| print $label; |
| } else { |
| print $pluralLabel; |
| } |
| print " $verb.\n"; |
| } |
| |
| sub defaultJsDriverArgsForMozillaTests |
| { |
| my @testsToSkipOnWindows = ( |
| # Various ecma/Date tests sometimes fail on Windows (but not Mac) https://bugs.webkit.org/show_bug.cgi?id=25160 |
| "ecma/Date/15.9.2.1.js", |
| "ecma/Date/15.9.2.2-1.js", |
| "ecma/Date/15.9.2.2-2.js", |
| "ecma/Date/15.9.2.2-3.js", |
| "ecma/Date/15.9.2.2-4.js", |
| "ecma/Date/15.9.2.2-5.js", |
| "ecma/Date/15.9.2.2-6.js", |
| "ecma/Date/15.9.5.31-1.js", |
| "ecma_3/Date/15.9.5.7.js", |
| "ecma/Date/15.9.3.1-1.js", |
| "ecma/Date/15.9.3.1-2.js", |
| "ecma/Date/15.9.3.1-3.js", |
| "ecma/Date/15.9.3.1-4.js", |
| "ecma/Date/15.9.3.1-5.js", |
| "ecma/Date/15.9.3.2-1.js", |
| "ecma/Date/15.9.3.2-2.js", |
| "ecma/Date/15.9.3.2-3.js", |
| "ecma/Date/15.9.3.2-4.js", |
| "ecma/Date/15.9.3.2-5.js", |
| "ecma/Date/15.9.4.2-1.js", |
| "ecma/Date/15.9.5.26-1.js", |
| "ecma/Date/15.9.5.27-1.js", |
| "ecma/Date/15.9.5.28-1.js", |
| "ecma/Date/15.9.5.30-1.js", |
| ); |
| |
| my @testsToSkipOnNonWindows = ( |
| # ecma_3/Date/15.9.5.7.js fails on Mac (but not Windows) https://bugs.webkit.org/show_bug.cgi?id=25161 |
| "ecma_3/Date/15.9.5.7.js", |
| ); |
| |
| my @testsToSkip = ( |
| # function.arguments have been disabled |
| "js1_4/Functions/function-001.js", |
| ); |
| |
| @testsToSkip = (@testsToSkip, @testsToSkipOnWindows) if isAppleWinWebKit(); |
| @testsToSkip = (@testsToSkip, @testsToSkipOnNonWindows) if !isAppleWinWebKit(); |
| my $jsDriverArgs = "-L " . join(" ", @testsToSkip) if @testsToSkip; |
| |
| return $jsDriverArgs; |
| } |
