| #!/usr/bin/env perl |
| |
| # Copyright (C) 2005-2017 Apple Inc. All rights reserved. |
| # Copyright (C) 2007 Eric Seidel <eric@webkit.org> |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # 3. Neither the name of Apple Inc. ("Apple") nor the names of |
| # its contributors may be used to endorse or promote products derived |
| # from this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY |
| # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| # Script to run the WebKit Open Source Project JavaScriptCore tests (adapted from Mozilla), |
| # as well as other tests: testapi on Mac and LayoutTests/js. |
| |
| use strict; |
| use warnings; |
| use File::Spec; |
| use FindBin; |
| use Getopt::Long qw(:config pass_through); |
| use JSON::PP; |
| use lib $FindBin::Bin; |
| use List::Util qw(min max); |
| use POSIX; |
| use webkitdirs; |
| use Text::ParseWords; |
| |
| # determine configuration |
| setConfiguration(); |
| my $configuration = configuration(); |
| |
| # These variables are intentionally left undefined. |
| my $root; |
| my $showHelp; |
| my @extraTests = (); |
| my $childProcesses; |
| my $shellRunner; |
| my $makeRunner; |
| my $rubyRunner; |
| my $testWriter; |
| my $memoryLimited; |
| |
| my $report; |
| my $buildbotMaster; |
| my $builderName; |
| my $buildNumber; |
| my $buildbotWorker; |
| |
| my $buildJSC = 1; |
| my $copyJSC = 1; |
| |
| use constant { |
| ENV_VAR_SAYS_DO_RUN => 4, |
| ENV_VAR_SAYS_DONT_RUN => 3, |
| RUN_IF_NO_TESTS_SPECIFIED => 2, |
| DO_RUN => 1, |
| DONT_RUN => 0, |
| }; |
| |
| my $runTestMasm = RUN_IF_NO_TESTS_SPECIFIED; |
| my $runTestAir = RUN_IF_NO_TESTS_SPECIFIED; |
| my $runTestB3 = RUN_IF_NO_TESTS_SPECIFIED; |
| my $runTestDFG = RUN_IF_NO_TESTS_SPECIFIED; |
| my $runTestAPI = RUN_IF_NO_TESTS_SPECIFIED; |
| my $runJSCStress = RUN_IF_NO_TESTS_SPECIFIED; |
| my $runMozillaTests = RUN_IF_NO_TESTS_SPECIFIED; |
| |
| # $runJITStressTests is special because it is not for enabling a different set of tests. |
| # Instead it is only meaningful for when we want to disable using JIT test configurations |
| # on the JSC stress test or mozilla tests. |
| my $runJITStressTests = 1; |
| |
| my $runQuickMode = 0; |
| |
| my $forceCollectContinuously = 0; |
| my $envVars = ""; |
| my $gmallocPath = undef; |
| my $gmallocDefaultPath = "/usr/lib/libgmalloc.dylib"; |
| |
| my $createTarball = 0; |
| my $remoteHost = 0; |
| my $model = 0; |
| my $version; |
| my $versionName; |
| my $sdk; |
| my $failFast = 1; |
| my %jsonData = (); |
| my %reportData = (); |
| my @testResults = (); |
| my $isTestFailed = 0; |
| my $remoteConfigFile; |
| my $jsonFileName; |
| my $verbose = 0; |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM}) { |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM} eq "true") { |
| $runTestMasm = ENV_VAR_SAYS_DO_RUN; |
| } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM} eq "false") { |
| $runTestMasm = ENV_VAR_SAYS_DONT_RUN; |
| } else { |
| print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTMASM environment variable: '" |
| . $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTMASM} . "'. Should be set to 'true' or 'false'.\n"; |
| } |
| } |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR}) { |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR} eq "true") { |
| $runTestAir = ENV_VAR_SAYS_DO_RUN; |
| } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR} eq "false") { |
| $runTestAir = ENV_VAR_SAYS_DONT_RUN; |
| } else { |
| print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTAIR environment variable: '" |
| . $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAIR} . "'. Should be set to 'true' or 'false'.\n"; |
| } |
| } |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3}) { |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3} eq "true") { |
| $runTestB3 = ENV_VAR_SAYS_DO_RUN; |
| } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3} eq "false") { |
| $runTestB3 = ENV_VAR_SAYS_DONT_RUN; |
| } else { |
| print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTB3 environment variable: '" |
| . $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTB3} . "'. Should be set to 'true' or 'false'.\n"; |
| } |
| } |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG}) { |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG} eq "true") { |
| $runTestDFG = ENV_VAR_SAYS_DO_RUN; |
| } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG} eq "false") { |
| $runTestDFG = ENV_VAR_SAYS_DONT_RUN; |
| } else { |
| print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTDFG environment variable: '" |
| . $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTDFG} . "'. Should be set to 'true' or 'false'.\n"; |
| } |
| } |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI}) { |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} eq "true") { |
| $runTestAPI = ENV_VAR_SAYS_DO_RUN; |
| } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} eq "false") { |
| $runTestAPI = ENV_VAR_SAYS_DONT_RUN; |
| } else { |
| print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_TESTAPI environment variable: '" |
| . $ENV{RUN_JAVASCRIPTCORE_TESTS_TESTAPI} . "'. Should be set to 'true' or 'false'.\n"; |
| } |
| } |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD}) { |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} eq "true") { |
| $buildJSC = 1; |
| } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} eq "false") { |
| $buildJSC = 0; |
| } else { |
| print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_BUILD environment variable: '" |
| . $ENV{RUN_JAVASCRIPTCORE_TESTS_BUILD} . "'. Should be set to 'true' or 'false'.\n"; |
| } |
| } |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_COPY}) { |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_COPY} eq "true") { |
| $copyJSC = 1; |
| } elsif ($ENV{RUN_JAVASCRIPTCORE_TESTS_COPY} eq "false") { |
| $copyJSC = 0; |
| } else { |
| print "Don't recognize value for RUN_JAVASCRIPTCORE_TESTS_COPY environment variable: '" |
| . $ENV{RUN_JAVASCRIPTCORE_TESTS_COPY} . "'. Should be set to 'true' or 'false'.\n"; |
| } |
| } |
| |
| if ($ENV{RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS}) { |
| push @extraTests, $ENV{RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS}; |
| } |
| |
| sub defaultStringForTestState { |
| my ($state) = @_; |
| if ($state == ENV_VAR_SAYS_DONT_RUN) { |
| return "will not run due to environment variable"; |
| } elsif ($state == DONT_RUN) { |
| return "will not run"; |
| } else { |
| return "will run"; |
| } |
| } |
| |
| # Additional environment parameters |
| push @ARGV, parse_line('\s+', 0, $ENV{'TEST_JSC_ARGS'}) if ($ENV{'TEST_JSC_ARGS'}); |
| |
| my $programName = basename($0); |
| my $buildJSCDefault = $buildJSC ? "will check" : "will not check"; |
| my $testmasmDefault = defaultStringForTestState($runTestMasm); |
| my $testairDefault = defaultStringForTestState($runTestAir); |
| my $testb3Default = defaultStringForTestState($runTestB3); |
| my $testDFGDefault = defaultStringForTestState($runTestDFG); |
| my $testapiDefault = defaultStringForTestState($runTestAPI); |
| my $jscStressDefault = defaultStringForTestState($runJSCStress); |
| my $mozillaTestsDefault = defaultStringForTestState($runMozillaTests); |
| my $jitStressTestsDefault = $runJITStressTests ? "will run" : " will not run"; |
| my $quickModeDefault = $runQuickMode ? "some" : "all"; |
| my $failFastDefault = $failFast ? "fail fast" : "don't fail fast"; |
| my $copyJSCDefault = $copyJSC ? "copy" : "do not copy"; |
| my $filter; |
| my $usage = <<EOF; |
| Usage: $programName [options] [options to pass to build system] |
| --help Show this help message |
| --root= Path to pre-built root containing jsc |
| --[no-]ftl-jit Turn the FTL JIT on or off |
| --[no-]build Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault) |
| --[no-]testmasm Only run (or don't run) testmasm (default: $testmasmDefault) |
| --[no-]testair Only run (or don't run) testair (default: $testairDefault) |
| --[no-]testb3 Only run (or don't run) testb3 (default: $testb3Default) |
| --[no-]testdfg Only run (or don't run) testdfg (default: $testDFGDefault) |
| --[no-]testapi Only run (or don't run) testapi (default: $testapiDefault) |
| --[no-]jsc-stress Only run (or don't run) the JSC stress tests (default: $jscStressDefault) |
| --[no-]mozilla-tests Only run (or don't run) the Mozilla tests (default: $mozillaTestsDefault) |
| --[no-]jit-stress-tests Run (or don't run) the JIT stress tests (default: $jitStressTestsDefault) |
| --[no-]quick Run some (or all) of the regular testing modes (default: $quickModeDefault) |
| If the runner only runs some it will run the default and no-cjit-validate modes. |
| Note, this will not change the behavior of tests that specify their own modes. |
| |
| --[no-]fail-fast Stop this script when a test family reports an error or failure (default: $failFastDefault) |
| --[no-]force-collectContinuously Enable the collectContinuously mode even if it was disabled on this platform. |
| --[no-]copy Copy (or don't copy) the JavaScriptCore build product before testing (default: $copyJSCDefault) |
| --json-output= Create a file at specified path, listing failed stress tests in JSON format. |
| --tarball Create a tarball of the bundle produced by running the JSC stress tests. |
| --remote= Run the JSC stress tests on the specified remote host. Implies --tarball. |
| --remote-config-file= Same as remote, but read config from JSON file. |
| --model= Specify remote hardware model, this info used for determine what jsc tests should run on remote |
| --version Specify the version number of the device running tests. |
| --version-name Specify the version name of the hardware running tests. |
| --sdk Specific SDK or OS version of the form ##*### |
| --extra-tests= Path to a file containing extra tests |
| --child-processes= Specify the number of child processes. |
| --shell-runner Uses the shell-based test runner instead of the default make-based runner. |
| In general the shell runner is slower than the make runner. |
| --make-runner Uses the faster make-based runner. |
| --ruby-runner Uses the ruby runner for machines without unix shell or make. |
| |
| --test-writer [writer] Specifies the test script format." |
| default is to use shell scripts to run the tests" |
| \"ruby\" to use ruby scripts for systems without a unix shell. |
| |
| --memory-limited Indicate that we are targeting the test for a memory limited device. |
| Skip tests tagged with //\@skip if \$memoryLimited |
| |
| --filter Only run tests whose name matches the given regular expression. |
| --env-vars Pass a list of environment variables to set before running tests. |
| Each environment variable should be separated by a space. |
| e.g. \"foo=bar x=y\" (no quotes). |
| --gmalloc: Run tests with Guard Malloc enabled (if no path is given: $gmallocDefaultPath is used) |
| --verbose: Verbose output (specify more than once to increase verbosity). |
| |
| --report: Results database url to report results to. |
| --buildbot-master: The url of the buildbot master. |
| --builder-name: The name of the buildbot builder tests were run on. |
| --build-number: The buildbot build number tests are associated with. |
| --buildbot-worker: The buildbot worker tests were run on. |
| |
| Environment Variables: |
| - set RUN_JAVASCRIPTCORE_TESTS_TESTMASM to "true" or "false" (no quotes) to determine if we run testmasm by default. |
| - set RUN_JAVASCRIPTCORE_TESTS_TESTAIR to "true" or "false" (no quotes) to determine if we run testair by default. |
| - set RUN_JAVASCRIPTCORE_TESTS_TESTB3 to "true" or "false" (no quotes) to determine if we run testb3 by default. |
| - set RUN_JAVASCRIPTCORE_TESTS_TESTDFG to "true" or "false" (no quotes) to determine if we run testdfg by default. |
| - set RUN_JAVASCRIPTCORE_TESTS_TESTAPI to "true" or "false" (no quotes) to determine if we run testapi by default. |
| - set RUN_JAVASCRIPTCORE_TESTS_BUILD to "true" or "false" (no quotes) to set the should-we-build-before-running-tests setting. |
| - set RUN_JAVASCRIPTCORE_TESTS_EXTRA_TESTS to the path of a yaml file or a directory of JS files to be run as part of run-javascriptcore-tests. |
| |
| If one or more --<testname> options are specified, only those tests will run. For example, |
| the following only runs testmasm and testapi: |
| \$ run-javascriptcore-tests --testmasm --testapi |
| |
| Otherwise, all tests will run unless the test is disabled using --no-<testname> or an |
| environment variable. |
| EOF |
| |
| GetOptions( |
| 'root=s' => \$root, |
| 'extra-tests=s' => \@extraTests, |
| 'build!' => \$buildJSC, |
| 'testmasm!' => \$runTestMasm, |
| 'testair!' => \$runTestAir, |
| 'testb3!' => \$runTestB3, |
| 'testdfg!' => \$runTestDFG, |
| 'testapi!' => \$runTestAPI, |
| 'jsc-stress!' => \$runJSCStress, |
| 'mozilla-tests!' => \$runMozillaTests, |
| 'jit-stress-tests!' => \$runJITStressTests, |
| 'quick!' => \$runQuickMode, |
| 'fail-fast!' => \$failFast, |
| 'force-collectContinuously!' => \$forceCollectContinuously, |
| 'copy!' => \$copyJSC, |
| 'json-output=s' => \$jsonFileName, |
| 'tarball!' => \$createTarball, |
| 'remote=s' => \$remoteHost, |
| 'model=s' => \$model, |
| 'version=s' => \$version, |
| 'version-name=s' => \$versionName, |
| 'sdk=s' => \$sdk, |
| 'remote-config-file=s' => \$remoteConfigFile, |
| 'child-processes=s' => \$childProcesses, |
| 'shell-runner' => \$shellRunner, |
| 'make-runner' => \$makeRunner, |
| 'ruby-runner' => \$rubyRunner, |
| 'test-writer=s' => \$testWriter, |
| 'memory-limited' => \$memoryLimited, |
| 'filter=s' => \$filter, |
| 'help' => \$showHelp, |
| 'env-vars=s' => \$envVars, |
| 'gmalloc:s' => \$gmallocPath, |
| 'verbose+' => \$verbose, |
| 'report=s' => \$report, |
| 'buildbot-master=s' => \$buildbotMaster, |
| 'builder-name=s' => \$builderName, |
| 'build-number=s' => \$buildNumber, |
| 'buildbot-worker=s' => \$buildbotWorker, |
| ); |
| |
| |
| my $specificTestsSpecified = 0; |
| if ($runTestMasm == DO_RUN |
| || $runTestAir == DO_RUN |
| || $runTestB3 == DO_RUN |
| || $runTestDFG == DO_RUN |
| || $runTestAPI == DO_RUN |
| || $runJSCStress == DO_RUN |
| || $runMozillaTests == DO_RUN) { |
| $specificTestsSpecified = 1; |
| } |
| |
| if ($version) { |
| $version = splitVersionString($version); |
| } |
| |
| sub enableTestOrNot { |
| my ($state) = @_; |
| if ($state == RUN_IF_NO_TESTS_SPECIFIED || $state == ENV_VAR_SAYS_DO_RUN) { |
| return $specificTestsSpecified ? DONT_RUN : DO_RUN; |
| } elsif ($state == ENV_VAR_SAYS_DONT_RUN) { |
| return DONT_RUN; |
| } |
| return $state; |
| } |
| |
| sub configurationForUpload() |
| { |
| my $platform; |
| my $sdk; |
| my $simulator = 0; |
| |
| if (index(xcodeSDKPlatformName(), "simulator") != -1) { |
| $simulator = 1; |
| } |
| |
| if (index($model, 'iPhone') != -1 || index($model, 'iPad') != -1) { |
| $platform = 'ios'; |
| if (!$version) { |
| $version = iosVersion(); |
| } |
| } elsif (index($model, 'watch') != -1 || index(xcodeSDKPlatformName(), "watch") != -1) { |
| $platform = 'watchos'; |
| die "No watchOS version specified" if !$version; |
| } elsif (index(xcodeSDKPlatformName(), "appletv") != -1) { |
| $platform = 'tvos'; |
| die "No tvOS version specified" if !$version; |
| } elsif (isGtk()) { |
| $platform = 'GTK'; |
| if (!$version) { |
| chomp($version = `uname -r`); |
| $version = splitVersionString($version); |
| } |
| } elsif (isWPE()) { |
| $platform = 'WPE'; |
| if (!$version) { |
| chomp($version = `uname -r`); |
| $version = splitVersionString($version); |
| } |
| } elsif (isAnyWindows()) { |
| $platform = 'win'; |
| if (!$version) { |
| $version = winVersion(); |
| } |
| } elsif (isAppleMacWebKit()) { |
| $platform = 'mac'; |
| chomp($model = `/usr/sbin/sysctl -n hw.model`); |
| if (!$version) { |
| $version = osXVersion(); |
| } |
| if (!$sdk) { |
| chomp($sdk = `/usr/bin/sw_vers -buildVersion`); |
| } |
| |
| if (!$versionName) { |
| if ($version->{minor} eq 15) { |
| $versionName = "Catalina"; |
| } elsif ($version->{minor} eq 14) { |
| $versionName = "Mojave"; |
| } elsif ($version->{minor} eq 13) { |
| $versionName = "High Sierra"; |
| } |
| } |
| } else { |
| # FIXME: This will be the kernel version, which is not always exactly what we want, but it will at least |
| # always give us something to work with |
| chomp($platform = `uname`); |
| $platform = lc $platform; |
| if (!$version) { |
| chomp($version = `uname -r`); |
| $version = splitVersionString($version); |
| } |
| } |
| |
| my $result = { |
| platform => $platform, |
| architecture => architecture(), |
| is_simulator => $simulator, |
| style => lc(configuration()), |
| version => "$version->{major}.$version->{minor}.$version->{subminor}", |
| }; |
| if ($model) { |
| $result->{model} = $model; |
| } |
| if ($versionName) { |
| $result->{version_name} = $versionName; |
| } |
| if ($sdk) { |
| $result->{sdk} = $sdk; |
| } |
| return $result; |
| } |
| |
| $runTestMasm = enableTestOrNot($runTestMasm); |
| $runTestAir = enableTestOrNot($runTestAir); |
| $runTestB3 = enableTestOrNot($runTestB3); |
| $runTestDFG = enableTestOrNot($runTestDFG); |
| $runTestAPI = enableTestOrNot($runTestAPI); |
| $runJSCStress = enableTestOrNot($runJSCStress); |
| $runMozillaTests = enableTestOrNot($runMozillaTests); |
| |
| # Assume any arguments left over from GetOptions are assumed to be build arguments |
| my @buildArgs = @ARGV; |
| |
| if ($showHelp) { |
| print STDERR $usage; |
| exit 1; |
| } |
| |
| setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root)); |
| configurationForUpload() if (defined($report)); |
| |
| if (defined($jsonFileName)) { |
| $jsonFileName = File::Spec->rel2abs($jsonFileName); |
| } |
| |
| if (!defined($root) && $buildJSC) { |
| chdirWebKit(); |
| |
| push(@buildArgs, argumentsForConfiguration()); |
| |
| print "Running: build-jsc " . join(" ", @buildArgs) . "\n"; |
| my $buildResult = system "perl", File::Spec->catfile("Tools", "Scripts", "build-jsc"), @buildArgs; |
| if ($buildResult) { |
| print STDERR "Compiling jsc failed!\n"; |
| exit exitStatus($buildResult); |
| } |
| } |
| |
| if (defined($gmallocPath)) { |
| if ($gmallocPath eq "") { |
| $envVars .= " DYLD_INSERT_LIBRARIES=" . $gmallocDefaultPath; |
| } else { |
| $envVars .= " DYLD_INSERT_LIBRARIES=" . $gmallocPath; |
| } |
| } |
| |
| my $productDir = jscProductDir(); |
| $ENV{DYLD_FRAMEWORK_PATH} = $productDir; |
| $ENV{JSCTEST_timeout} = 120 unless $ENV{JSCTEST_timeout}; # Set a 120 second timeout on all jsc tests (if environment variable not defined already). |
| $ENV{TZ}="US/Pacific"; # Some tests fail if the time zone is not set to US/Pacific (<https://webkit.org/b/136363>) |
| setPathForRunningWebKitApp(\%ENV) if isCygwin(); |
| |
| my $startTime = time(); |
| my $endTime = $startTime; |
| |
| sub testPath { |
| my ($productDir, $testName) = @_; |
| $testName .= "_debug" if configuration() eq "Debug_All"; |
| return File::Spec->catfile($productDir, $testName); |
| } |
| |
| sub runTest { |
| my ($testName, $jsonTestStatusName) = @_; |
| |
| chdirWebKit(); |
| chdir($productDir) or die "Failed to switch directory to '$productDir'\n"; |
| my @command = (testPath($productDir, $testName)); |
| unshift @command, ("xcrun", "-sdk", xcodeSDK(), "sim") if willUseIOSSimulatorSDK(); |
| unshift @command, wrapperPrefixIfNeeded() if shouldUseJhbuild(); |
| |
| if ($envVars ne "") { |
| foreach my $var (split(/\s+/, $envVars)) { |
| if ($var =~ /([^=]*)=(.*)/) { |
| $ENV{$1} = $2; |
| } |
| } |
| } |
| |
| my $testResult = 0; |
| my $failureFlag = 0; |
| my $lastOptimizeLevel; |
| |
| open(TEST, "-|", "@command 2>&1") or die "Failed to run @command"; |
| while ( my $line = <TEST> ) { |
| my $subTestName; |
| print $line if ($verbose); |
| |
| # e.g.: 'O2: testRotRWithImmShift(uint64-max, uint32-min): OK!'' |
| if ($line =~ /(O\d): (\S+)(\(.*\)): ([A-Za-z]+)!\n/) { |
| $lastOptimizeLevel = $1; |
| $subTestName = "$testName.$1.$2$3"; |
| $failureFlag += 1 if ($4 ne "OK"); |
| } |
| # Assertion in binary |
| # e.g.: ASSERTION FAILED: things[0] == 2 |
| elsif ($line =~ /ASSERTION FAILED:/) { |
| $failureFlag += 1; |
| next; |
| } |
| # e.g.: file/path(397) : void (anonymous namespace)::testShuffleSimpleBroadcast() |
| elsif ($failureFlag && $line =~ /.*\(\d+\) : \S+ .*::(\S+)(\(.*\))/) { |
| if (defined $lastOptimizeLevel) { |
| $subTestName = "$testName.$lastOptimizeLevel.$1$2"; |
| } else { |
| $subTestName = "$testName.$1$2"; |
| } |
| } |
| # sub-test from the c-api, of the form: |
| # e.g.: 'testapi[15422:16904790] parallelPromiseResolveTest(): done.'' |
| elsif ($line =~ /\[\d+:\d+\] (\S+)(): done\.\n/) { |
| $subTestName = "$testName.objective-c.$1"; |
| } |
| # End of C-API tests (meaning we just finished the ungrouped main tests) |
| elsif ($line =~ /Starting C-API tests in C++/) { |
| $subTestName = "$testName.objective-c.main"; |
| } |
| # End of testapi |
| # e.g.: FAIL: Some tests failed. |
| elsif ($line =~ /FAIL: Some tests failed\./) { |
| $failureFlag = 0; |
| next; |
| } |
| # Test cases inside API tests |
| # e.g.: 'testapi[15422:16904790] TEST: "2 + 2": PASSED' |
| elsif ($line =~ /\[\d+:\d+\] TEST: ".*": ([A-Z]+)\n/) { |
| $failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS"); |
| next; |
| } |
| # e.g.: 'testBitXorAndAndArgs(0, 0,uint32-max): OK!'' |
| elsif ($line =~ /(\S+)(\(.*\)): ([A-Za-z]+)!\n/) { |
| $subTestName = "$testName.$1$2"; |
| $failureFlag += 1 if ($3 ne "OK"); |
| } |
| # General testcase statement, filtered to exclude some junk logging |
| # e.g.: 'PASS: DFG script timed out as expected when no callback is specified.'' |
| elsif ($line =~ /([A-Z]+): (.*)\.*\n/ && !($line =~ /[A-Z]+: \d+\n/ || $line =~ /\[\d+:\d+\] [A-Z]+:/)) { |
| $subTestName = "$testName.$2"; |
| $failureFlag += 1 if ($1 ne "PASS"); |
| } |
| # e.g.: 'test description: PASSED' |
| elsif ($line =~ /.*: ([A-Z]+)\n/) { |
| $failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS"); |
| next; |
| } else { |
| next; |
| } |
| |
| if (!defined $reportData{$subTestName}) { |
| print "$subTestName failed\n" if ($failureFlag && !$verbose); |
| $reportData{$subTestName} = $failureFlag ? {actual => "FAIL"} : {actual => "PASS"}; |
| } |
| $failureFlag = 0; |
| } |
| $testResult = close(TEST) ? 0 : $?; |
| |
| my $exitStatus = exitStatus($testResult); |
| print "$testName completed with rc=$testResult ($exitStatus)\n\n"; |
| |
| if ($testResult) { |
| $isTestFailed = 1; |
| push @testResults, $testName; |
| } |
| if (defined($jsonFileName)) { |
| my $testStatus = ($exitStatus == 0)? JSON::PP::true: JSON::PP::false; |
| $jsonData{$jsonTestStatusName} = $testStatus; |
| } |
| |
| if ($testResult && $failFast) { |
| reportTestFailures(); |
| writeJsonDataIfApplicable(); |
| |
| $endTime = time(); |
| uploadResults(); |
| exit $exitStatus; |
| } |
| } |
| |
| sub reportTestFailures { |
| my $numJSCtestFailures = @testResults; |
| if ($numJSCtestFailures) { |
| print "\n** The following JSC test binaries failures have been introduced:\n"; |
| foreach my $testFailure (@testResults) { |
| print "\t$testFailure\n"; |
| } |
| } |
| print "\n"; |
| |
| print "Results for JSC test binaries:\n"; |
| printThingsFound($numJSCtestFailures, "failure", "failures", "found"); |
| print " OK.\n" if $numJSCtestFailures == 0; |
| |
| print "\n"; |
| } |
| |
| if ($runTestMasm) { runTest("testmasm", "allMasmTestsPassed") } |
| if ($runTestAir) { runTest("testair", "allAirTestsPassed") } |
| if ($runTestB3) { runTest("testb3", "allB3TestsPassed") } |
| if ($runTestDFG) { runTest("testdfg", "allDFGTestsPassed") } |
| if ($runTestAPI) { runTest("testapi", "allApiTestsPassed") } |
| |
| |
| # Find JavaScriptCore directory |
| chdirWebKit(); |
| |
| # FIXME: report stress test failures https://bugs.webkit.org/show_bug.cgi?id=204096 |
| runJSCStressTests(); |
| reportTestFailures(); |
| |
| $endTime = time(); |
| uploadResults(); |
| |
| if ($isTestFailed) { |
| exit(1); |
| } |
| |
| sub runJSCStressTests |
| { |
| my $jscStressResultsDir = $productDir . "/jsc-stress-results"; |
| |
| my $hasTestsToRun = 0; |
| my @testList; |
| if ($runJSCStress) { |
| @testList = ( |
| "PerformanceTests/SunSpider/tests/sunspider-1.0", |
| "PerformanceTests/JetStream/cdjs/cdjs-tests.yaml", |
| "PerformanceTests/ARES-6/Air/airjs-tests.yaml", |
| "PerformanceTests/ARES-6/Basic/basic-tests.yaml", |
| "JSTests/executableAllocationFuzz.yaml", |
| "JSTests/exceptionFuzz.yaml", |
| "PerformanceTests/SunSpider/no-architecture-specific-optimizations.yaml", |
| "PerformanceTests/SunSpider/shadow-chicken.yaml", |
| "PerformanceTests/SunSpider/tests/v8-v6", |
| "JSTests/stress", |
| "JSTests/microbenchmarks", |
| "JSTests/slowMicrobenchmarks.yaml", |
| "PerformanceTests/SunSpider/profiler-test.yaml", |
| "LayoutTests/jsc-layout-tests.yaml", |
| "JSTests/typeProfiler.yaml", |
| "JSTests/controlFlowProfiler.yaml", |
| "JSTests/es6.yaml", |
| "JSTests/modules.yaml", |
| "JSTests/complex.yaml", |
| "JSTests/ChakraCore.yaml", |
| "JSTests/wasm.yaml"); |
| $hasTestsToRun = 1; |
| } |
| |
| if ($runMozillaTests) { |
| push(@testList, "JSTests/mozilla/mozilla-tests.yaml"); |
| $hasTestsToRun = 1; |
| } |
| |
| # Set LANG environment variable so the stress tests will work with newer ruby (<rdar://problem/15010705>) |
| $ENV{LANG}="en_US.UTF-8"; |
| my @jscStressDriverCmd = ( |
| "/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests", |
| "-j", jscPath($productDir), "-o", $jscStressResultsDir); |
| |
| push(@jscStressDriverCmd, @testList); |
| |
| if (isWindows() && !isCygwin()) { |
| shift @jscStressDriverCmd; # Remove /usr/bin/env |
| } |
| |
| if (configuration() eq "Debug") { |
| push(@jscStressDriverCmd, "--debug"); |
| } |
| |
| if (!$copyJSC) { |
| push(@jscStressDriverCmd, "--no-copy"); |
| } |
| |
| if ($forceCollectContinuously) { |
| push(@jscStressDriverCmd, "--force-collectContinuously"); |
| } |
| |
| if ($envVars ne "") { |
| push(@jscStressDriverCmd, "--env-vars"); |
| push(@jscStressDriverCmd, $envVars); |
| } |
| |
| if ($runQuickMode) { |
| push(@jscStressDriverCmd, "--quick"); |
| } |
| |
| if (!$runJITStressTests) { |
| push(@jscStressDriverCmd, "--no-jit"); |
| } |
| if ($createTarball) { |
| push(@jscStressDriverCmd, "--tarball"); |
| } |
| |
| if ($remoteHost) { |
| push(@jscStressDriverCmd, "--remote"); |
| push(@jscStressDriverCmd, $remoteHost); |
| } |
| |
| if ($remoteConfigFile) { |
| push(@jscStressDriverCmd, "--remote-config-file"); |
| push(@jscStressDriverCmd, $remoteConfigFile); |
| } |
| |
| if ($model) { |
| push(@jscStressDriverCmd, "--model"); |
| push(@jscStressDriverCmd, $model); |
| } |
| |
| if ($childProcesses) { |
| push(@jscStressDriverCmd, "--child-processes"); |
| push(@jscStressDriverCmd, $childProcesses); |
| } |
| |
| if ($shellRunner) { |
| push(@jscStressDriverCmd, "--shell-runner"); |
| } |
| |
| if ($makeRunner) { |
| push(@jscStressDriverCmd, "--make-runner"); |
| } |
| |
| if ($rubyRunner) { |
| push(@jscStressDriverCmd, "--ruby-runner"); |
| } |
| |
| if ($testWriter) { |
| push(@jscStressDriverCmd, "--test-writer"); |
| push(@jscStressDriverCmd, $testWriter); |
| } |
| |
| if ($memoryLimited) { |
| push(@jscStressDriverCmd, "--memory-limited"); |
| } |
| |
| if ($filter) { |
| push(@jscStressDriverCmd, "--filter"); |
| push(@jscStressDriverCmd, $filter); |
| } |
| |
| push(@jscStressDriverCmd, ("--verbose") x $verbose) if ($verbose > 0); |
| |
| if (isPlayStation()) { |
| push(@jscStressDriverCmd, "--os=playstation"); |
| push(@jscStressDriverCmd, "--no-copy"); |
| } |
| |
| unshift @jscStressDriverCmd, wrapperPrefixIfNeeded() if shouldUseJhbuild(); |
| |
| # End option processing, the rest of the arguments are tests |
| push(@jscStressDriverCmd, "--"); |
| |
| for my $testSuite (@extraTests) { |
| push(@jscStressDriverCmd, $testSuite); |
| $hasTestsToRun = 1; |
| } |
| if (defined($ENV{"EXTRA_JSC_TESTS"})) { |
| push(@jscStressDriverCmd, $ENV{"EXTRA_JSC_TESTS"}); |
| $hasTestsToRun = 1; |
| } |
| |
| if (!$hasTestsToRun) { |
| exit(0); |
| } |
| |
| print "Running: " . join(" ", @jscStressDriverCmd) . "\n"; |
| my $result = system(@jscStressDriverCmd); |
| exit exitStatus($result) if $result; |
| |
| my @jscStressFailList = readAllLines($jscStressResultsDir . "/failed"); |
| @jscStressFailList = sort @jscStressFailList; |
| my $numJSCStressFailures = @jscStressFailList; |
| |
| if ($numJSCStressFailures) { |
| $isTestFailed = 1; |
| print "\n** The following JSC stress test failures have been introduced:\n"; |
| foreach my $testFailure (@jscStressFailList) { |
| print "\t$testFailure\n"; |
| } |
| } |
| print "\n"; |
| |
| print "Results for JSC stress tests:\n"; |
| printThingsFound($numJSCStressFailures, "failure", "failures", "found"); |
| print " OK.\n" if $numJSCStressFailures == 0; |
| |
| print "\n"; |
| |
| if (defined($jsonFileName)) { |
| $jsonData{'stressTestFailures'} = \@jscStressFailList; |
| } |
| |
| writeJsonDataIfApplicable(); |
| } |
| |
| sub readAllLines |
| { |
| my ($filename) = @_; |
| my @array = (); |
| eval { |
| open FILE, $filename or die; |
| while (<FILE>) { |
| chomp; |
| push @array, $_; |
| } |
| close FILE; |
| }; |
| return @array; |
| } |
| |
| sub printThingsFound |
| { |
| my ($number, $label, $pluralLabel, $verb) = @_; |
| print " $number "; |
| if ($number == 1) { |
| print $label; |
| } else { |
| print $pluralLabel; |
| } |
| print " $verb.\n"; |
| } |
| |
| sub writeJsonDataIfApplicable |
| { |
| if (defined($jsonFileName)) { |
| open(my $fileHandler, ">", $jsonFileName) or die; |
| print $fileHandler "${\encode_json(\%jsonData)}\n"; |
| close($fileHandler); |
| } |
| } |
| |
| sub uploadResults |
| { |
| if (not defined $report) { |
| print "Skipping upload to results database since no report URL was specified\n"; |
| return 0; |
| } |
| |
| my %upload = ( |
| version => 0, |
| suite => 'javascriptcore-tests', |
| # FIXME: Integrate branches, https://bugs.webkit.org/show_bug.cgi?id=204094 |
| # FIXME: Integrate the Safari repository, https://bugs.webkit.org/show_bug.cgi?id=204095 |
| commits => [{ |
| repository_id => 'webkit', |
| id => determineCurrentSVNRevision(), |
| }], |
| configuration => configurationForUpload(), |
| test_results => { |
| run_stats => { |
| start_time => $startTime, |
| end_time => $endTime, |
| tests_skipped => 0, |
| }, |
| results => \%reportData, |
| }, |
| timestamp => $startTime, |
| ); |
| if (defined $buildbotMaster and defined $builderName and defined $buildNumber and defined $buildbotWorker) { |
| $upload{test_results}{details} = { |
| 'buildbot-master' => $buildbotMaster, |
| 'builder-name' => $builderName, |
| 'build-number' => $buildNumber, |
| 'buildbot-worker' => $buildbotWorker, |
| }; |
| } else { |
| print " No buildbot details provided, test run will not be linked to CI system\n"; |
| } |
| if (defined $ENV{'RESULTS_SERVER_API_KEY'}) { |
| $upload{'api_key'} = $ENV{'RESULTS_SERVER_API_KEY'}; |
| } |
| |
| print "Uploading results to $report\n"; |
| |
| open(HANDLE, "|-", "curl -X POST $report/api/upload -H 'Content-Type: application/json' -f -d '\@-'") or die "Failed to open curl"; |
| |
| # Json conforming to https://results.webkit.org/documentation#API-Uploads. |
| my $encodedUpload = encode_json(\%upload); |
| print HANDLE "$encodedUpload\n\0"; |
| my $success = close HANDLE; |
| |
| if ($success) { |
| print "Upload successful!\n"; |
| return 0; |
| } |
| print "Upload to $report failed\n"; |
| return 1; |
| } |
