Tools/Scripts/run-javascriptcore-tests

#!/usr/bin/perl -w

# Copyright (C) 2005, 2013 Apple Computer, Inc.  All rights reserved.
# Copyright (C) 2007 Eric Seidel <eric@webkit.org>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer. 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution. 
# 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
#     its contributors may be used to endorse or promote products derived
#     from this software without specific prior written permission. 
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Script to run the WebKit Open Source Project JavaScriptCore tests (adapted from Mozilla),
# as well as other tests: testapi on Mac and LayoutTests/js.

use strict;
use FindBin;
use Getopt::Long qw(:config pass_through);
use List::Util qw(min max);
use lib $FindBin::Bin;
use webkitdirs;
use POSIX;

# determine configuration
setConfiguration();
my $configuration = configuration();

my @testsToSkip = (
    # Various ecma/Date tests sometimes fail on Windows (but not Mac) https://bugs.webkit.org/show_bug.cgi?id=25160
    "ecma/Date/15.9.2.1.js",
    "ecma/Date/15.9.2.2-1.js",
    "ecma/Date/15.9.2.2-2.js",
    "ecma/Date/15.9.2.2-3.js",
    "ecma/Date/15.9.2.2-4.js",
    "ecma/Date/15.9.2.2-5.js",
    "ecma/Date/15.9.2.2-6.js",
    # ecma_3/Date/15.9.5.7.js fails on Mac (but not Windows) https://bugs.webkit.org/show_bug.cgi?id=25161
    "ecma_3/Date/15.9.5.6.js",
    "ecma_3/Date/15.9.5.7.js",
    # These three fail on Linux in certain time zones, at certain times
    # of the year (!): https://bugs.webkit.org/show_bug.cgi?id=71371
    "ecma/Date/15.9.5.14.js",
    "ecma/Date/15.9.5.31-1.js",
    "ecma/Date/15.9.5.34-1.js",
);

my $jsDriverArgs = "-L " . join(" ", @testsToSkip);
# These variables are intentionally left undefined.
my $root;
my $showHelp;
my $extraTests;

my $buildJSC = 1;

my $runTestAPI = isAppleMacWebKit() || isAppleWinWebKit();

my $runMozilla = 0;

# FIXME: run-jsc-stress-tests should be ported to other platforms.
# https://bugs.webkit.org/show_bug.cgi?id=120809
my $runJSCStress = !isAppleWinWebKit() && !isWinCairo();

my $enableFTL = 0;

my $programName = basename($0);
my $buildJSCDefault = $buildJSC ? "will check" : "will not check";
my $testapiDefault = $runTestAPI ? "will run" : "will not run";
my $mozillaDefault = $runMozilla ? "will run" : "will not run";
my $jscStressDefault = $runJSCStress ? "will run" : " will not run";
my $usage = <<EOF;
Usage: $programName [options] [options to pass to build system]
  --help                        Show this help message
  --jsDriver-args=              A string of arguments to pass to jsDriver.pl
  --root=                       Path to pre-built root containing jsc
  --extra-tests=                Path to a file containing extra tests
  --[no-]ftl-jit                Turn the FTL JIT on or off
  --[no-]build                  Check (or don't check) to see if the jsc build is up-to-date (default: $buildJSCDefault)
  --[no-]testapi                Run (or don't run) testapi (default: $testapiDefault)
  --[no-]jsc-stress             Run (or don't run) the JSC stress tests (default: $jscStressDefault)
EOF

GetOptions(
    'j|jsDriver-args=s' => \$jsDriverArgs,
    'root=s' => \$root,
    'extra-tests=s' => \$extraTests,
    'build!' => \$buildJSC,
    'ftl-jit!' => \$enableFTL,
    'testapi!' => \$runTestAPI,
    'jsc-stress!' => \$runJSCStress,
    'help' => \$showHelp
);

# Assume any arguments left over from GetOptions are assumed to be build arguments
my @buildArgs = @ARGV;

# The --ftl-jit argument gets passed as a build argument.
if ($enableFTL) {
    push(@buildArgs, '--ftl-jit');
}

# Arguments passed to --jsDriver-args (if any) are passed to jsDriver.pl
my @jsArgs = split(" ", $jsDriverArgs);

if ($showHelp) {
   print STDERR $usage;
   exit 1;
}

setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));

if (!defined($root) && $buildJSC) {
    chdirWebKit();

    push(@buildArgs, argumentsForConfiguration());
    
    print "Running: build-jsc " . join(" ", @buildArgs) . "\n";
    my $buildResult = system "perl", "Tools/Scripts/build-jsc", @buildArgs;
    if ($buildResult) {
        print STDERR "Compiling jsc failed!\n";
        exit exitStatus($buildResult);
    }
}


my $productDir = jscProductDir();
$ENV{DYLD_FRAMEWORK_PATH} = $productDir;
$ENV{JSC_timeout} = 60; # Set a 60 second timeout on all jsc tests.
setPathForRunningWebKitApp(\%ENV) if isCygwin();

sub testapiPath($)
{
    my ($productDir) = @_;
    my $jscName = "testapi";
    $jscName .= "_debug" if configuration() eq "Debug_All";
    return "$productDir/$jscName";
}

#run api tests
if ($runTestAPI) {
    chdirWebKit();
    chdir($productDir) or die "Failed to switch directory to '$productDir'\n";
    my @command = (testapiPath($productDir));
    unshift @command, ("xcrun", "-sdk", xcodeSDK(), "sim") if willUseIOSSimulatorSDKWhenBuilding();

    # Use an "indirect object" so that system() won't get confused if the path
    # contains spaces (see perldoc -f exec).
    my $testapiResult = system { $command[0] } @command;
    exit exitStatus($testapiResult)  if $testapiResult;
}

# Find JavaScriptCore directory
chdirWebKit();

$runMozilla = !$runJSCStress;

my %mozillaFailures;
my %newMozillaFailures;

if ($runMozilla) {
    chdir("Source/JavaScriptCore");
    chdir "tests/mozilla" or die "Failed to switch directory to 'tests/mozilla'\n";
    my @jsMozillaDriverCmd = ("perl", "jsDriver.pl", "-e", "squirrelfish", "-s", jscPath($productDir), "-f", "actual.html", @jsArgs);
    if (isGtk() || isEfl()) {
        my @jhbuildPrefix = sourceDir() . "/Tools/jhbuild/jhbuild-wrapper";
    
        if (isEfl()) {
            push(@jhbuildPrefix, '--efl');
        } elsif (isGtk()) {
            push(@jhbuildPrefix, '--gtk');
        }
        push(@jhbuildPrefix, 'run');
    
        unshift(@jsMozillaDriverCmd, @jhbuildPrefix);
    } elsif (isIOSWebKit() && willUseIOSSimulatorSDKWhenBuilding()) {
        push @jsMozillaDriverCmd, ("--sdk", xcodeSDK());
    }
    print "Running: " . join(" ", @jsMozillaDriverCmd) . "\n";
    my $result = system(@jsMozillaDriverCmd);
    exit exitStatus($result)  if $result;
    
    open EXPECTED, "expected.html" or die "Failed to open 'expected.html'\n";
    while (<EXPECTED>) {
        last if /failures reported\.$/;
    }
    while (<EXPECTED>) {
        chomp;
        $mozillaFailures{$_} = 1;
    }
    close EXPECTED;
    
    open ACTUAL, "actual.html" or die "Failed to open 'actual.html'";
    while (<ACTUAL>) {
        last if /failures reported\.$/;
    }
    while (<ACTUAL>) {
        chomp;
        if ($mozillaFailures{$_}) {
            delete $mozillaFailures{$_};
        } else {
            $newMozillaFailures{$_} = 1;
        }
    }
    close ACTUAL;
}

chdirWebKit();
my $jscStressResultsDir = $productDir . "/jsc-stress-results";

if ($runJSCStress) {
    # Set LANG environment variable so the stress tests will work with newer ruby (<rdar://problem/15010705>)
    $ENV{LANG}="en_US.UTF-8";
    my @jscStressDriverCmd = (
        "/usr/bin/env", "ruby", "Tools/Scripts/run-jsc-stress-tests",
        "-j", jscPath($productDir), "-o", $jscStressResultsDir,
        "PerformanceTests/SunSpider/tests/sunspider-1.0",
        "PerformanceTests/SunSpider/tests/v8-v6",
        "Source/JavaScriptCore/tests/mozilla/mozilla-tests.yaml",
        "LayoutTests/js/regress/script-tests",
        "PerformanceTests/SunSpider/profiler-test.yaml",
        "LayoutTests/jsc-layout-tests.yaml"
    );
    if ($enableFTL) {
        push(@jscStressDriverCmd, "--ftl-jit");
    }
    if (defined($extraTests)) {
        push(@jscStressDriverCmd, $extraTests);
    }
    if (defined($ENV{"EXTRA_JSC_TESTS"})) {
        push(@jscStressDriverCmd, $ENV{"EXTRA_JSC_TESTS"});
    }
    print "Running: " . join(" ", @jscStressDriverCmd) . "\n";
    my $result = system(@jscStressDriverCmd);
    exit exitStatus($result) if $result;
}

my $numNewMozillaFailures = keys %newMozillaFailures;
if ($numNewMozillaFailures) {
    print "\n** The following Mozilla test failures have been introduced:\n";
    foreach my $mozillaFailure (sort keys %newMozillaFailures) {
        print "\t$mozillaFailure\n";
    }
}

my $numOldMozillaFailures = keys %mozillaFailures;
if ($numOldMozillaFailures) {
    print "\nYou fixed the following test";
    print "s" if $numOldMozillaFailures != 1;
    print ":\n";
    foreach my $mozillaFailure (sort keys %mozillaFailures) {
        print "\t$mozillaFailure\n";
    }
}

sub readAllLines
{
    my ($filename) = @_;
    my @array = ();
    eval {
        open FILE, $filename or die;
        while (<FILE>) {
            push @array, $_;
        }
        close FILE;
    };
    return @array;
}

sub printThingsFound
{
    my ($number, $label, $pluralLabel, $verb) = @_;
    print "    $number ";
    if ($number == 1) {
        print $label;
    } else {
        print $pluralLabel;
    }
    print " $verb.\n";
}

my @jscStressFailList = readAllLines($jscStressResultsDir . "/failed");
my $numJSCStressFailures = @jscStressFailList;

if ($numJSCStressFailures) {
    print "\n** The following JSC stress test failures have been introduced:\n";
    foreach my $testFailure (@jscStressFailList) {
        print "\t$testFailure";
    }
}

print "\n";

if ($runMozilla) {
    print "Results for Mozilla tests:\n";
    printThingsFound($numNewMozillaFailures, "regression", "regressions", "found");
    printThingsFound($numOldMozillaFailures, "test", "tests", "fixed");
    print "    OK.\n" if $numNewMozillaFailures == 0;
    
    print "\n";
}

if ($runJSCStress) {
    print "Results for JSC stress tests:\n";
    printThingsFound($numJSCStressFailures, "failure", "failures", "found");
    print "    OK.\n" if $numJSCStressFailures == 0;
    
    print "\n";
}

exit(1)  if ($runMozilla && $numNewMozillaFailures) || ($runJSCStress && $numJSCStressFailures);