#!/usr/bin/env perl

# Copyright (C) 2010-2012, 2014-2015 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.

use strict;
use warnings;

use File::Basename;
use File::Spec;
use FindBin;
use Getopt::Long qw(:config pass_through);
use IPC::Open3;
use JSON::PP;
use lib $FindBin::Bin;
use sigtrap qw(die normal-signals);
use webkitdirs;
use VCSUtils;

sub buildTestTool();
sub dumpTestsBySuite(\@);
sub listAllTests();
sub runTest($$);
sub runTestsBySuite(\@);
sub prepareEnvironmentForRunningTestTool();
sub archCommandLineArgumentsForRestrictedEnvironmentVariables();
sub testToolPaths();
sub writeJSONDataIfApplicable();

# Defined in VCSUtils.
sub possiblyColored($$);

# Timeout for individual test, in sec
my $timeout = 30;

my $showHelp = 0;
my $verbose = 0;
my $showLeaks = 0;
my $dumpTests = 0;
my $disableTimeout = 0;
my $build = 1;
my $root;
my $buildDefault = $build ? "build" : "do not build";
my @testsFailed;
my @testsTimedOut;
my $wtfOnly = 0;
my %testToToolMap;
my %jsonData = ();
my $jsonFileName;


my $programName = basename($0);
my $usage = <<EOF;
Usage: $programName [options] [suite or test prefixes]
  --help                Show this help message
  -v|--verbose          Verbose output
  -d|--dump-tests       Dump the names of testcases without running them
  --[no-]build          Build (or do not build) unit tests prior to running (default: $buildDefault)
  --json-output=        Create a file at the specified path, listing test failures and timeouts in JSON format.
  --root=               Path to the pre-built root containing TestWebKitAPI
  --show-leaks          Show leaks in the output
  --no-timeout          Disable test timeouts
  --wtf-only            Only build and run TestWTF

Platform options:
  --ios-simulator       Run tests in the iOS Simulator
  --simulator           DEPRECATED alias of --ios-simulator

@{[ sharedCommandLineOptionsUsage(indent => 2, switchWidth => 21) ]}
Examples

The following command will run a single test:
    $programName WebKit.AboutBlank

The following command will run all tests in suites that begin with 'WebKit':
    $programName WebKit

EOF

my $getOptionsResult = GetOptions(
    sharedCommandLineOptions(),
    'help' => \$showHelp,
    'verbose|v' => \$verbose,
    'show-leaks' => \$showLeaks,
    'no-timeout' => \$disableTimeout,
    'json-output=s' => \$jsonFileName,
    'dump|d' => \$dumpTests,
    'build!' => \$build,
    'root=s' => \$root,
    'wtf-only' => \$wtfOnly,
);

if (!$getOptionsResult || $showHelp) {
   print STDERR $usage;
   exit 1;
}

setConfiguration();

setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));

if (defined($jsonFileName)) {
    $jsonFileName = File::Spec->rel2abs($jsonFileName);
}

buildTestTool() if $build && !defined($root);
setPathForRunningWebKitApp(\%ENV);

my $simulatorDevice;
if (willUseIOSSimulatorSDK()) {
    $simulatorDevice = findOrCreateSimulatorForIOSDevice(SIMULATOR_DEVICE_SUFFIX_FOR_WEBKIT_DEVELOPMENT);
    relaunchIOSSimulator($simulatorDevice);
}
my @testsToRun = listAllTests();

@testsToRun = grep { my $test = $_; grep { $test =~ m/^\Q$_\E/ } @ARGV; } @testsToRun if @ARGV;

if ($dumpTests) {
    dumpTestsBySuite(@testsToRun);
    exit 0;
}

END { shutDownIOSSimulatorDevice($simulatorDevice) if $simulatorDevice; }

exit runTestsBySuite(@testsToRun);

sub isSupportedPlatform()
{
    return isAppleCocoaWebKit() || isAppleWinWebKit();
}

sub dumpTestsBySuite(\@)
{
    my ($tests) = @_;
    print "Dumping test cases\n";
    print "------------------\n";
    my $lastSuite = "";
    for my $suiteAndTest (sort @$tests) {
        my ($suite, $test) = split(/\./, $suiteAndTest);
        if ($lastSuite ne $suite) {
            $lastSuite = $suite;
            print "$suite:\n";
        }
        print "   $test\n";
    }
    print "------------------\n";
}

sub runTestsBySuite(\@)
{
    my ($tests) = @_;
    for my $suiteAndTest (sort @$tests) {
        my ($suite, $test) = split(/\./, $suiteAndTest);
        runTest($suite, $test);
    }

    if (@testsFailed) {
        print "\nTests that failed:\n";
        for my $test (@testsFailed) {
            print "  $test\n";
        }
    }
    if (@testsTimedOut) {
        print "\nTests that timed out:\n";
        for my $test (@testsTimedOut) {
            print "  $test\n";
        }
    }

    if (defined($jsonFileName)) {
        $jsonData{'failures'} = \@testsFailed;
        $jsonData{'timeouts'} = \@testsTimedOut;
    }

    writeJSONDataIfApplicable();

    return @testsFailed > 0 || @testsTimedOut > 0;
}

sub runTest($$)
{
    my ($suite, $testName) = @_;
    my $test = $suite . "." . $testName;

    my $gtestArg = "--gtest_filter=" . $test;

    my $result = 0;
    my $timedOut = 0;

    die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();

    local %ENV = %ENV;
    prepareEnvironmentForRunningTestTool();

    local *DEVNULL;
    my ($childIn, $childOut, $childErr);
    if ($verbose || $showLeaks) {
        $childErr = 0;
    } else {
        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
        $childErr = ">&DEVNULL";
    }

    my $pid;
    my @commonArguments = ($testToToolMap{$test}, $gtestArg, @ARGV);
    if (willUseIOSSimulatorSDK()) {
        $pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to run test: $test.";
    } elsif (isAppleCocoaWebKit() && architecture()) {
        $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to run test: $test.";
    } else {
        $pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to run test: $test.";
    }

    eval {
        if ($disableTimeout) {
            waitpid($pid, 0);    
        } else {
            local $SIG{ALRM} = sub { die "alarm\n" };
            alarm $timeout;
            waitpid($pid, 0);
            alarm 0;
        }
        $result = $?;
    };
    if ($@) {
        die unless $@ eq "alarm\n";
        kill SIGTERM, $pid or kill SIGKILL, $pid;
        $timedOut = 1;
    }

    my @testOutput = <$childOut>;
    @testOutput = grep { !/^LEAK:/ } @testOutput unless $showLeaks;
    map { s/\*\*PASS\*\*/possiblyColored("bold green", "PASS")/eg } @testOutput;
    map { s/\*\*FAIL\*\*/possiblyColored("bold red", "FAIL")/eg } @testOutput;

    if ($result) {
        push @testsFailed, $test;
        if (!$timedOut && index("@testOutput", $test) == -1) {
            print STDOUT possiblyColored("bold red", "UNEXPECTEDLY EXITED"), " $test\n";
        }
    } elsif ($timedOut) {
        push @testsTimedOut, $test;
        print STDOUT possiblyColored("bold yellow", "TIMEOUT"), " $test\n";
    }

    print STDOUT @testOutput;

    close($childIn);
    close($childOut);
    close($childErr) unless ($verbose || $showLeaks);
    close(DEVNULL) unless ($verbose || $showLeaks);

    if ($timedOut || $result) {
        return $timedOut || $result;
    }

    return 0;
}

sub listAllTests()
{
    my @toolOutput;
    my $timedOut;

    die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();

    prepareEnvironmentForRunningTestTool();

    local *DEVNULL;
    my ($childIn, $childOut, $childErr);
    if ($verbose) {
        $childErr = ">&STDERR";
    } else {
        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
        $childErr = ">&DEVNULL";
    }

    my @tests = ();
    foreach (testToolPaths()) {
        my $pid;
        my $testTool = $_;
        my @commonArguments = ($testTool, "--gtest_list_tests");
        if (isIOSWebKit()) {
            $pid = open3($childIn, $childOut, $childErr, qw(xcrun --sdk iphonesimulator simctl spawn), $simulatorDevice->{UDID}, @commonArguments) or die "Failed to build list of tests!";
        } elsif (isAppleCocoaWebKit() && architecture()) {
            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), archCommandLineArgumentsForRestrictedEnvironmentVariables(), @commonArguments) or die "Failed to build list of tests!";
        } else {
            $pid = open3($childIn, $childOut, $childErr, @commonArguments) or die "Failed to build list of tests!";
        }

        close($childIn);
        @toolOutput = <$childOut>;
        close($childOut);
        close($childErr);

        waitpid($pid, 0);
        my $result = $?;

        if ($result) {
            print STDERR "Failed to build list of tests!--\n";
            exit exitStatus($result);
        }

        my $suite;
        for my $line (@toolOutput) {
           $line =~ s/[\r\n]*$//;
           if ($line =~ m/\.$/) {
              $suite = $line; # "SuiteName."
           } else {
              $line =~ s/^\s*//; # "TestName"
              my $fullName = $suite . $line; # "SuiteName.TestName";
              push @tests, $fullName;
              $testToToolMap{$fullName} = $testTool;
            }
        }
    }

    close(DEVNULL) unless ($verbose);

    return @tests;
}

sub buildTestTool()
{
    my $originalCwd = getcwd();

    chdirWebKit();

    my $buildTestTool = "build-api-tests";
    print STDERR "Running $buildTestTool\n";

    local *DEVNULL;
    my ($childIn, $childOut, $childErr);
    if ($verbose) {
        # When not quiet, let the child use our stdout/stderr.
        $childOut = ">&STDOUT";
        $childErr = ">&STDERR";
    } else {
        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
        $childOut = ">&DEVNULL";
        $childErr = ">&DEVNULL";
    }

    my @args = argumentsForConfiguration();
    if ($wtfOnly) {
        push @args, "--wtf-only";
    }

    my $pathToBuildTestTool = File::Spec->catfile("Tools", "Scripts", $buildTestTool);
    my $buildProcess = open3($childIn, $childOut, $childErr, "perl", $pathToBuildTestTool, @args) or die "Failed to run " . $buildTestTool;

    close($childIn);
    close($childOut);
    close($childErr);
    close(DEVNULL) unless ($verbose);

    waitpid($buildProcess, 0);
    my $buildResult = $?;

    if ($buildResult) {
        print STDERR "Compiling TestWebKitAPI failed!\n";
        exit exitStatus($buildResult);
    }

    chdir $originalCwd;
}

sub prepareEnvironmentForRunningTestTool()
{
    return unless isAppleCocoaWebKit();

    if (willUseIOSSimulatorSDK()) {
        my %simulatorENV;
        {
            local %ENV;
            setupIOSWebKitEnvironment(productDir());
            %simulatorENV = %ENV;
        }
        # Prefix the environment variables with SIMCTL_CHILD_ per `xcrun simctl help launch`.
        foreach my $key (keys %simulatorENV) {
            $ENV{"SIMCTL_CHILD_$key"} = $simulatorENV{$key};
        }
        return;
    }
    setupMacWebKitEnvironment(productDir());
}

sub testToolPaths()
{
    if (!isAppleWinWebKit()) {
        my @toolPaths = ();
        if (!$wtfOnly) {
            push @toolPaths, File::Spec->catfile(productDir(), "TestWebKitAPI");
        }
        push @toolPaths, File::Spec->catfile(productDir(), "TestWTF");
        return @toolPaths;
    }

    my $binDir = isWin64() ? "bin64" : "bin32";
    my $pathWTF = File::Spec->catfile(productDir(), $binDir, "TestWTF");
    my $pathWebCore = File::Spec->catfile(productDir(), $binDir, "TestWebCore");
    my $pathWebKit = File::Spec->catfile(productDir(), $binDir, "TestWebKitLegacy");

    my $suffix;
    if (configuration() eq "Debug_All") {
        $suffix = "_debug";
    } else {
        $suffix = "";
    }
    return ("$pathWTF$suffix.exe", "$pathWebCore$suffix.exe", "$pathWebKit$suffix.exe");
}

sub writeJSONDataIfApplicable()
{
    if (defined($jsonFileName)) {
        open(my $fileHandler, ">", $jsonFileName) or die;
        print $fileHandler "${\encode_json(\%jsonData)}\n";
        close($fileHandler);
    }
}