#!/usr/bin/env ruby

# Copyright (C) 2013-2016 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer. 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution. 
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

require 'fileutils'
require 'getoptlong'
require 'pathname'
require 'rbconfig'
require 'uri'
require 'yaml'

module URI
    class SSH < Generic
        DEFAULT_PORT = 22
    end
    @@schemes['SSH'] = SSH
end

class String
    def scrub
        encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
    end
end

RemoteHost = Struct.new(:name, :user, :host, :port, :remoteDirectory)

THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
JETSTREAM2_PATH = WEBKIT_PATH + "PerformanceTests/JetStream2"
CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"

HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"

begin
    require 'shellwords'
rescue Exception => e
    $stderr.puts "Warning: did not find shellwords, not running any tests."
    exit 0
end

$canRunDisplayProfilerOutput = false

begin
    require 'rubygems'
    require 'json'
    require 'highline'
    $canRunDisplayProfilerOutput = true
rescue Exception => e
    $stderr.puts "Warning: did not find json or highline; some features will be disabled."
    $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
    $stderr.puts "Error: #{e.inspect}"
end

def printCommandArray(*cmd)
    begin
        commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
    rescue
        commandArray = cmd.join(' ')
    end
    $stderr.puts ">> #{commandArray}"
end

def mysys(*cmd)
    printCommandArray(*cmd) if $verbosity >= 1
    raise "Command failed: #{$?.inspect}" unless system(*cmd)
end

def escapeAll(array)
    array.map {
        | v |
        raise "Detected a non-string in #{inspect}" unless v.is_a? String
        Shellwords.shellescape(v)
    }.join(' ')
end


$jscPath = nil
$doNotMessWithVMPath = false
$jitTests = true
$memoryLimited = false
$outputDir = Pathname.new("results")
$verbosity = 0
$bundle = nil
$tarball = false
$tarFileName = "payload.tar.gz"
$copyVM = false
$testRunnerType = nil
$testWriter = "default"
$remoteHosts = []
$architecture = nil
$forceArchitecture = nil
$hostOS = nil
$model = nil
$filter = nil
$envVars = []
$mode = "full"
$buildType = "release"
$forceCollectContinuously = false
$reportExecutionTime = false

def usage
    puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
    puts
    puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
    puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
    puts "                            --jsc specifies an already present JavaScriptCore to test."
    puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
    puts "                            Skip tests tagged with //@skip if $memoryLimited"
    puts "--no-jit                    Do not run JIT specific tests."
    puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
    puts "                            platform."
    puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
    puts "--verbose            (-v)   Print more things while running."
    puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
    puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
    puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
    puts "--force-architecture        Override the architecture to run tests with."
    puts "                            e.g. x86, x86_64, arm."
    puts "--os                        Specify os instead of determining from JavaScriptCore build."
    puts "                            e.g. darwin, linux & windows."
    puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
    puts "                            In general the shell runner is slower than the make runner."
    puts "--make-runner               Uses the faster make-based runner."
    puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
    puts "--test-writer [writer]      Specifies the test script format."
    puts "                            default is to use shell scripts to run the tests"
    puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
    puts "--remote                    Specify a remote host on which to run tests from command line argument."
    puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
    puts "--report-execution-time     Print execution time for each test."
    puts "--child-processes    (-c)   Specify the number of child processes."
    puts "--filter                    Only run tests whose name matches the given regular expression."
    puts "--help               (-h)   Print this message."
    puts "--env-vars                  Add a list of environment variables to set before running jsc."
    puts "                            Each environment variable should be separated by a space."
    puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
    puts "                            it will override the default value."
    puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
    puts "--basic                     Run with default and these additional modes: no-llint,"
    puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
    puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
    puts "                            ftl-no-cjit-small-pool."
    exit 1
end

jscArg = nil

GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
               ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
               ['--no-copy', GetoptLong::NO_ARGUMENT],
               ['--memory-limited', GetoptLong::NO_ARGUMENT],
               ['--no-jit', GetoptLong::NO_ARGUMENT],
               ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
               ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
               ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
               ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
               ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
               ['--arch', GetoptLong::REQUIRED_ARGUMENT],
               ['--force-architecture', GetoptLong::REQUIRED_ARGUMENT],
               ['--os', GetoptLong::REQUIRED_ARGUMENT],
               ['--shell-runner', GetoptLong::NO_ARGUMENT],
               ['--make-runner', GetoptLong::NO_ARGUMENT],
               ['--ruby-runner', GetoptLong::NO_ARGUMENT],
               ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
               ['--remote', GetoptLong::REQUIRED_ARGUMENT],
               ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
               ['--report-execution-time', GetoptLong::NO_ARGUMENT],
               ['--model', GetoptLong::REQUIRED_ARGUMENT],
               ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
               ['--filter', GetoptLong::REQUIRED_ARGUMENT],
               ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
               ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
               ['--debug', GetoptLong::NO_ARGUMENT],
               ['--release', GetoptLong::NO_ARGUMENT],
               ['--quick', '-q', GetoptLong::NO_ARGUMENT],
               ['--basic', GetoptLong::NO_ARGUMENT]).each {
    | opt, arg |
    case opt
    when '--help'
        usage
    when '--jsc'
        jscArg = arg
    when '--no-copy'
        $doNotMessWithVMPath = true
    when '--output-dir'
        $outputDir = Pathname.new(arg)
    when '--memory-limited'
        $memoryLimited = true
    when '--no-jit'
        $jitTests = false
    when '--force-collectContinuously'
        $forceCollectContinuously = true;
    when '--verbose'
        $verbosity += 1
    when '--run-bundle'
        $bundle = Pathname.new(arg)
    when '--tarball'
        $tarball = true
        $copyVM = true
        $tarFileName = arg unless arg == ''
    when '--force-vm-copy'
        $copyVM = true
    when '--shell-runner'
        $testRunnerType = :shell
    when '--make-runner'
        $testRunnerType = :make
    when '--ruby-runner'
        $testRunnerType = :ruby
    when '--test-writer'
        $testWriter = arg
    when '--remote'
        $copyVM = true
        $tarball = true
        $remote = true
        uri = URI("ssh://" + arg)
        $remoteHosts << RemoteHost.new("default-#{$remoteHosts.length}", uri.user, uri.host, uri.port)
    when '--remote-config-file'
        $remoteConfigFile = arg
    when '--report-execution-time'
        $reportExecutionTime = true
    when '--child-processes'
        $numChildProcesses = arg.to_i
    when '--filter'
        $filter = Regexp.new(arg)
    when '--arch'
        $architecture = arg
    when '--force-architecture'
        $architecture = arg unless $architecture
        $forceArchitecture = arg
    when '--os'
        $hostOS = arg
    when '--model'
        $model = arg
    when '--env-vars'
        $envVars = arg.gsub(/\s+/, ' ').split(' ')
    when '--quick'
        $mode = "quick"
    when '--basic'
        $mode = "basic"
    when '--debug'
        $buildType = "debug"
    when '--release'
        $buildType = "release"
    end
}

if $remoteConfigFile
    file = File.read($remoteConfigFile)
    config = JSON.parse(file)

    # old style config allowing for only one remote
    if !$remote and config['remote']
        $copyVM = true
        $tarball = true
        $remote = true
        uri = URI("ssh://" + config['remote'])
        $remoteHosts = [ RemoteHost.new("default", uri.user, uri.host, uri.port) ]
        if config['remoteDirectory']
            $remoteHosts[0].remoteDirectory = config['remoteDirectory']
        end
    end

    # we can combine --remote and a new style config
    if config['remotes']
        $copyVM = true
        $tarball = true
        $remote = true
        $remoteHosts += config['remotes'].map {
            | remote |
            uri = URI("ssh://" + remote['address'])

            host = RemoteHost.new(remote['name'], uri.user, uri.host, uri.port)
            if remote['remoteDirectory']
                host.remoteDirectory = remote['remoteDirectory']
            end
            host
        }
    end
end

unless jscArg
    # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
    command = SCRIPTS_PATH.join("webkit-build-directory").to_s
    command += ($buildType == "release") ? " --release" : " --debug"
    command += " --executablePath"

    output = `#{command}`.split("\n")
    if !output.length
        $stderr.puts "Error: must specify --jsc <path>"
        exit 1
    end

    output = output[0]
    jscArg = Pathname.new(output).join("jsc")
    jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Helpers", "jsc") if !File.file?(jscArg)
    jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
    if !File.file?(jscArg)
        $stderr.puts "Error: must specify --jsc <path>"
        exit 1
    end

    puts "Using the following jsc path: #{jscArg}"
end

if $doNotMessWithVMPath
    $jscPath = Pathname.new(jscArg)
else
    $jscPath = Pathname.new(jscArg).realpath
end

$progressMeter = ($verbosity == 0 and $stdout.tty? and $remoteHosts.length <= 1)

if $bundle
    $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Helpers" + "jsc"
    $outputDir = $bundle
end

# Try to determine architecture. Return nil on failure.
def machOArchitectureCode
    begin 
        otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
        otoolLines.each_with_index {
            | value, index |
            if value =~ /magic/ and value =~ /cputype/
                return otoolLines[index + 1].split[1].to_i
            end
        }
    rescue
        $stderr.puts "Warning: unable to execute otool."
    end
    $stderr.puts "Warning: unable to determine architecture."
    nil
end

def determineArchitectureFromMachOBinary
    code = machOArchitectureCode
    return nil unless code
    is64BitFlag = 0x01000000
    case code
    when 7
        "x86"
    when 7 | is64BitFlag
        "x86-64"
    when 12
        "arm"
    when 12 | is64BitFlag
        "arm64"
    else
        $stderr.puts "Warning: unable to determine architecture from code: #{code}"
        nil
    end
end

def determineArchitectureFromELFBinary
    f = File.open($jscPath.to_s)
    data = f.read(20)

    if !(data[0,4] == "\x7F\x45\x4C\x46")
        $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
        return nil
    end

    # MIPS and PowerPC may be either big- or little-endian. S390 (which includes
    # S390x) is big-endian. The rest are little-endian.
    code = data[18, 20]
    case code
    when "\x03\0"
        "x86"
    when "\x08\0"
        "mips"
    when "\0\x08"
        "mips"
    when "\x14\0"
        "powerpc"
    when "\0\x14"
        "powerpc"
    when "\x15\0"
        "powerpc64"
    when "\0\x15"
        "powerpc64"
    when "\0\x16"
        "s390"
    when "\x28\0"
        "arm"
    when "\x3E\0"
        "x86-64"
    when "\xB7\0"
        "arm64"
    else
        $stderr.puts "Warning: unable to determine architecture from code: #{code}"
        nil
    end
end

def determineArchitectureFromPEBinary
    f = File.open($jscPath.to_s)
    data = f.read(1024)

    if !(data[0, 2] == "MZ")
        $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
        return nil
    end

    peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian

    if !(data[peHeaderAddr, 4] == "PE\0\0")
        $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
        return nil
    end

    machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian

    case machine
    when 0x014c
        "x86"
    when 0x8664
        "x86-64"
    else
        $stderr.puts "Warning: unsupported machine type: #{machine}"
        nil
    end
end

def determineArchitecture
    case $hostOS
    when "darwin"
        determineArchitectureFromMachOBinary
    when "linux"
        determineArchitectureFromELFBinary
    when "windows"
        determineArchitectureFromPEBinary
    when "playstation"
        "x86-64"
    else
        $stderr.puts "Warning: unable to determine architecture on this platform."
        nil
    end
end

def determineOS
    case RbConfig::CONFIG["host_os"]
    when /darwin/i
        "darwin"
    when /linux/i
        "linux"
    when /mswin|mingw|cygwin/
        "windows"
    else
        $stderr.puts "Warning: unable to determine host operating system"
        nil
    end
end

$hostOS = determineOS unless $hostOS
$architecture = determineArchitecture unless $architecture
$isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows" || $hostOS == "playstation")

if $architecture == "x86"
    # The JIT is temporarily disabled on this platform since
    # https://trac.webkit.org/changeset/237547
    $jitTests = false
end

def isFTLEnabled
    $jitTests && $isFTLPlatform
end

if !$testRunnerType
    if $remote and $hostOS == "darwin"
        $testRunnerType = :shell
    else
        $testRunnerType = :make
    end
end

if $remoteHosts.length > 1 and $testRunnerType != :make
    raise "Multiple remote hosts only supported with make runner"
end

if $hostOS == "playstation" && $testWriter == "default"
    $testWriter = "playstation"
end

if $testWriter
    if /[^-a-zA-Z0-9_]/.match($testWriter)
        raise "Invalid test writer #{$testWriter} given"
    end
end

$numFailures = 0
$numPasses = 0

# We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--thresholdForOMGOptimizeAfterWarmUp=20", "--thresholdForOMGOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true", "--repatchBufferingCountdown=0"]
# NOTE: Tests rely on this using scribbleFreeCells.
NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
B3O0_OPTIONS = ["--maxDFGNodesInBasicBlockForPreciseAnalysis=100", "--defaultB3OptLevel=0"]
FTL_OPTIONS = ["--useFTLJIT=true"]
FORCE_LLINT_EXIT_OPTIONS = ["--forceOSRExitToLLInt=true"]

require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"

def shouldCollectContinuously?
    $buildType == "release" or $forceCollectContinuously
end

COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []

$runlist = []

def frameworkFromJSCPath(jscPath)
    parentDirectory = jscPath.dirname
    if (parentDirectory.basename.to_s == "Resources" or parentDirectory.basename.to_s == "Helpers") and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
        parentDirectory.dirname
    elsif $hostOS == "playstation"
        jscPath.dirname
    elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
        jscPath.dirname + "JavaScriptCore.framework"
    else
        $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
        nil
    end
end

def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
    dir = Pathname.new(".")
    $benchmarkDirectory.each_filename {
        | pathComponent |
        dir += ".."
    }
    dir + resourcePath
end

def pathToVM
    pathToBundleResourceFromBenchmarkDirectory($jscPath)
end

def vmCommand
    if ($forceArchitecture)
        ["/usr/bin/arch", "-" + $forceArchitecture,  pathToVM.to_s]
    else
        [pathToVM.to_s]
    end
end

def pathToHelpers
    pathToBundleResourceFromBenchmarkDirectory(".helpers")
end

$runCommandOptions = {}
$testSpecificRequiredOptions = []

$uniqueFilenameCounter = 0
def uniqueFilename(extension)
    payloadDir = $outputDir + "_payload"
    Dir.mkdir payloadDir unless payloadDir.directory?
    result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
    $uniqueFilenameCounter += 1
    result
end

def baseOutputName(kind)
    "#{$collectionName}/#{$benchmark}.#{kind}"
end

def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
    $didAddRunCommand = true
    name = baseOutputName(kind)
    if $filter and name !~ $filter
        return
    end
    plan = Plan.new(
        $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
        errorHandler)
    plan.additionalEnv.push(*additionalEnv)
    if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
        $runlist.unshift plan
    else
        $runlist << plan
    end
end

# Returns true if there were run commands found in the file ($benchmarkDirectory +
# $benchmark), in which case those run commands have already been executed. Otherwise
# returns false, in which case you're supposed to add your own run commands.
def parseRunCommands
    oldDidAddRunCommand = $didAddRunCommand
    $didAddRunCommand = false
    $skipped = false

    Dir.chdir($outputDir) {
        File.open($benchmarkDirectory + $benchmark) {
            | inp |
            inp.each_line {
                | line |
                begin
                    doesMatch = line =~ /^\/\/@/
                rescue Exception => e
                    # Apparently this happens in the case of some UTF8 stuff in some files, where
                    # Ruby tries to be strict and throw exceptions.
                    next
                end
                next unless doesMatch
                eval $~.post_match
                if $skipped
                    break
                end
            }
        }
    }

    result = $didAddRunCommand
    $didAddRunCommand = result or oldDidAddRunCommand
    result
end

def slow!
    $runCommandOptions[:isSlow] = true
    skip() if ($mode == "quick")
end

def crashOK!
    $testSpecificRequiredOptions += ["-s"]
    $runCommandOptions[:crashOK] = true
end

def requireOptions(*options)
    $testSpecificRequiredOptions += options
end

def runWithOutputHandler(kind, outputHandler, *options)
    addRunCommand(kind, vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
end

def runWithOutputHandlerWithoutBaseOption(kind, outputHandler, *options)
    addRunCommand(kind, vmCommand + $testSpecificRequiredOptions + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
end

def run(kind, *options)
    runWithOutputHandler(kind, silentOutputHandler, *options)
end

def runWithoutBaseOption(kind, *options)
    runWithOutputHandlerWithoutBaseOption(kind, silentOutputHandler, *options)
end

def runNoFTL(*optionalTestSpecificOptions)
    run("no-ftl", *optionalTestSpecificOptions)
end

def runWithRAMSize(size, *optionalTestSpecificOptions)
    run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
end

def runOneLargeHeap(*optionalTestSpecificOptions)
    if $memoryLimited
        $didAddRunCommand = true
        puts "Skipping #{$collectionName}/#{$benchmark}"
    else
        run("default", *optionalTestSpecificOptions)
    end
end

def runNoJIT(*optionalTestSpecificOptions)
    run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
end

def runNoLLInt(*optionalTestSpecificOptions)
    if $jitTests
        run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
    end
end

# NOTE: Tests rely on this using scribbleFreeCells.
def runNoCJITValidate(*optionalTestSpecificOptions)
    run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runNoCJITValidatePhases(*optionalTestSpecificOptions)
    run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", "--useRandomizingExecutableIslandAllocation=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
    run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
end

def runDefault(*optionalTestSpecificOptions)
    run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
end

def runBytecodeCacheImpl(optionalTestSpecificOptions, *additionalEnv)
    unless $hostOS == "darwin"
        skip
        return
    end
    options = BASE_OPTIONS + $testSpecificRequiredOptions + FTL_OPTIONS + optionalTestSpecificOptions
    addRunCommand("bytecode-cache", ["sh", (pathToHelpers + "bytecode-cache-test-helper.sh").to_s, pathToVM.to_s, $benchmark.to_s] + options, silentOutputHandler, simpleErrorHandler, *additionalEnv)
end

def runBytecodeCache(*optionalTestSpecificOptions)
    runBytecodeCacheImpl(optionalTestSpecificOptions)
end

def runBytecodeCacheNoAssertion(*optionalTestSpecificOptions)
    runBytecodeCacheImpl(optionalTestSpecificOptions, "JSC_forceDiskCache=false")
end

def runBigIntEnabled(*optionalTestSpecificOptions)
    run("big-int-enabled", "--useBigInt=true" , *(FTL_OPTIONS + optionalTestSpecificOptions))
end

def runBigIntEnabledNoJIT(*optionalTestSpecificOptions)
    run("big-int-enabled-nojit", "--useBigInt=true", "--useJIT=false", *optionalTestSpecificOptions)
end

def runBigIntEnabledBaseline(*optionalTestSpecificOptions)
    run("big-int-enabled-baseline", "--useBigInt=true", "--useDFGJIT=0", *optionalTestSpecificOptions)
end

def runFTLNoCJIT(*optionalTestSpecificOptions)
    run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLNoCJITB3O0(*optionalTestSpecificOptions)
    run("ftl-no-cjit-b3o0", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O0_OPTIONS + FORCE_LLINT_EXIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLNoCJITValidate(*optionalTestSpecificOptions)
    run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
    run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
    run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", "--useB3HoistLoopInvariantValues=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
    run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runDFGEager(*optionalTestSpecificOptions)
    run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + FORCE_LLINT_EXIT_OPTIONS + optionalTestSpecificOptions))
end

def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
    run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
end

def runFTLEager(*optionalTestSpecificOptions)
    run("ftl-eager", "--airForceBriggsAllocator=true", "--useRandomizingExecutableIslandAllocation=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
end

def runFTLEagerWatchdog(*optionalTestSpecificOptions)
    timeout = rand(100)
    run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
end

def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
    run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + FORCE_LLINT_EXIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
    run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
end

def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
    run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
end

def runNoCJITNoASO(*optionalTestSpecificOptions)
    run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
    run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
    run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
    run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runNoCJIT(*optionalTestSpecificOptions)
    run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runEagerJettisonNoCJIT(*optionalTestSpecificOptions)
    run("eager-jettison-no-cjit", "--useRandomizingExecutableIslandAllocation=true", "--forceCodeBlockToJettisonDueToOldAge=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
end

def runShadowChicken(*optionalTestSpecificOptions)
    run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
end

def runMiniMode(*optionalTestSpecificOptions)
    run("mini-mode", "--forceMiniVMMode=true", *optionalTestSpecificOptions)
end

def runLogicalAssignmentOperatorsEnabled(*optionalTestSpecificOptions)
    run("logical-assignment-operators-enabled", "--useLogicalAssignmentOperators=true" , *(FTL_OPTIONS + optionalTestSpecificOptions))
end

def defaultRun
    if $mode == "quick"
        defaultQuickRun
    else
        runDefault
        runBytecodeCache
        runMiniMode
        if $jitTests
            runNoLLInt
            runNoCJITValidatePhases
            runNoCJITCollectContinuously if shouldCollectContinuously?
            runDFGEager
            if $mode != "basic"
                runDFGEagerNoCJITValidate
                runEagerJettisonNoCJIT
            end

            return if !$isFTLPlatform

            runNoFTL
            runFTLEager
            runFTLEagerNoCJITValidate
            runFTLNoCJITSmallPool

            return if $mode == "basic"

            runFTLNoCJITValidate
            runFTLNoCJITB3O0
            runFTLNoCJITNoPutStackValidate
            runFTLNoCJITNoInlineValidate
            runFTLEagerNoCJITB3O1
        end
    end
end

def defaultNoNoLLIntRun
    if $mode == "quick"
        defaultQuickRun
    else
        runDefault
        if $jitTests
            runNoCJITValidatePhases
            runNoCJITCollectContinuously if shouldCollectContinuously?
            runDFGEager
            if $mode != "basic"
                runDFGEagerNoCJITValidate
                runEagerJettisonNoCJIT
            end

            return if !$isFTLPlatform

            runNoFTL
            runFTLNoCJITValidate
            runFTLNoCJITSmallPool

            return if $mode == "basic"

            runFTLNoCJITB3O0
            runFTLNoCJITNoPutStackValidate
            runFTLNoCJITNoInlineValidate
            runFTLEager
            runFTLEagerNoCJITValidate
        end
    end
end

def defaultQuickRun
    runDefault
    if $jitTests
        runNoCJITValidate

        return if !$isFTLPlatform

        runNoFTL
        runFTLNoCJITValidate
    end
end

def defaultSpotCheckNoMaximalFlush
    defaultQuickRun
    runNoCJITNoAccessInlining

    return if !$isFTLPlatform

    runFTLNoCJITOSRValidation
    runFTLNoCJITNoAccessInlining
    runFTLNoCJITB3O0
end

def defaultSpotCheck
    defaultSpotCheckNoMaximalFlush
    runEagerJettisonNoCJIT
end

# This is expected to not do eager runs because eager runs can have a lot of recompilations
# for reasons that don't arise in the real world. It's used for tests that assert convergence
# by counting recompilations.
def defaultNoEagerRun(*optionalTestSpecificOptions)
    runDefault(*optionalTestSpecificOptions)
    if $jitTests
        runNoLLInt(*optionalTestSpecificOptions)
        runNoCJITValidatePhases(*optionalTestSpecificOptions)
        runNoCJITCollectContinuously(*optionalTestSpecificOptions) if shouldCollectContinuously?

        return if !$isFTLPlatform

        runNoFTL(*optionalTestSpecificOptions)
        runFTLNoCJITValidate(*optionalTestSpecificOptions)

        return if $mode == "basic"

        runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
        runFTLNoCJITB3O0(*optionalTestSpecificOptions)
    end
end

def defaultNoSamplingProfilerRun
    runDefault
    if $jitTests
        runNoLLInt
        runNoCJITValidatePhases
        runNoCJITCollectContinuously if shouldCollectContinuously?
        runDFGEager
        runDFGEagerNoCJITValidate
        runEagerJettisonNoCJIT

        return if !$isFTLPlatform

        runNoFTL
        runFTLNoCJITNoPutStackValidate
        runFTLNoCJITNoInlineValidate
        runFTLEager
        runFTLEagerNoCJITValidate
        runFTLNoCJITSmallPool
    end
end

def runProfiler
    if $remote or $memoryLimited or ($hostOS == "windows") or ($hostOS == "playstation")
        skip
        return
    end

    profilerOutput = uniqueFilename(".json")
    if $canRunDisplayProfilerOutput
        addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
    else
        puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
        run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
    end
end

def runExceptionFuzz
    subCommand = escapeAll(vmCommand + ["--useDollarVM=true", "--useExceptionFuzz=true", $benchmark.to_s])
    addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
end

def runExecutableAllocationFuzz(name, *options)
    subCommand = escapeAll(vmCommand + ["--useDollarVM=true", $benchmark.to_s] + options)
    addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
end

def runTypeProfiler
    if !$jitTests
        return
    end

    run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
    run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))

    return if !$isFTLPlatform

    run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
end

def runControlFlowProfiler
    if !$jitTests
        return
    end

    return if !$isFTLPlatform

    run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end

def runTest262(mode, exception, includeFiles, flags)
    failsWithException = exception != "NoException"
    isStrict = false
    isModule = false
    isAsync = false

    flags.each {
        | flag |
        case flag
        when :strict
            isStrict = true
        when :module
            isModule = true
        when :async
            isAsync = true
        else
            raise "Invalid flag for runTest262, #{flag}"
        end
    }

    prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)

    args = vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions
    args << "--exception=" + exception if failsWithException
    args << "--test262-async" if isAsync
    args += includeFiles

    case mode
    when :normal
        errorHandler = simpleErrorHandler
        outputHandler = silentOutputHandler
    when :fail
        errorHandler = expectedFailErrorHandler
        outputHandler = noisyOutputHandler
    when :failDueToOutdatedOrBadTest
        errorHandler = expectedFailErrorHandler
        outputHandler = noisyOutputHandler
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end

    if isStrict
        kind = "default-strict"
        args << "--strict-file=#{$benchmark}"
    else
        kind = "default"
        if isModule
            args << "--module-file=#{$benchmark}"
        else
            args << $benchmark.to_s
        end
    end

    addRunCommand(kind, args, outputHandler, errorHandler)
end

def prepareTest262Fixture
    # This function is used to add the files used by Test262 modules tests.
    prepareExtraRelativeFiles([""], $collection)
end

def runES6(mode)
    args = vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions + [$benchmark.to_s]
    case mode
    when :normal
        errorHandler = simpleErrorHandler
    when :fail
        errorHandler = expectedFailErrorHandler
    when :failDueToOutdatedOrBadTest
        errorHandler = expectedFailErrorHandler
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end
    addRunCommand("default", args, noisyOutputHandler, errorHandler)
end

def defaultRunModules(noLLInt: true)
    run("default-modules", "-m")

    if !$jitTests
        return
    end

    run("no-llint-modules", "-m", "--useLLInt=false") if noLLInt
    run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
    run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
    run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))

    return if !$isFTLPlatform

    run("default-ftl-modules", "-m", *FTL_OPTIONS)
    run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
    run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
    run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
    run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
    run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end

def noNoLLIntRunModules
    defaultRunModules(noLLInt: false)
end

def runWebAssembly
    return if !$jitTests
    return if !$isFTLPlatform
    run("default-wasm", "-m", *FTL_OPTIONS)
    if $mode != "quick"
        run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
        run("wasm-eager", "-m", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + EAGER_OPTIONS))
        run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", *FTL_OPTIONS)
        run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
        run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
        run("wasm-b3", "-m", "--useWasmLLInt=false", "--wasmBBQUsesAir=false", *FTL_OPTIONS)
        run("wasm-air", "-m", "--useWasmLLInt=false", "--useRandomizingExecutableIslandAllocation=true", *FTL_OPTIONS)
        run("wasm-collect-continuously", "-m", "--collectContinuously=true", *FTL_OPTIONS) if shouldCollectContinuously?
    end
end

def runWebAssemblyJetStream2
    return if !$jitTests
    return if !$isFTLPlatform

    if $memoryLimited
        skip
        return
    end

    prepareExtraAbsoluteFiles(JETSTREAM2_PATH, ["JetStreamDriver.js"])
    prepareExtraRelativeFilesWithBaseDirectory(Dir[JETSTREAM2_PATH + "wasm" + "*.js"].map { |f| "wasm/" + File.basename(f) }, $collection.dirname, $extraFilesBaseDir.dirname)
    prepareExtraRelativeFilesWithBaseDirectory(Dir[JETSTREAM2_PATH + "wasm" + "*.wasm"].map { |f| "wasm/" + File.basename(f) }, $collection.dirname, $extraFilesBaseDir.dirname)

    run("default-wasm", *FTL_OPTIONS)

    if $mode != "quick"
        run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
        run("wasm-eager", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + EAGER_OPTIONS))
        run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
        run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
        run("wasm-slow-memory", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
        run("wasm-b3", "--useWasmLLInt=false", "--wasmBBQUsesAir=false", *FTL_OPTIONS)
        run("wasm-air", "--useWasmLLInt=false", "--useRandomizingExecutableIslandAllocation=true", *FTL_OPTIONS)
        run("wasm-collect-continuously", "--collectContinuously=true", *FTL_OPTIONS) if shouldCollectContinuously?
    end
end

def runWebAssemblySuite(*optionalTestSpecificOptions)
    return if !$jitTests
    return if !$isFTLPlatform
    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
    prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
    prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
    run("default-wasm", "-m", *(FTL_OPTIONS + optionalTestSpecificOptions))
    if $mode != "quick"
        run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
        run("wasm-eager", "-m", *(FTL_OPTIONS + EAGER_OPTIONS + optionalTestSpecificOptions))
        run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-b3", "-m", "--useWasmLLInt=false", "--wasmBBQUsesAir=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-air", "-m", "--useWasmLLInt=false", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-collect-continuously", "-m", "--collectContinuously=true", *(FTL_OPTIONS + optionalTestSpecificOptions)) if shouldCollectContinuously?
    end
end

def runHarnessTest(kind, *options)
    wasmFiles = allWasmFiles($collection)
    wasmFiles.each {
        | file |
        basename = file.basename.to_s
        addRunCommand("(" + basename + ")-" + kind, [pathToVM.to_s] + $testSpecificRequiredOptions + options + [$benchmark.to_s, "--", basename], silentOutputHandler, simpleErrorHandler)
    }
end

def runWebAssemblyWithHarness(*optionalTestSpecificOptions)
    raise unless $benchmark.to_s =~ /harness\.m?js/
    return if !$jitTests
    return if !$isFTLPlatform

    wasmFiles = allWasmFiles($collection)
    prepareExtraRelativeFiles(wasmFiles.map { |f| f.basename }, $collection)

    runHarnessTest("default-wasm", *(FTL_OPTIONS + optionalTestSpecificOptions))
    if $mode != "quick"
        runHarnessTest("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
        runHarnessTest("wasm-eager", *(FTL_OPTIONS + EAGER_OPTIONS + optionalTestSpecificOptions))
        runHarnessTest("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runHarnessTest("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runHarnessTest("wasm-slow-memory", "--useWebAssemblyFastMemory=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runHarnessTest("wasm-b3", "--useWasmLLInt=false", "--wasmBBQUsesAir=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runHarnessTest("wasm-no-air", "--useWasmLLInt=false", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runHarnessTest("wasm-collect-continuously", "--collectContinuously=true", *(FTL_OPTIONS + optionalTestSpecificOptions)) if shouldCollectContinuously?
    end
end

def runWebAssemblyEmscripten(mode)
    case mode
    when :skip
        return
    end
    return if !$jitTests
    return if !$isFTLPlatform
    wasm = $benchmark.to_s.sub! '.js', '.wasm'
    prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
    run("default-wasm", *FTL_OPTIONS)
    if $mode != "quick"
        run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
        run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", *FTL_OPTIONS)
        run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
        run("wasm-b3", "--useWasmLLInt=false", "--wasmBBQUsesAir=false", *FTL_OPTIONS)
        run("wasm-air", "--useWasmLLInt=false", "--useRandomizingExecutableIslandAllocation=true", *FTL_OPTIONS)
        run("wasm-collect-continuously", "--collectContinuously=true", *FTL_OPTIONS) if shouldCollectContinuously?
    end
end

def runWebAssemblySpecTest(mode)
    case mode
    when :skip
        return
    end
    return if !$jitTests
    return if !$isFTLPlatform
    prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])

    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
    prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)

    harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
    prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)

    runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
    if $mode != "quick"
      runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
      runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", *FTL_OPTIONS)
      runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
      runWithOutputHandler("wasm-b3", noisyOutputHandler, "../spec-harness.js", "--useWasmLLInt=false", "--wasmBBQUsesAir=false", *FTL_OPTIONS)
      runWithOutputHandler("wasm-air", noisyOutputHandler, "../spec-harness.js", "--useWasmLLInt=false", "--useRandomizingExecutableIslandAllocation=true", *FTL_OPTIONS)
      runWithOutputHandler("wasm-collect-continuously", noisyOutputHandler, "../spec-harness.js", "--collectContinuously=true", *FTL_OPTIONS) if shouldCollectContinuously?
    end
end

def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
    return if !$jitTests
    return if !$isFTLPlatform
    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
    prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
    prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
    # Only let WebAssembly get executable memory.
    run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
end

def runChakra(mode, exception, baselineFile, extraFiles)
    raise unless $benchmark.to_s =~ /\.js$/
    failsWithException = exception != "NoException"
    testName = $~.pre_match

    prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
    prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)

    args = vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions
    args += FTL_OPTIONS if $isFTLPlatform
    args += EAGER_OPTIONS
    args << "--exception=" + exception if failsWithException
    args << "--dumpException" if failsWithException
    args += ["jsc-lib.js"]

    case mode
    when :baseline
        prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
        errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
        outputHandler = noisyOutputHandler
    when :pass
        errorHandler = chakraPassFailErrorHandler
        outputHandler = noisyOutputHandler
    when :skipDueToOutdatedOrBadTest
        return
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end

    kind = "default"
    args << $benchmark.to_s

    addRunCommand(kind, args, outputHandler, errorHandler)
end

def runLayoutTest(kind, *options)
    raise unless $benchmark.to_s =~ /\.js$/
    testName = $~.pre_match
    if kind
        kind = "layout-" + kind
    else
        kind = "layout"
    end

    prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
    prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])

    args = vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions + options +
        [(Pathname.new("resources") + "standalone-pre.js").to_s,
         $benchmark.to_s,
         (Pathname.new("resources") + "standalone-post.js").to_s]
    addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
end

def runLayoutTestNoFTL
    runLayoutTest("no-ftl")
end

def runLayoutTestNoLLInt
    runLayoutTest("no-llint", "--useLLInt=false")
end

def runLayoutTestNoCJIT
    runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
end

def runLayoutTestDFGEagerNoCJIT
    runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def runLayoutTestDefault
    runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
end

def runLayoutTestFTLNoCJIT
    runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end

def runLayoutTestFTLEagerNoCJIT
    runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def runLayoutTestFTLEagerNoCJITB3O1
    runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
end

def noFTLRunLayoutTest
    if !$jitTests
        return
    end

    runLayoutTestNoLLInt
    runLayoutTestNoCJIT
    runLayoutTestDFGEagerNoCJIT
end

def defaultQuickRunLayoutTest
    runLayoutTestDefault
    if $jitTests
        if $isFTLPlatform
            runLayoutTestNoFTL
            runLayoutTestFTLNoCJIT
            runLayoutTestFTLEagerNoCJIT
        else
            noFTLRunLayoutTest
        end
    end
end

def defaultRunLayoutTest
    if $mode == "quick"
        defaultQuickRunLayoutTest
    else
        runLayoutTestDefault
        if $jitTests
            noFTLRunLayoutTest

            return if !$isFTLPlatform

            runLayoutTestNoFTL
            runLayoutTestFTLNoCJIT
            runLayoutTestFTLEagerNoCJIT
        end
    end
end

def noEagerNoNoLLIntTestsRunLayoutTest
    runLayoutTestDefault
    if $jitTests
        runLayoutTestNoCJIT

        return if !$isFTLPlatform

        runLayoutTestNoFTL
        runLayoutTestFTLNoCJIT
    end
end

def noNoLLIntRunLayoutTest
    runLayoutTestDefault
    if $jitTests
        runLayoutTestNoCJIT
        runLayoutTestDFGEagerNoCJIT

        return if !$isFTLPlatform

        runLayoutTestNoFTL
        runLayoutTestFTLNoCJIT
        runLayoutTestFTLEagerNoCJIT
    end
end

def prepareExtraRelativeFilesWithBaseDirectory(extraFiles, destination, baseDirectory)
    Dir.chdir($outputDir) {
        extraFiles.each {
            | file |
            dest = destination + file
            FileUtils.mkdir_p(dest.dirname)
            FileUtils.cp baseDirectory + file, dest
        }
    }
end

def prepareExtraRelativeFiles(extraFiles, destination)
    prepareExtraRelativeFilesWithBaseDirectory(extraFiles, destination, $extraFilesBaseDir)
end

def baseDirForCollection(collectionName)
    Pathname(".tests") + collectionName
end

def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
    raise unless absoluteBase.absolute?
    Dir.chdir($outputDir) {
        collectionBaseDir = baseDirForCollection($collectionName)
        extraFiles.each {
            | file |
            destination = collectionBaseDir + file
            FileUtils.mkdir_p destination.dirname unless destination.directory?
            FileUtils.cp absoluteBase + file, destination
        }
    }
end

def runComplexTest(before, after, additionalEnv, *options)
    prepareExtraRelativeFiles(before.map{|v| (Pathname("..") + v).to_s}, $collection)
    prepareExtraRelativeFiles(after.map{|v| (Pathname("..") + v).to_s}, $collection)
    args = vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions + options + before.map{|v| v.to_s} + [$benchmark.to_s] + after.map{|v| v.to_s}
    addRunCommand("complex", args, noisyOutputHandler, simpleErrorHandler, *additionalEnv)
end

def runMozillaTest(kind, mode, extraFiles, *options)
    if kind
        kind = "mozilla-" + kind
    else
        kind = "mozilla"
    end
    prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
    args = vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
    case mode
    when :normal
        errorHandler = mozillaErrorHandler
    when :negative
        errorHandler = mozillaExit3ErrorHandler
    when :fail
        errorHandler = mozillaFailErrorHandler
    when :failDueToOutdatedOrBadTest
        errorHandler = mozillaFailErrorHandler
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end
    addRunCommand(kind, args, noisyOutputHandler, errorHandler)
end

def runMozillaTestDefault(mode, *extraFiles)
    runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
end

def runMozillaTestNoFTL(mode, *extraFiles)
    runMozillaTest("no-ftl", mode, extraFiles)
end

def runMozillaTestLLInt(mode, *extraFiles)
    runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
end

def runMozillaTestBaselineJIT(mode, *extraFiles)
    runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
end

def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
    runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
    runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def defaultQuickRunMozillaTest(mode, *extraFiles)
    if $jitTests
        runMozillaTestDefault(mode, *extraFiles)
        runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
    else
        runMozillaTestNoFTL(mode, *extraFiles)
        if $jitTests
            runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
        end
    end
end

def defaultRunMozillaTest(mode, *extraFiles)
    if $mode == "quick"
        defaultQuickRunMozillaTest(mode, *extraFiles)
    else
        runMozillaTestNoFTL(mode, *extraFiles)
        if $jitTests
            runMozillaTestLLInt(mode, *extraFiles)
            runMozillaTestBaselineJIT(mode, *extraFiles)
            runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
            runMozillaTestDefault(mode, *extraFiles)
            runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
        end
    end
end

def runNoisyTestImpl(kind, options, additionalEnv)
    addRunCommand(kind, vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
end

def runNoisyTest(kind, *options)
    runNoisyTestImpl(kind, options, [])
end

def runNoisyTestWithEnv(kind, *additionalEnv)
    runNoisyTestImpl(kind, [], additionalEnv)
end

def runNoisyTestDefault
    runNoisyTest("default", *FTL_OPTIONS)
end

def runNoisyTestNoFTL
    runNoisyTest("no-ftl")
end

def runNoisyTestNoCJIT
    runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
end

def runNoisyTestNoCJITB3O1
    runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
end

def runNoisyTestEagerNoCJIT
    runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
end

def defaultRunNoisyTest
    runNoisyTestDefault
    if $jitTests and $isFTLPlatform
        runNoisyTestNoFTL
        runNoisyTestNoCJIT
        runNoisyTestNoCJITB3O1
        runNoisyTestEagerNoCJIT
    end
end

def skip
    $didAddRunCommand = true
    $skipped = true
    puts "Skipping #{$collectionName}/#{$benchmark}"
end

def allWasmFiles(path)
    if path.file?
        [path]
    else
        result = []
        Dir.foreach(path) {
            | filename |
            next unless filename =~ /\.m?wasm$/
            next unless (path + filename).file?
            result << path + filename
        }
        result
    end
end

def allJSFiles(path)
    if path.file?
        [path]
    else
        result = []
        Dir.foreach(path) {
            | filename |
            next unless filename =~ /\.m?js$/
            next unless (path + filename).file?
            result << path + filename
        }
        result
    end
end

def uniqueifyName(names, name)
    result = name.to_s
    toAdd = 1
    while names[result]
        result = "#{name}-#{toAdd}"
        toAdd += 1
    end
    names[result] = true
    result
end

def simplifyCollectionName(collectionPath)
    outerDir = collectionPath.dirname
    name = collectionPath.basename
    lastName = name
    if collectionPath.directory?
        while lastName.to_s =~ /test/
            lastName = outerDir.basename
            name = lastName + name
            outerDir = outerDir.dirname
        end
    end
    uniqueifyName($collectionNames, name)
end

def prepareCollection(name)
    FileUtils.mkdir_p $outputDir + name

    absoluteCollection = $collection.realpath

    Dir.chdir($outputDir) {
        bundleDir = baseDirForCollection(name)

        # Create the proper directory structures.
        FileUtils.mkdir_p bundleDir
        if bundleDir.basename == $collection.basename
            FileUtils.cp_r absoluteCollection, bundleDir.dirname
            $collection = bundleDir
        else
            FileUtils.cp_r absoluteCollection, bundleDir
            $collection = bundleDir + $collection.basename
        end

        $extraFilesBaseDir = absoluteCollection
    }
end

$collectionNames = {}

def handleCollectionFile(collection)
    collectionName = simplifyCollectionName(collection)
   
    paths = {}
    subCollections = []
    YAML::load(IO::read(collection)).each {
        | entry |
        if entry["collection"]
            subCollections << entry["collection"]
            next
        end
        
        if Pathname.new(entry["path"]).absolute?
            raise "Absolute path: " + entry["path"] + " in #{collection}"
        end
        
        if paths[entry["path"]]
            raise "Duplicate path: " + entry["path"] + " in #{collection}"
        end
        
        subCollection = collection.dirname + entry["path"]
        
        if subCollection.file?
            subCollectionName = Pathname.new(entry["path"]).dirname
        else
            subCollectionName = entry["path"]
        end
        
        $collection = subCollection
        $collectionName = Pathname.new(collectionName)
        Pathname.new(subCollectionName).each_filename {
            | filename |
            next if filename =~ /^\./
            $collectionName += filename
        }
        $collectionName = $collectionName.to_s
        
        prepareCollection($collectionName)
      
        Dir.chdir($outputDir) {
            pathsToSearch = [$collection]
            if entry["tests"]
                if entry["tests"].is_a? Array
                    pathsToSearch = entry["tests"].map {
                        | testName |
                        pathsToSearch[0] + testName
                    }
                else
                    pathsToSearch[0] += entry["tests"]
                end
            end
            pathsToSearch.each {
                | pathToSearch |
                allJSFiles(pathToSearch).each {
                    | path |
                    
                    $benchmark = path.basename
                    $benchmarkDirectory = path.dirname
                    
                    $runCommandOptions = {}
                    $testSpecificRequiredOptions = []
                    eval entry["cmd"]
                }
            }
        }
    }
    
    subCollections.each {
        | subCollection |
        handleCollection(collection.dirname + subCollection)
    }
end

def handleCollectionDirectory(collection)
    collectionName = simplifyCollectionName(collection)
    
    $collection = collection
    $collectionName = collectionName
    prepareCollection(collectionName)
   
    Dir.chdir($outputDir) {
        $benchmarkDirectory = $collection
        allJSFiles($collection).each {
            | path |
            
            $benchmark = path.basename
            
            $runCommandOptions = {}
            $testSpecificRequiredOptions = []
            defaultRun unless parseRunCommands
        }
    }
end

def handleCollection(collection)
    collection = Pathname.new(collection)
    
    if collection.file?
        handleCollectionFile(collection)
    else
        handleCollectionDirectory(collection)
    end
end

def appendFailure(plan)
    File.open($outputDir + "failed", "a") {
        | outp |
        outp.puts plan.name
    }
    $numFailures += 1
end

def appendPass(plan)
    File.open($outputDir + "passed", "a") {
        | outp |
        outp.puts plan.name
    }
    $numPasses += 1
end

def appendResult(plan, didPass)
    File.open($outputDir + "results", "a") {
        | outp |
        outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
    }
end

def prepareBundle
    raise if $bundle

    if $doNotMessWithVMPath
        if !$remote and !$tarball
            $testingFrameworkPath = (frameworkFromJSCPath($jscPath) || $jscPath.dirname).realpath
            $jscPath = Pathname.new($jscPath).realpath
        else
            $testingFrameworkPath = frameworkFromJSCPath($jscPath)
        end
    else
        originalJSCPath = $jscPath
        vmDir = $outputDir + ".vm"
        FileUtils.mkdir_p vmDir
        
        frameworkPath = frameworkFromJSCPath($jscPath)
        destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
        $jscPath = destinationFrameworkPath + "Helpers" + "jsc"
        $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath

        if frameworkPath
            source = frameworkPath
            destination = Pathname.new(".vm")
        elsif $hostOS == "windows"
            # Make sure to copy dll along with jsc on Windows
            originalJSCDir = File.dirname(originalJSCPath)
            source = [originalJSCPath] + [originalJSCDir + "/jscLib.dll"]

            # Check for and copy JavaScriptCore.dll and WTF.dll for dynamic builds
            javaScriptCoreDLLPath = File.join(originalJSCDir, "JavaScriptCore.dll")
            wtfDLLPath = File.join(originalJSCDir, "WTF.dll")
            if (File.exists?(javaScriptCoreDLLPath))
                source = source + [javaScriptCoreDLLPath]
            end
            if (File.exists?(wtfDLLPath))
                source = source + [wtfDLLPath]
            end

            destination = $jscPath.dirname

            Dir.chdir($outputDir) {
                FileUtils.mkdir_p destination
            }
        else
            source = originalJSCPath
            destination = $jscPath

            Dir.chdir($outputDir) {
                FileUtils.mkdir_p $jscPath.dirname
            }
        end

        Dir.chdir($outputDir) {
            if $copyVM
                FileUtils.cp_r source, destination
            else
                begin 
                    FileUtils.ln_s source, destination
                rescue Exception
                    $stderr.puts "Warning: unable to create soft link, trying to copy."
                    FileUtils.cp_r source, destination
                end
            end

            if $remote and $hostOS == "linux"
                begin
                    dependencies = `ldd #{source}`
                    dependencies.split(/\n/).each {
                        | dependency |
                        FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
                    }
                rescue
                    $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
                end
            end
        }
    end
    
    Dir.chdir($outputDir) {
        FileUtils.cp_r HELPERS_PATH, ".helpers"
    }

    ARGV.each {
        | collection |
        handleCollection(collection)
    }

    puts
end

def cleanOldResults
    raise unless $bundle

    eachResultFile($outputDir) {
        | path |
        FileUtils.rm_f path
    }
end

def cleanEmptyResultFiles
    eachResultFile($outputDir) {
        | path |
        next unless path.basename.to_s =~ /\.out$/
        next unless FileTest.size(path) == 0
        FileUtils.rm_f path
    }
end

def eachResultFile(startingDir, &block)
    dirsToClean = [startingDir]
    until dirsToClean.empty?
        nextDir = dirsToClean.pop
        Dir.foreach(nextDir) {
            | entry |
            next if entry =~ /^\./
            path = nextDir + entry
            if path.directory?
                dirsToClean.push(path)
            else
                block.call(path)
            end
        }
    end
end

def prepareTestRunner(remoteIndex=0)
    raise if $bundle

    $runlist.each_with_index {
        | plan, index |
        plan.index = index
    }

    Dir.mkdir($runnerDir) unless $runnerDir.directory?
    toDelete = []
    Dir.foreach($runnerDir) {
        | filename |
        if filename =~ /^test_/
            toDelete << filename
        end
    }
    
    toDelete.each {
        | filename |
        File.unlink($runnerDir + filename)
    }

    $runlist.each {
        | plan |
        plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
    }

    case $testRunnerType
    when :make
        prepareMakeTestRunner(remoteIndex)
    when :shell
        prepareShellTestRunner
    when :ruby
        prepareRubyTestRunner
    else
        raise "Unknown test runner type: #{$testRunnerType.to_s}"
    end
end

def cleanRunnerDirectory
    raise unless $bundle
    Dir.foreach($runnerDir) {
        | filename |
        next unless filename =~ /^test_fail/
        FileUtils.rm_f $runnerDir + filename
    }
end

def sshRead(cmd, remoteIndex=0)
    raise unless $remote

    remoteHost = $remoteHosts[remoteIndex]

    result = ""
    IO.popen("ssh -o NoHostAuthenticationForLocalhost=yes -p #{remoteHost.port} #{remoteHost.user}@#{remoteHost.host} '#{cmd}'", "r") {
      | inp |
      inp.each_line {
        | line |
        result += line
      }
    }
    raise "#{$?}" unless $?.success?
    result
end

def runCommandOnTester(cmd)
    if $remote
        result = sshRead(cmd)
    else
        result = `#{cmd}`
    end
end

def numberOfProcessors
    if $hostOS == "windows"
        numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
    else
        begin
            numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
        rescue
            numProcessors = 0
        end

        if numProcessors == 0
            begin
                numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
            rescue
                numProcessors == 0
            end
        end
    end

    if numProcessors == 0
        numProcessors = 1
    end
    return numProcessors
end

$runnerDirMutex = Mutex.new
def runAndMonitorTestRunnerCommand(*cmd)
    numberOfTests = 0
    $runnerDirMutex.synchronize {
        Dir.chdir($runnerDir) {
            # -1 for the runscript, and -2 for '..' and '.'
            numberOfTests = Dir.entries(".").count - 3
        }
    }
    unless $progressMeter
        mysys(cmd.join(' '))
    else
       running = {}
       didRun = {}
       didFail = {}
       blankLine = true
       prevStringLength = 0
       IO.popen(cmd.join(' '), mode="r") {
           | inp |
           inp.each_line {
               | line |
               line = line.scrub.chomp
               if line =~ /^Running /
                   running[$~.post_match] = true
               elsif line =~ /^PASS: /
                   didRun[$~.post_match] = true
               elsif line =~ /^FAIL: /
                   didRun[$~.post_match] = true
                   didFail[$~.post_match] = true
               else
                   unless blankLine
                       print("\r" + " " * prevStringLength + "\r")
                   end
                   puts line
                   blankLine = true
               end

               def lpad(str, chars)
                   str = str.to_s
                   if str.length > chars
                       str
                   else
                      "%#{chars}s"%(str)
                   end
               end

               string  = ""
               string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
               unless didFail.empty?
                   string += " (failed #{didFail.size})"
               end
               string += " "
               (running.size - didRun.size).times {
                   string += "."
               }
               if string.length < prevStringLength
                   print string
                   print(" " * (prevStringLength - string.length))
               end
               print string
               prevStringLength = string.length
               blankLine = false
               $stdout.flush
           }
       }
       puts
       raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
    end
end

def runTestRunner(remoteIndex=0)
    if $remote
        remoteHost = $remoteHosts[remoteIndex]
        if !remoteHost.remoteDirectory
            remoteHost.remoteDirectory = JSON::parse(sshRead("cat ~/.bencher", remoteIndex))["tempPath"]
        end
        mysys("ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", "mkdir -p #{remoteHost.remoteDirectory}")
        mysys("scp", "-o", "NoHostAuthenticationForLocalhost=yes", "-P", remoteHost.port.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{remoteHost.user}@#{remoteHost.host}:#{remoteHost.remoteDirectory}")
        remoteScript = "\""
        remoteScript += "cd #{remoteHost.remoteDirectory} && "
        remoteScript += "rm -rf #{$outputDir.basename} && "
        remoteScript += "tar xzf #{$tarFileName} && "
        remoteScript += "cd #{$outputDir.basename}/.runner && "
        remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
        remoteScript += "export LD_LIBRARY_PATH=#{remoteHost.remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
        remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
        remoteScript += "export JSCTEST_memoryLimit=#{Shellwords.shellescape(ENV['JSCTEST_memoryLimit'])} && "
        remoteScript += "export TZ=#{Shellwords.shellescape(ENV['TZ'])} && "
        $envVars.each { |var| remoteScript += "export " << var << "\n" }
        remoteScript += "#{testRunnerCommand(remoteIndex)}\""
        runAndMonitorTestRunnerCommand("ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", remoteScript)
    else
        Dir.chdir($runnerDir) {
            runAndMonitorTestRunnerCommand(testRunnerCommand)
        }
    end
end

def detectFailures
    raise if $bundle
    failures = []
    if $remote
        $remoteHosts.each_with_index {
            | host, remoteIndex |
            output = sshRead("cd #{host.remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"", remoteIndex)
            output.split(/\n/).each {
                | line |
                next unless line =~ /test_fail_/
                failures << $~.post_match.to_i
            }
        }
    else
        Dir.foreach($runnerDir) {
            | filename |
            next unless filename =~ /test_fail_/
            failures << $~.post_match.to_i
        }
    end

    failureSet = {}

    failures.each {
        | failure | 
        appendFailure($runlist[failure])
        failureSet[failure] = true
    }

    familyMap = {}
    $runlist.each_with_index {
        | plan, index |
        unless familyMap[plan.family]
            familyMap[plan.family] = []
        end
        if failureSet[index]
            appendResult(plan, false)
            familyMap[plan.family] << {:result => "FAIL", :plan => plan};
            next
        else
            appendResult(plan, true)
            familyMap[plan.family] << {:result => "PASS", :plan => plan};
        end
        appendPass(plan)
    }

    File.open($outputDir + "resultsByFamily", "w") {
        | outp |
        first = true
        familyMap.keys.sort.each {
            | familyName |
            if first
                first = false
            else
                outp.puts
            end
            
            outp.print "#{familyName}:"

            numPassed = 0
            familyMap[familyName].each {
                | entry |
                if entry[:result] == "PASS"
                    numPassed += 1
                end
            }

            if numPassed == familyMap[familyName].size
                outp.puts " PASSED"
            elsif numPassed == 0
                outp.puts " FAILED"
            else
                outp.puts
                familyMap[familyName].each {
                    | entry |
                    outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
                }
            end
        }
    }
end

def compressBundle
    cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
    $stderr.puts ">> #{cmd}" if $verbosity >= 2
    raise unless system(cmd)
end

def clean(file)
    FileUtils.rm_rf file unless $bundle
end

clean($outputDir + "failed")
clean($outputDir + "passed")
clean($outputDir + "results")
clean($outputDir + "resultsByFamily")
clean($outputDir + ".vm")
clean($outputDir + ".helpers")
clean($outputDir + ".runner")
clean($outputDir + ".tests")
clean($outputDir + "_payload")

Dir.mkdir($outputDir) unless $outputDir.directory?

$outputDir = $outputDir.realpath
$runnerDir = $outputDir + ".runner"

if !$numChildProcesses
    if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
        $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
    else
        $numChildProcesses = numberOfProcessors
    end
end

if ENV["JSCTEST_timeout"]
    # In the worst case, the processors just interfere with each other.
    # Increase the timeout proportionally to the number of processors.
    ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
end

if !ENV["JSCTEST_memoryLimit"] && $memoryLimited
    ENV["JSCTEST_memoryLimit"] = (600 * 1024 * 1024).to_s
end

# Some tests fail if the time zone is not set to US/Pacific
# https://webkit.org/b/136363
# Set as done in run-javascript-tests
ENV["TZ"] = "US/Pacific"; 

def runBundle
    raise unless $bundle

    cleanRunnerDirectory
    cleanOldResults
    runTestRunner
    cleanEmptyResultFiles
end

def runNormal
    raise if $bundle or $tarball

    prepareBundle
    prepareTestRunner
    runTestRunner
    cleanEmptyResultFiles
    detectFailures
end

def runTarball
    raise unless $tarball

    prepareBundle 
    prepareTestRunner
    compressBundle
end

def runRemote
    raise unless $remote

    prepareBundle
    $remoteHosts.each_index {
        | index |
        prepareTestRunner(index)
    }
    compressBundle
    threads = []
    $remoteHosts.each_index {
        | index |
        threads << Thread.new {
            runTestRunner(index)
        }
    }
    threads.each { |thr| thr.join }
    detectFailures
end

puts
if $bundle
    runBundle
elsif $remote
    runRemote
elsif $tarball
    runTarball
else
    runNormal
end