Tools/Scripts/run-jsc-stress-tests - WebKit - Git at Google

 #!/usr/bin/env ruby

 # Copyright (C) 2013 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #
 # 1.  Redistributions of source code must retain the above copyright
 #     notice, this list of conditions and the following disclaimer.
 # 2.  Redistributions in binary form must reproduce the above copyright
 #     notice, this list of conditions and the following disclaimer in the
 #     documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 require 'getoptlong'
 require 'pathname'
 require 'yaml'

 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"

 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"

 $haveShellwords = false

 begin
     require 'shellwords'
     $haveShellwords = true
 rescue Exception => e
     $stderr.puts "Warning: did not find shellwords; some features will be disabled."
     $stderr.puts "Error: #{e.inspect}"
 end

 $canRunDisplayProfilerOutput = false

 begin
     require 'json'
     require 'highline'
     $canRunDisplayProfilerOutput = true
 rescue Exception => e
     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
     $stderr.puts "Error: #{e.inspect}"
 end

 def printCommandArray(*cmd)
     begin
         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
     rescue
         commandArray = cmd.join(' ')
     end
     $stderr.puts ">> #{commandArray}"
 end

 def mysys(*cmd)
     printCommandArray(*cmd)
     raise "Command failed: #{$?.inspect}" unless system(*cmd)
 end

 numProcessors = `sysctl -n hw.availcpu`.to_i

 $jscPath = nil
 $enableFTL = false
 $collections = []
 $outputDir = Pathname.new("results")
 $parallel = ($haveShellwords and numProcessors > 1)
 $verbosity = 0

 def usage
     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
     puts
     puts "--jsc            (-j)   Path to JavaScriptCore. This option is required."
     puts "--ftl-jit               Indicate that we have the FTL JIT."
     puts "--[no-]parallel         Run in parallel, or not. Default is #{$parallel}."
     puts "--output-dir     (-o)   Path where to put results. Default is #{$outputDir}."
     puts "--verbose        (-v)   Print more things while running."
     puts "--help           (-h)   Print this message."
     exit 1
 end

 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
                ['--parallel', GetoptLong::NO_ARGUMENT],
                ['--no-parallel', GetoptLong::NO_ARGUMENT],
                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
     | opt, arg |
     case opt
     when '--help'
         usage
     when '--jsc'
         $jscPath = Pathname.new(arg).realpath
     when '--output-dir'
         $outputDir = Pathname.new(arg)
     when '--ftl-jit'
         $enableFTL = true
     when '--parallel'
         $parallel = true
     when '--no-parallel'
         $parallel = false
     when '--verbose'
         $verbosity += 1
     end
 }

 unless $jscPath
     $stderr.puts "Error: must specify -j <path>."
     exit 1
 end

 $numFailures = 0

 EAGER_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20"]

 $runlist = []

 class Plan
     attr_reader :directory, :arguments, :name
     attr_accessor :index

     def initialize(directory, arguments, name)
         @directory = directory.realpath
         @arguments = arguments
         @name = name
     end

     def writeTestScript(filename, failCommand)
         File.open(filename, "w") {
             | outp |
             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
             cmd = ("(cd #{Shellwords.shellescape(@directory.to_s)} && " +
                    @arguments.map{|v| Shellwords.shellescape(v)}.join(' ') +
                    ") || #{failCommand}")
             if $verbosity >= 1
                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
             end
             outp.puts cmd
         }
     end
 end

 $uniqueFilenameCounter = 0
 def uniqueFilename(extension)
     payloadDir = $outputDir + "_payload"
     Dir.mkdir payloadDir unless payloadDir.directory?
     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
     $uniqueFilenameCounter += 1
     result
 end

 def addRunCommand(kind, command)
     $runlist << Plan.new($benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}.#{kind}")
 end

 def run(kind, *options)
     addRunCommand(kind, [$jscPath.to_s] + options + [$benchmark.to_s])
 end

 def runDefault
     run("default")
 end

 def runNoCJIT
     run("no-cjit", "--enableConcurrentJIT=false")
 end

 def runDefaultFTL
     run("default-ftl", "--useExperimentalFTL=true")
 end

 def runFTLNoCJIT
     run("ftl-no-cjit", "--enableConcurrentJIT=false", "--useExperimentalFTL=true")
 end

 def runDFGEager
     run("dfg-eager", *EAGER_OPTIONS)
 end

 def runFTLEager
     run("ftl-eager", "--useExperimentalFTL=true", *EAGER_OPTIONS)
 end

 def runProfiler
     profilerOutput = uniqueFilename(".json")
     if $haveShellwords and $canRunDisplayProfilerOutput
         addRunCommand("profiler", ["ruby", (HELPERS_PATH + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, $jscPath.to_s, "-p", profilerOutput.to_s, $benchmark.to_s])
     else
         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
         run("profiler-simple", "-p", profilerOutput.to_s)
     end
 end

 def defaultRun
     runDefault
     runNoCJIT
     if $enableFTL
         runDefaultFTL
         runFTLNoCJIT
         runFTLEager
     else
         runDFGEager
     end
 end

 def skip
     puts "Skipping #{$collectionName}/#{$benchmark}"
 end

 Dir.mkdir($outputDir) unless $outputDir.directory?
 begin
     File.delete($outputDir + "failed")
 rescue
 end

 $outputDir = $outputDir.realpath

 def allJSFiles(path)
     if path.file?
         [path]
     else
         result = []
         Dir.foreach(path) {
             | filename |
             next unless filename =~ /\.js$/
             next unless (path + filename).file?
             result << path + filename
         }
         result
     end
 end

 # Returns [collectionPath, collectionName]
 def simplifyCollectionName(collectionNames, collectionPath)
     outerDir = collectionPath.dirname
     name = collectionPath.basename
     lastName = name
     if collectionPath.directory?
         while lastName.to_s =~ /test/
             lastName = outerDir.basename
             name = lastName + name
             outerDir = outerDir.dirname
         end
     end
     collectionName = name.to_s
     toAdd = 1
     while collectionNames[collectionName]
         collectionName = File.basename(name.to_s) + "-#{toAdd}"
         toAdd += 1
     end
     collectionNames[collectionName] = true
     [collectionPath, collectionName]
 end

 def prepareCollection(name)
     dir = $outputDir
     Pathname.new(name).each_filename {
         | filename |
         dir = dir + filename
         Dir.mkdir(dir) unless dir.directory?
     }
 end

 collectionNames = {}

 ARGV.each {
     | collection |
     collection, collectionName = simplifyCollectionName(collectionNames, Pathname.new(collection))

     if collection.file?
         subCollectionNames = {}
         YAML::load(IO::read(collection)).each {
             | entry |
             path = collection.dirname + entry["path"]

             subCollection, subCollectionName = simplifyCollectionName(subCollectionNames, path)

             $collection = subCollection
             $collectionName = (Pathname.new(collectionName) + subCollectionName).to_s

             prepareCollection($collectionName)

             allJSFiles(path).each {
                 | path |

                 path = path.realpath

                 $benchmark = path.basename
                 $benchmarkDirectory = path.dirname

                 eval entry["cmd"]
             }
         }
     else
         prepareCollection(collectionName)

         $collection = collection
         $collectionName = collectionName
         $benchmarkDirectory = $collection
         allJSFiles($collection).each {
             | path |

             $benchmark = path.basename

             didRun = false
             File.open($collection + $benchmark) {
                 | inp |
                 inp.each_line {
                     | line |
                     next unless line =~ /^\/\/@/
                     eval $~.post_match
                     didRun = true
                 }
             }

             defaultRun unless didRun
         }
     end
 }

 def appendFailure(plan)
     File.open($outputDir + "failed", "a") {
         | outp |
         outp.puts plan.name
     }
     filename = $outputDir + plan.name
     begin
         plan.writeTestScript(filename, "exit 1")
     rescue => e
         $stderr.puts "Warning: failed to create repro file at #{filename}: #{e.inspect}"
     end
     $numFailures += 1
 end

 if $enableFTL and ENV["JSC_timeout"]
     # Currently, using the FTL is a performance regression particularly in real
     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
 end

 if $parallel
     if ENV["JSC_timeout"]
         # In the worst case, the processors just interfere with each other.
         # Increase the timeout proportionally to the number of processors.
         ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt(numProcessors)).to_i.to_s
     end

     # The goals of our parallel test runner are scalability and simplicity. The
     # simplicity part is particularly important. We don't want to have to have
     # a full-time contributor just philosophising about parallel testing.
     #
     # As such, we just pass off all of the hard work to 'make'. This creates a
     # dummy directory ("$outputDir/.parallel") in which we create a dummy
     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
     # That is, for each test we know we will run, there is a rule in the
     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
     # Makefile results in 'make' doing all of the hard work:
     #
     # - Load balancing just works. Most systems have a great load balancer in
     #   'make'. If your system doesn't then just install a real 'make'.
     #
     # - Interruptions just work. For example Ctrl-C handling in 'make' is
     #   exactly right. You don't have to worry about zombie processes.
     #
     # We then do some tricks to make failure detection work and to make this
     # totally sound. If a test fails, we don't want the whole 'make' job to
     # stop. We also don't have any facility for makefile-escaping of path names.
     # We do have such a thing for shell-escaping, though. We fix both problems
     # by having the actual work for each of the test rules be done in a shell
     # script on the side. There is one such script per test. The script responds
     # to failure by printing something on the console and then touching a
     # failure file for that test, but then still returns 0. This makes 'make'
     # continue past that failure and complete all the tests anyway.
     #
     # In the end, this script collects all of the failures by searching for
     # files in the .parallel directory whose name matches /^test_fail_/, where
     # the thing after the 'fail_' is the test index. Those are the files that
     # would be created by the test scripts if they detect failure. We're
     # basically using the filesystem as a concurrent database of test failures.
     # Even if two tests fail at the same time, since they're touching different
     # files we won't miss any failures.

     runIndices = []
     $runlist.each_with_index {
         | plan, index |
         runIndices << index
         plan.index = index
     }

     parallelDir = $outputDir + ".parallel"
     Dir.mkdir(parallelDir) unless parallelDir.directory?
     toDelete = []
     Dir.foreach(parallelDir) {
         | filename |
         if filename =~ /^test_/
             toDelete << filename
         end
     }

     toDelete.each {
         | filename |
         File.unlink(parallelDir + filename)
     }

     $runlist.each {
         | plan |
         failCommand  = "{\n"
         failCommand += "    echo FAIL: #{Shellwords.shellescape(plan.name)}\n"
         failCommand += "    touch test_fail_#{plan.index}\n"
         failCommand += "}"
         plan.writeTestScript(parallelDir + "test_script_#{plan.index}", failCommand)
     }

     File.open(parallelDir + "Makefile", "w") {
         | outp |
         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
         runIndices.each {
             | index |
             plan = $runlist[index]
             outp.puts "test_done_#{index}:"
             outp.puts "\tsh test_script_#{plan.index}"
             outp.puts "\ttouch test_done_#{index}"
         }
     }

     Dir.chdir(parallelDir) {
         mysys("make", "-j", numProcessors.to_s, "-s", "-f", "Makefile")
     }

     Dir.foreach(parallelDir) {
         | filename |
         next unless filename =~ /test_fail_/
         appendFailure($runlist[$~.post_match.to_i])
     }
 else
     $runlist.each {
         | plan |
         print "#{plan.name}: "

         Dir.chdir(plan.directory) {
             if $verbosity >= 1
                 printCommandArray(*plan.arguments)
             end
             if system(*plan.arguments)
                 puts "OK."
             else
                 puts "FAIL: #{$?.inspect}"
                 appendFailure(plan)
             end
         }
     }
 end

 puts "Failed #{$numFailures} tests."
	#!/usr/bin/env ruby

	# Copyright (C) 2013 Apple Inc. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	#
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	#
	# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
	# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
	# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
	# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	require 'getoptlong'
	require 'pathname'
	require 'yaml'

	THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
	SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
	raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
	raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"

	HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"

	$haveShellwords = false

	begin
	require 'shellwords'
	$haveShellwords = true
	rescue Exception => e
	$stderr.puts "Warning: did not find shellwords; some features will be disabled."
	$stderr.puts "Error: #{e.inspect}"
	end

	$canRunDisplayProfilerOutput = false

	begin
	require 'json'
	require 'highline'
	$canRunDisplayProfilerOutput = true
	rescue Exception => e
	$stderr.puts "Warning: did not find json or highline; some features will be disabled."
	$stderr.puts "Error: #{e.inspect}"
	end

	def printCommandArray(*cmd)
	begin
	commandArray = cmd.each{\|value\| Shellwords.shellescape(value.to_s)}.join(' ')
	rescue
	commandArray = cmd.join(' ')
	end
	$stderr.puts ">> #{commandArray}"
	end

	def mysys(*cmd)
	printCommandArray(*cmd)
	raise "Command failed: #{$?.inspect}" unless system(*cmd)
	end

	numProcessors = `sysctl -n hw.availcpu`.to_i

	$jscPath = nil
	$enableFTL = false
	$collections = []
	$outputDir = Pathname.new("results")
	$parallel = ($haveShellwords and numProcessors > 1)
	$verbosity = 0

	def usage
	puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
	puts
	puts "--jsc (-j) Path to JavaScriptCore. This option is required."
	puts "--ftl-jit Indicate that we have the FTL JIT."
	puts "--[no-]parallel Run in parallel, or not. Default is #{$parallel}."
	puts "--output-dir (-o) Path where to put results. Default is #{$outputDir}."
	puts "--verbose (-v) Print more things while running."
	puts "--help (-h) Print this message."
	exit 1
	end

	GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
	['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
	['--ftl-jit', GetoptLong::NO_ARGUMENT],
	['--parallel', GetoptLong::NO_ARGUMENT],
	['--no-parallel', GetoptLong::NO_ARGUMENT],
	['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
	['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
	\| opt, arg \|
	case opt
	when '--help'
	usage
	when '--jsc'
	$jscPath = Pathname.new(arg).realpath
	when '--output-dir'
	$outputDir = Pathname.new(arg)
	when '--ftl-jit'
	$enableFTL = true
	when '--parallel'
	$parallel = true
	when '--no-parallel'
	$parallel = false
	when '--verbose'
	$verbosity += 1
	end
	}

	unless $jscPath
	$stderr.puts "Error: must specify -j <path>."
	exit 1
	end

	$numFailures = 0

	EAGER_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20"]

	$runlist = []

	class Plan
	attr_reader :directory, :arguments, :name
	attr_accessor :index

	def initialize(directory, arguments, name)
	@directory = directory.realpath
	@arguments = arguments
	@name = name
	end

	def writeTestScript(filename, failCommand)
	File.open(filename, "w") {
	\| outp \|
	outp.puts "echo Running #{Shellwords.shellescape(@name)}"
	cmd = ("(cd #{Shellwords.shellescape(@directory.to_s)} && " +
	@arguments.map{\|v\| Shellwords.shellescape(v)}.join(' ') +
	") \|\| #{failCommand}")
	if $verbosity >= 1
	outp.puts "echo #{Shellwords.shellescape(cmd)}"
	end
	outp.puts cmd
	}
	end
	end

	$uniqueFilenameCounter = 0
	def uniqueFilename(extension)
	payloadDir = $outputDir + "_payload"
	Dir.mkdir payloadDir unless payloadDir.directory?
	result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
	$uniqueFilenameCounter += 1
	result
	end

	def addRunCommand(kind, command)
	$runlist << Plan.new($benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}.#{kind}")
	end

	def run(kind, *options)
	addRunCommand(kind, [$jscPath.to_s] + options + [$benchmark.to_s])
	end

	def runDefault
	run("default")
	end

	def runNoCJIT
	run("no-cjit", "--enableConcurrentJIT=false")
	end

	def runDefaultFTL
	run("default-ftl", "--useExperimentalFTL=true")
	end

	def runFTLNoCJIT
	run("ftl-no-cjit", "--enableConcurrentJIT=false", "--useExperimentalFTL=true")
	end

	def runDFGEager
	run("dfg-eager", *EAGER_OPTIONS)
	end

	def runFTLEager
	run("ftl-eager", "--useExperimentalFTL=true", *EAGER_OPTIONS)
	end

	def runProfiler
	profilerOutput = uniqueFilename(".json")
	if $haveShellwords and $canRunDisplayProfilerOutput
	addRunCommand("profiler", ["ruby", (HELPERS_PATH + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, $jscPath.to_s, "-p", profilerOutput.to_s, $benchmark.to_s])
	else
	puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
	run("profiler-simple", "-p", profilerOutput.to_s)
	end
	end

	def defaultRun
	runDefault
	runNoCJIT
	if $enableFTL
	runDefaultFTL
	runFTLNoCJIT
	runFTLEager
	else
	runDFGEager
	end
	end

	def skip
	puts "Skipping #{$collectionName}/#{$benchmark}"
	end

	Dir.mkdir($outputDir) unless $outputDir.directory?
	begin
	File.delete($outputDir + "failed")
	rescue
	end

	$outputDir = $outputDir.realpath

	def allJSFiles(path)
	if path.file?
	[path]
	else
	result = []
	Dir.foreach(path) {
	\| filename \|
	next unless filename =~ /\.js$/
	next unless (path + filename).file?
	result << path + filename
	}
	result
	end
	end

	# Returns [collectionPath, collectionName]
	def simplifyCollectionName(collectionNames, collectionPath)
	outerDir = collectionPath.dirname
	name = collectionPath.basename
	lastName = name
	if collectionPath.directory?
	while lastName.to_s =~ /test/
	lastName = outerDir.basename
	name = lastName + name
	outerDir = outerDir.dirname
	end
	end
	collectionName = name.to_s
	toAdd = 1
	while collectionNames[collectionName]
	collectionName = File.basename(name.to_s) + "-#{toAdd}"
	toAdd += 1
	end
	collectionNames[collectionName] = true
	[collectionPath, collectionName]
	end

	def prepareCollection(name)
	dir = $outputDir
	Pathname.new(name).each_filename {
	\| filename \|
	dir = dir + filename
	Dir.mkdir(dir) unless dir.directory?
	}
	end

	collectionNames = {}

	ARGV.each {
	\| collection \|
	collection, collectionName = simplifyCollectionName(collectionNames, Pathname.new(collection))

	if collection.file?
	subCollectionNames = {}
	YAML::load(IO::read(collection)).each {
	\| entry \|
	path = collection.dirname + entry["path"]

	subCollection, subCollectionName = simplifyCollectionName(subCollectionNames, path)

	$collection = subCollection
	$collectionName = (Pathname.new(collectionName) + subCollectionName).to_s

	prepareCollection($collectionName)

	allJSFiles(path).each {
	\| path \|

	path = path.realpath

	$benchmark = path.basename
	$benchmarkDirectory = path.dirname

	eval entry["cmd"]
	}
	}
	else
	prepareCollection(collectionName)

	$collection = collection
	$collectionName = collectionName
	$benchmarkDirectory = $collection
	allJSFiles($collection).each {
	\| path \|

	$benchmark = path.basename

	didRun = false
	File.open($collection + $benchmark) {
	\| inp \|
	inp.each_line {
	\| line \|
	next unless line =~ /^\/\/@/
	eval $~.post_match
	didRun = true
	}
	}

	defaultRun unless didRun
	}
	end
	}

	def appendFailure(plan)
	File.open($outputDir + "failed", "a") {
	\| outp \|
	outp.puts plan.name
	}
	filename = $outputDir + plan.name
	begin
	plan.writeTestScript(filename, "exit 1")
	rescue => e
	$stderr.puts "Warning: failed to create repro file at #{filename}: #{e.inspect}"
	end
	$numFailures += 1
	end

	if $enableFTL and ENV["JSC_timeout"]
	# Currently, using the FTL is a performance regression particularly in real
	# (i.e. non-loopy) benchmarks. Account for this in the timeout.
	ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
	end

	if $parallel
	if ENV["JSC_timeout"]
	# In the worst case, the processors just interfere with each other.
	# Increase the timeout proportionally to the number of processors.
	ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt(numProcessors)).to_i.to_s
	end

	# The goals of our parallel test runner are scalability and simplicity. The
	# simplicity part is particularly important. We don't want to have to have
	# a full-time contributor just philosophising about parallel testing.
	#
	# As such, we just pass off all of the hard work to 'make'. This creates a
	# dummy directory ("$outputDir/.parallel") in which we create a dummy
	# Makefile. The Makefile has an 'all' rule that depends on all of the tests.
	# That is, for each test we know we will run, there is a rule in the
	# Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
	# Makefile results in 'make' doing all of the hard work:
	#
	# - Load balancing just works. Most systems have a great load balancer in
	# 'make'. If your system doesn't then just install a real 'make'.
	#
	# - Interruptions just work. For example Ctrl-C handling in 'make' is
	# exactly right. You don't have to worry about zombie processes.
	#
	# We then do some tricks to make failure detection work and to make this
	# totally sound. If a test fails, we don't want the whole 'make' job to
	# stop. We also don't have any facility for makefile-escaping of path names.
	# We do have such a thing for shell-escaping, though. We fix both problems
	# by having the actual work for each of the test rules be done in a shell
	# script on the side. There is one such script per test. The script responds
	# to failure by printing something on the console and then touching a
	# failure file for that test, but then still returns 0. This makes 'make'
	# continue past that failure and complete all the tests anyway.
	#
	# In the end, this script collects all of the failures by searching for
	# files in the .parallel directory whose name matches /^test_fail_/, where
	# the thing after the 'fail_' is the test index. Those are the files that
	# would be created by the test scripts if they detect failure. We're
	# basically using the filesystem as a concurrent database of test failures.
	# Even if two tests fail at the same time, since they're touching different
	# files we won't miss any failures.

	runIndices = []
	$runlist.each_with_index {
	\| plan, index \|
	runIndices << index
	plan.index = index
	}

	parallelDir = $outputDir + ".parallel"
	Dir.mkdir(parallelDir) unless parallelDir.directory?
	toDelete = []
	Dir.foreach(parallelDir) {
	\| filename \|
	if filename =~ /^test_/
	toDelete << filename
	end
	}

	toDelete.each {
	\| filename \|
	File.unlink(parallelDir + filename)
	}

	$runlist.each {
	\| plan \|
	failCommand = "{\n"
	failCommand += " echo FAIL: #{Shellwords.shellescape(plan.name)}\n"
	failCommand += " touch test_fail_#{plan.index}\n"
	failCommand += "}"
	plan.writeTestScript(parallelDir + "test_script_#{plan.index}", failCommand)
	}

	File.open(parallelDir + "Makefile", "w") {
	\| outp \|
	outp.puts("all: " + runIndices.map{\|v\| "test_done_#{v}"}.join(' '))
	runIndices.each {
	\| index \|
	plan = $runlist[index]
	outp.puts "test_done_#{index}:"
	outp.puts "\tsh test_script_#{plan.index}"
	outp.puts "\ttouch test_done_#{index}"
	}
	}

	Dir.chdir(parallelDir) {
	mysys("make", "-j", numProcessors.to_s, "-s", "-f", "Makefile")
	}

	Dir.foreach(parallelDir) {
	\| filename \|
	next unless filename =~ /test_fail_/
	appendFailure($runlist[$~.post_match.to_i])
	}
	else
	$runlist.each {
	\| plan \|
	print "#{plan.name}: "

	Dir.chdir(plan.directory) {
	if $verbosity >= 1
	printCommandArray(*plan.arguments)
	end
	if system(*plan.arguments)
	puts "OK."
	else
	puts "FAIL: #{$?.inspect}"
	appendFailure(plan)
	end
	}
	}
	end

	puts "Failed #{$numFailures} tests."