blob: 0553f2eadd3f72f06f3f32a72c533d2bdc229e75 [file] [log] [blame]
# Copyright (C) 2013-2016 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
def prefixCommand(prefix)
"awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
end
def redirectAndPrefixCommand(prefix)
prefixCommand(prefix) + " 2>&1"
end
def pipeAndPrefixCommand(outputFilename, prefix)
"tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
end
# Output handler for tests that are expected to be silent.
def silentOutputHandler
Proc.new {
| name |
" | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
}
end
# Output handler for tests that are expected to produce meaningful output.
def noisyOutputHandler
Proc.new {
| name |
" | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
}
end
# Error handler for tests that fail exactly when they return non-zero exit status.
# This is useful when a test is expected to fail.
def simpleErrorHandler
Proc.new {
| outp, plan |
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
# Error handler for tests that fail exactly when they return zero exit status.
def expectedFailErrorHandler
Proc.new {
| outp, plan |
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "else"
outp.puts " (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that fail exactly when they return non-zero exit status and produce
# lots of spew. This will echo that spew when the test fails.
def noisyErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
# Error handler for tests that diff their output with some expectation.
def diffErrorHandler(expectedFilename)
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
outp.puts "then"
outp.puts " diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
outp.puts " if [ $? -eq 0 ]"
outp.puts " then"
outp.puts " " + plan.successCommand
outp.puts " else"
outp.puts " (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts " fi"
outp.puts "else"
outp.puts " (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that report error by saying "failed!". This is used by Mozilla
# tests.
def mozillaErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif grep -i -q failed! #{outputFilename}"
outp.puts "then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
# Error handler for tests that report error by saying "failed!", and are expected to
# fail. This is used by Mozilla tests.
def mozillaFailErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "elif grep -i -q failed! #{outputFilename}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "else"
outp.puts " (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that report error by saying "failed!", and are expected to have
# an exit code of 3.
def mozillaExit3ErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " if [ `cat #{plan.failFile}` -eq 3 ]"
outp.puts " then"
outp.puts " if grep -i -q failed! #{outputFilename}"
outp.puts " then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts " else"
outp.puts " " + plan.successCommand
outp.puts " fi"
outp.puts " else"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts " fi"
outp.puts "else"
outp.puts " (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "fi"
}
end
# Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
# This is used by Chakra tests.
def chakraPassFailErrorHandler
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif grep -i -q FAILED #{outputFilename}"
outp.puts "then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
outp.puts "fi"
}
end
class Plan
attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler, :additionalEnv
attr_accessor :index
def initialize(directory, arguments, family, name, outputHandler, errorHandler)
@directory = directory
@arguments = arguments
@family = family
@name = name
@outputHandler = outputHandler
@errorHandler = errorHandler
@isSlow = !!$runCommandOptions[:isSlow]
@additionalEnv = []
end
def shellCommand
# It's important to remember that the test is actually run in a subshell, so if we change directory
# in the subshell when we return we will be in our original directory. This is nice because we don't
# have to bend over backwards to do things relative to the root.
script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
($envVars + additionalEnv).each { |var| script += "export " << var << "; " }
script += "\"$@\" " + escapeAll(@arguments) + "))"
return script
end
def reproScriptCommand
# We have to find our way back to the .runner directory since that's where all of the relative
# paths assume they start out from.
script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
script += "cd $CURRENT_DIR\n"
Pathname.new(@name).dirname.each_filename {
| pathComponent |
script += "cd ..\n"
}
script += "cd .runner\n"
script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
$envVars.each { |var| script += "export " << var << "\n" }
script += "#{shellCommand} || exit 1"
"echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
end
def failCommand
"echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
end
def successCommand
if $progressMeter or $verbosity >= 2
"rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
else
"rm -f #{failFile}"
end
end
def failFile
"test_fail_#{@index}"
end
def writeRunScript(filename)
File.open(filename, "w") {
| outp |
outp.puts "echo Running #{Shellwords.shellescape(@name)}"
cmd = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
cmd += @outputHandler.call(@name)
if $verbosity >= 3
outp.puts "echo #{Shellwords.shellescape(cmd)}"
end
outp.puts cmd
@errorHandler.call(outp, self)
}
end
end
def prepareShellTestRunner
FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
end
def prepareMakeTestRunner
# The goals of our parallel test runner are scalability and simplicity. The
# simplicity part is particularly important. We don't want to have to have
# a full-time contributor just philosophising about parallel testing.
#
# As such, we just pass off all of the hard work to 'make'. This creates a
# dummy directory ("$outputDir/.runner") in which we create a dummy
# Makefile. The Makefile has an 'all' rule that depends on all of the tests.
# That is, for each test we know we will run, there is a rule in the
# Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
# Makefile results in 'make' doing all of the hard work:
#
# - Load balancing just works. Most systems have a great load balancer in
# 'make'. If your system doesn't then just install a real 'make'.
#
# - Interruptions just work. For example Ctrl-C handling in 'make' is
# exactly right. You don't have to worry about zombie processes.
#
# We then do some tricks to make failure detection work and to make this
# totally sound. If a test fails, we don't want the whole 'make' job to
# stop. We also don't have any facility for makefile-escaping of path names.
# We do have such a thing for shell-escaping, though. We fix both problems
# by having the actual work for each of the test rules be done in a shell
# script on the side. There is one such script per test. The script responds
# to failure by printing something on the console and then touching a
# failure file for that test, but then still returns 0. This makes 'make'
# continue past that failure and complete all the tests anyway.
#
# In the end, this script collects all of the failures by searching for
# files in the .runner directory whose name matches /^test_fail_/, where
# the thing after the 'fail_' is the test index. Those are the files that
# would be created by the test scripts if they detect failure. We're
# basically using the filesystem as a concurrent database of test failures.
# Even if two tests fail at the same time, since they're touching different
# files we won't miss any failures.
runIndices = []
$runlist.each {
| plan |
runIndices << plan.index
}
File.open($runnerDir + "Makefile", "w") {
| outp |
outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
runIndices.each {
| index |
plan = $runlist[index]
outp.puts "test_done_#{index}:"
outp.puts "\tsh test_script_#{plan.index}"
}
}
end
def prepareRubyTestRunner
File.open($runnerDir + "runscript", "w") {
| outp |
$runlist.each {
| plan |
outp.puts "print `sh test_script_#{plan.index} 2>&1`"
}
}
end
def testRunnerCommand
case $testRunnerType
when :shell
command = "sh runscript"
when :make
command = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
when :ruby
command = "ruby runscript"
else
raise "Unknown test runner type: #{$testRunnerType.to_s}"
end
return command
end