blob: 60c0624dbb6b349f9dab0a91b161dd3eb9189c6a [file] [log] [blame]
#!/usr/bin/env ruby
# Copyright (C) 2016 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Steps to update test262 test resources and yaml:
#
# 1. Run importer to update the yaml and test resources.
# $ import-test262-tests /path/to/test262
# 2. Run the tests to produce a list of failures.
# $ run-jsc-stress-tests JSTests/test262.yaml
# 3. Run importer to update the yaml with a new list of failures.
# $ import-test262-tests --failures ./results/failed /path/to/test262
require 'fileutils'
require 'getoptlong'
require 'pathname'
require 'yaml'
require 'find'
require 'shellwords'
THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
TEST262_YAML_PATH = WEBKIT_PATH + "JSTests/test262.yaml"
TEST262_REVISION_PATH = WEBKIT_PATH + "JSTests/test262/test262-Revision.txt"
raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
def usage
puts "#{File.basename $0} [options] <path-to-test262-repository>"
puts
puts "-h, --help print this help message."
puts "-f, --failures FAILURES Supplied file will be used to determine which tests fail."
puts " If a failures file is not provided all tests are assumed to pass,"
puts " and test resources are not updated, only the yaml is updated."
exit 1
end
$failures = nil
JS_TEST_REGEXP = /.*\.js/
JS_FIXTURE_REGEXP = /.*(_FIXTURE\.js|_\.js)/
GET_YAML_REGEXP = /\/\*---(?<yaml>.*?)---\*\//m
GetoptLong.new(["--help", "-h", GetoptLong::NO_ARGUMENT],
["--failures", "-f", GetoptLong::REQUIRED_ARGUMENT]).each {
| opt, arg |
case opt
when "--help"
usage
when "--failures"
$failures = {}
File.open(Pathname.new(arg)).readlines.each {
| line |
match = line.match(/test262\/(.*?\.default.*?$)/)
if (match)
$failures[match[1]] = true
end
}
end
}
def didPassForMode(path, strict)
if $failures
if strict == :strict
return !$failures.key?(path.to_s + ".default-strict")
else
return !$failures.key?(path.to_s + ".default")
end
else
return true
end
end
class Test
attr_writer :failsWithException, :isModule, :isAsync
attr_accessor :includeFiles, :needsStrict, :needsNonStrict
attr_reader :path
def initialize(path)
@path = path
@failsWithException = nil
@includeFiles = []
@needsStrict = true
@needsNonStrict = true
@isModule = false
@isAsync = false
end
def check
# Throw an exception here since I'm not sure if the test infrastructure works in these cases.
raise if !@needsStrict and !@needsNonStrict
raise if @isModule and !@needsNonStrict
end
def formatFlags(strict)
flags = []
flags << strict if strict == :strict
flags << :module if @isModule
flags << :async if @isAsync
return flags.to_s
end
def formatCmdArguments(strict)
raise if strict == :strict ? !@needsStrict : !@needsNonStrict
passed = didPassForMode(@path, strict)
cmd = "runTest262"
cmd += passed ? " :normal, " : " :fail, "
cmd += @failsWithException ? @failsWithException.inspect : "\"NoException\""
cmd += ", #{@includeFiles.inspect}"
cmd += ", #{formatFlags(strict)}"
end
def finalizeIncludes
if @isAsync
@includeFiles << "doneprintHandle.js"
end
dir = Pathname.new(".")
@path.dirname.each_filename {
| part |
dir += ".."
}
dir += "harness"
@includeFiles.map! { | file | (dir + file).to_s }
end
end
def processTestFile(path)
/\/\*---(?<yaml>.*?)---\*\//m =~ File::read(path)
test = Test.new(path)
# These should be included by all the tests
test.includeFiles = ["assert.js", "sta.js"]
begin
yamlElements = YAML::load(yaml)
rescue Exception => e
puts "Failed to parse YAML for #{path}, threw exception:"
puts e.inspect
end
yamlElements.each {
| option |
case option[0]
when "negative"
test.failsWithException = option[1]["type"].to_s
when "includes"
test.includeFiles += option[1]
when "flags"
option[1].each {
| flag |
case flag
when "raw", "noStrict"
test.needsStrict = false
when "onlyStrict"
test.needsNonStrict = false
when "module"
test.isModule = true
test.needsStrict = false
when "async"
test.isAsync = true
when "generated"
else
raise "Invalid Metadata flag option, #{flag}, when parsing #{$benchmarkDirectory + $benchmark}"
end
}
end
}
test.finalizeIncludes
test.check
return test
end
class Fixture
attr_reader :path, :needsNonStrict, :needsStrict
def initialize(path)
@path = path
@needsNonStrict = true
@needsStrict = false
end
def formatCmdArguments(strict)
return "prepareTest262Fixture"
end
end
def processFixtureFile(path)
Fixture.new(path)
end
def processFilesRecursively(path)
tests = []
Dir.chdir(path) {
Find.find(Pathname.new("test")) {
| file |
if File.file?(file) and JS_TEST_REGEXP =~ file.to_s
path = Pathname.new(file)
if JS_FIXTURE_REGEXP =~ file.to_s
tests << processFixtureFile(path)
else
tests << processTestFile(path)
end
end
}
}
return tests
end
def printYAML(tests)
File.open(TEST262_YAML_PATH, "w+") {
| outp |
outp.puts "---"
tests.each {
| test |
if test.needsNonStrict
outp.puts "- path: test262/" + test.path.to_s
outp.puts " cmd: " + test.formatCmdArguments(:nonStrict)
end
if test.needsStrict
outp.puts "- path: test262/" + test.path.to_s
outp.puts " cmd: " + test.formatCmdArguments(:strict)
end
}
}
end
def printRevision(test262Path)
url = "unknown"
branchname = url
revision = url
Dir.chdir(test262Path) {
tracking = `git rev-parse --abbrev-ref --symbolic-full-name @{u}`.chomp
remoteName, branchName = tracking.split("/")
url = `git remote get-url #{remoteName}`
revision = `git rev-parse HEAD`.strip
}
File.open(TEST262_REVISION_PATH, "w+") {
| f |
puts "test262 remote url: " + url
puts "test262 revision: " + revision
f.puts "test262 remote url: " + url
f.puts "test262 revision: " + revision
}
end
def replaceResources(test262Path)
harnessSource = File.join(test262Path, "harness")
harnessDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "harness")
testSource = File.join(test262Path, "test")
testDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "test")
FileUtils.rm_r harnessDestination.to_s
FileUtils.cp_r harnessSource.to_s, harnessDestination.to_s
FileUtils.rm_r testDestination.to_s
FileUtils.cp_r testSource.to_s, testDestination.to_s
end
if ARGV.empty?
usage
end
puts "Writing #{File.basename(TEST262_REVISION_PATH)}..."
test262Path = Pathname.new(ARGV[0])
printRevision(test262Path)
puts "Enumerating tests..."
tests = processFilesRecursively(test262Path)
puts "Writing #{File.basename(TEST262_YAML_PATH)}..."
printYAML(tests)
if !$failures
puts "Replacing test262 resources..."
replaceResources(test262Path)
end