Tools/Scripts/import-test262-tests - WebKit - Git at Google

 #!/usr/bin/env ruby

 # Copyright (C) 2016 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 #
 # 1.  Redistributions of source code must retain the above copyright
 #     notice, this list of conditions and the following disclaimer.
 # 2.  Redistributions in binary form must reproduce the above copyright
 #     notice, this list of conditions and the following disclaimer in the
 #     documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 # Steps to update test262 test resources and yaml:
 #
 #   1. Run importer to update the yaml and test resources.
 #       $ import-test262-tests /path/to/test262
 #   2. Run the tests to produce a list of failures.
 #       $ run-jsc-stress-tests JSTests/test262.yaml
 #   3. Run importer to update the yaml with a new list of failures.
 #       $ import-test262-tests --failures ./results/failed /path/to/test262

 require 'fileutils'
 require 'getoptlong'
 require 'pathname'
 require 'yaml'
 require 'find'
 require 'shellwords'

 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
 TEST262_YAML_PATH = WEBKIT_PATH + "JSTests/test262.yaml"
 TEST262_REVISION_PATH = WEBKIT_PATH + "JSTests/test262/test262-Revision.txt"

 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"

 def usage
     puts "#{File.basename $0} [options] <path-to-test262-repository>"
     puts
     puts "-h, --help                   print this help message."
     puts "-f, --failures FAILURES      Supplied file will be used to determine which tests fail."
     puts "                             If a failures file is not provided all tests are assumed to pass,"
     puts "                             and test resources are not updated, only the yaml is updated."
     exit 1
 end

 $failures = nil
 JS_TEST_REGEXP = /.*\.js/
 JS_FIXTURE_REGEXP = /.*(_FIXTURE\.js|_\.js)/
 GET_YAML_REGEXP = /\/\*---(?<yaml>.*?)---\*\//m

 GetoptLong.new(["--help", "-h", GetoptLong::NO_ARGUMENT],
                ["--failures", "-f", GetoptLong::REQUIRED_ARGUMENT]).each {
     | opt, arg |
     case opt
     when "--help"
         usage
     when "--failures"
         $failures = {}
         File.open(Pathname.new(arg)).readlines.each {
             | line |
             match = line.match(/test262\/(.*?\.default.*?$)/)
             if (match)
                 $failures[match[1]] = true
             end
         }
     end
 }

 def didPassForMode(path, strict)
     if $failures
         if strict == :strict
             return !$failures.key?(path.to_s + ".default-strict")
         else
             return !$failures.key?(path.to_s + ".default")
         end
     else
         return true
     end
 end

 class Test
     attr_writer :failsWithException, :isModule, :isAsync
     attr_accessor :includeFiles, :needsStrict, :needsNonStrict
     attr_reader :path

     def initialize(path)
         @path = path
         @failsWithException = nil
         @includeFiles = []
         @needsStrict = true
         @needsNonStrict = true
         @isModule = false
         @isAsync = false
     end

     def check
         # Throw an exception here since I'm not sure if the test infrastructure works in these cases.
         raise if !@needsStrict and !@needsNonStrict
         raise if @isModule and !@needsNonStrict
     end

     def formatFlags(strict)
         flags = []
         flags << strict if strict == :strict
         flags << :module if @isModule
         flags << :async if @isAsync

         return flags.to_s
     end

     def formatCmdArguments(strict)
         raise if strict == :strict ? !@needsStrict : !@needsNonStrict
         passed = didPassForMode(@path, strict)
         cmd = "runTest262"
         cmd += passed ? " :normal, " : " :fail, "
         cmd += @failsWithException ? @failsWithException.inspect : "\"NoException\""
         cmd += ", #{@includeFiles.inspect}"
         cmd += ", #{formatFlags(strict)}"
     end

     def finalizeIncludes
         if @isAsync
             @includeFiles << "doneprintHandle.js"
         end

         dir = Pathname.new(".")
         @path.dirname.each_filename {
             | part |
             dir += ".."
         }
         dir += "harness"

         @includeFiles.map! { | file | (dir + file).to_s }
     end

 end

 def processTestFile(path)
     /\/\*---(?<yaml>.*?)---\*\//m =~ File::read(path)

     test = Test.new(path)
     # These should be included by all the tests
     test.includeFiles = ["assert.js", "sta.js"]

     begin
         yamlElements = YAML::load(yaml)
     rescue Exception => e
         puts "Failed to parse YAML for #{path}, threw exception:"
         puts e.inspect
     end
     yamlElements.each {
         | option |
         case option[0]
         when "negative"
             test.failsWithException = option[1]["type"].to_s
         when "includes"
             test.includeFiles += option[1]
         when "flags"
             option[1].each {
                 | flag |
                 case flag
                 when "raw", "noStrict"
                     test.needsStrict = false
                 when "onlyStrict"
                     test.needsNonStrict = false
                 when "module"
                     test.isModule = true
                     test.needsStrict = false
                 when "async"
                     test.isAsync = true
                 when "generated"
                 else
                     raise "Invalid Metadata flag option, #{flag}, when parsing #{$benchmarkDirectory + $benchmark}"
                 end
             }
         end
     }

     test.finalizeIncludes
     test.check

     return test
 end

 class Fixture
     attr_reader :path, :needsNonStrict, :needsStrict

     def initialize(path)
         @path = path
         @needsNonStrict = true
         @needsStrict = false
     end

     def formatCmdArguments(strict)
         return "prepareTest262Fixture"
     end
 end

 def processFixtureFile(path)
     Fixture.new(path)
 end

 def processFilesRecursively(path)
     tests = []

     Dir.chdir(path) {
         Find.find(Pathname.new("test")) {
             | file |
             if File.file?(file) and JS_TEST_REGEXP =~ file.to_s
                 path = Pathname.new(file)
                 if JS_FIXTURE_REGEXP =~ file.to_s
                     tests << processFixtureFile(path)
                 else
                     tests << processTestFile(path)
                 end
             end
         }
     }

     return tests
 end

 def printYAML(tests)
     File.open(TEST262_YAML_PATH, "w+") {
         | outp |
         outp.puts "---"
         tests.each {
             | test |
             if test.needsNonStrict
                 outp.puts "- path: test262/" + test.path.to_s
                 outp.puts "  cmd: " + test.formatCmdArguments(:nonStrict)
             end
             if test.needsStrict
                 outp.puts "- path: test262/" + test.path.to_s
                 outp.puts "  cmd: " + test.formatCmdArguments(:strict)
             end
         }
     }
 end

 def printRevision(test262Path)
     url = "unknown"
     branchname = url
     revision = url
     Dir.chdir(test262Path) {
         tracking = `git rev-parse --abbrev-ref --symbolic-full-name @{u}`.chomp
         remoteName, branchName = tracking.split("/")
         url = `git remote get-url #{remoteName}`
         revision = `git rev-parse HEAD`.strip
     }

     File.open(TEST262_REVISION_PATH, "w+") {
         | f |
         puts "test262 remote url: " + url
         puts "test262 revision: " + revision
         f.puts "test262 remote url: " + url
         f.puts "test262 revision: " + revision
     }
 end

 def replaceResources(test262Path)
     harnessSource = File.join(test262Path, "harness")
     harnessDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "harness")
     testSource = File.join(test262Path, "test")
     testDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "test")

     FileUtils.rm_r harnessDestination.to_s
     FileUtils.cp_r harnessSource.to_s, harnessDestination.to_s

     FileUtils.rm_r testDestination.to_s
     FileUtils.cp_r testSource.to_s, testDestination.to_s
 end

 if ARGV.empty?
     usage
 end

 puts "Writing #{File.basename(TEST262_REVISION_PATH)}..."
 test262Path = Pathname.new(ARGV[0])
 printRevision(test262Path)

 puts "Enumerating tests..."
 tests = processFilesRecursively(test262Path)

 puts "Writing #{File.basename(TEST262_YAML_PATH)}..."
 printYAML(tests)

 if !$failures
     puts "Replacing test262 resources..."
     replaceResources(test262Path)
 end
	#!/usr/bin/env ruby

	# Copyright (C) 2016 Apple Inc. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	#
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	#
	# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
	# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
	# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
	# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	# Steps to update test262 test resources and yaml:
	#
	# 1. Run importer to update the yaml and test resources.
	# $ import-test262-tests /path/to/test262
	# 2. Run the tests to produce a list of failures.
	# $ run-jsc-stress-tests JSTests/test262.yaml
	# 3. Run importer to update the yaml with a new list of failures.
	# $ import-test262-tests --failures ./results/failed /path/to/test262

	require 'fileutils'
	require 'getoptlong'
	require 'pathname'
	require 'yaml'
	require 'find'
	require 'shellwords'

	THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
	SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
	WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
	TEST262_YAML_PATH = WEBKIT_PATH + "JSTests/test262.yaml"
	TEST262_REVISION_PATH = WEBKIT_PATH + "JSTests/test262/test262-Revision.txt"

	raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
	raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"

	def usage
	puts "#{File.basename $0} [options] <path-to-test262-repository>"
	puts
	puts "-h, --help print this help message."
	puts "-f, --failures FAILURES Supplied file will be used to determine which tests fail."
	puts " If a failures file is not provided all tests are assumed to pass,"
	puts " and test resources are not updated, only the yaml is updated."
	exit 1
	end

	$failures = nil
	JS_TEST_REGEXP = /.*\.js/
	JS_FIXTURE_REGEXP = /.*(_FIXTURE\.js\|_\.js)/
	GET_YAML_REGEXP = /\/\---(?<yaml>.?)---\*\//m

	GetoptLong.new(["--help", "-h", GetoptLong::NO_ARGUMENT],
	["--failures", "-f", GetoptLong::REQUIRED_ARGUMENT]).each {
	\| opt, arg \|
	case opt
	when "--help"
	usage
	when "--failures"
	$failures = {}
	File.open(Pathname.new(arg)).readlines.each {
	\| line \|
	match = line.match(/test262\/(.?\.default.?$)/)
	if (match)
	$failures[match[1]] = true
	end
	}
	end
	}

	def didPassForMode(path, strict)
	if $failures
	if strict == :strict
	return !$failures.key?(path.to_s + ".default-strict")
	else
	return !$failures.key?(path.to_s + ".default")
	end
	else
	return true
	end
	end

	class Test
	attr_writer :failsWithException, :isModule, :isAsync
	attr_accessor :includeFiles, :needsStrict, :needsNonStrict
	attr_reader :path

	def initialize(path)
	@path = path
	@failsWithException = nil
	@includeFiles = []
	@needsStrict = true
	@needsNonStrict = true
	@isModule = false
	@isAsync = false
	end

	def check
	# Throw an exception here since I'm not sure if the test infrastructure works in these cases.
	raise if !@needsStrict and !@needsNonStrict
	raise if @isModule and !@needsNonStrict
	end

	def formatFlags(strict)
	flags = []
	flags << strict if strict == :strict
	flags << :module if @isModule
	flags << :async if @isAsync

	return flags.to_s
	end

	def formatCmdArguments(strict)
	raise if strict == :strict ? !@needsStrict : !@needsNonStrict
	passed = didPassForMode(@path, strict)
	cmd = "runTest262"
	cmd += passed ? " :normal, " : " :fail, "
	cmd += @failsWithException ? @failsWithException.inspect : "\"NoException\""
	cmd += ", #{@includeFiles.inspect}"
	cmd += ", #{formatFlags(strict)}"
	end

	def finalizeIncludes
	if @isAsync
	@includeFiles << "doneprintHandle.js"
	end

	dir = Pathname.new(".")
	@path.dirname.each_filename {
	\| part \|
	dir += ".."
	}
	dir += "harness"

	@includeFiles.map! { \| file \| (dir + file).to_s }
	end

	end

	def processTestFile(path)
	/\/\---(?<yaml>.?)---\*\//m =~ File::read(path)

	test = Test.new(path)
	# These should be included by all the tests
	test.includeFiles = ["assert.js", "sta.js"]

	begin
	yamlElements = YAML::load(yaml)
	rescue Exception => e
	puts "Failed to parse YAML for #{path}, threw exception:"
	puts e.inspect
	end
	yamlElements.each {
	\| option \|
	case option[0]
	when "negative"
	test.failsWithException = option[1]["type"].to_s
	when "includes"
	test.includeFiles += option[1]
	when "flags"
	option[1].each {
	\| flag \|
	case flag
	when "raw", "noStrict"
	test.needsStrict = false
	when "onlyStrict"
	test.needsNonStrict = false
	when "module"
	test.isModule = true
	test.needsStrict = false
	when "async"
	test.isAsync = true
	when "generated"
	else
	raise "Invalid Metadata flag option, #{flag}, when parsing #{$benchmarkDirectory + $benchmark}"
	end
	}
	end
	}

	test.finalizeIncludes
	test.check

	return test
	end

	class Fixture
	attr_reader :path, :needsNonStrict, :needsStrict

	def initialize(path)
	@path = path
	@needsNonStrict = true
	@needsStrict = false
	end

	def formatCmdArguments(strict)
	return "prepareTest262Fixture"
	end
	end

	def processFixtureFile(path)
	Fixture.new(path)
	end

	def processFilesRecursively(path)
	tests = []

	Dir.chdir(path) {
	Find.find(Pathname.new("test")) {
	\| file \|
	if File.file?(file) and JS_TEST_REGEXP =~ file.to_s
	path = Pathname.new(file)
	if JS_FIXTURE_REGEXP =~ file.to_s
	tests << processFixtureFile(path)
	else
	tests << processTestFile(path)
	end
	end
	}
	}

	return tests
	end

	def printYAML(tests)
	File.open(TEST262_YAML_PATH, "w+") {
	\| outp \|
	outp.puts "---"
	tests.each {
	\| test \|
	if test.needsNonStrict
	outp.puts "- path: test262/" + test.path.to_s
	outp.puts " cmd: " + test.formatCmdArguments(:nonStrict)
	end
	if test.needsStrict
	outp.puts "- path: test262/" + test.path.to_s
	outp.puts " cmd: " + test.formatCmdArguments(:strict)
	end
	}
	}
	end

	def printRevision(test262Path)
	url = "unknown"
	branchname = url
	revision = url
	Dir.chdir(test262Path) {
	tracking = `git rev-parse --abbrev-ref --symbolic-full-name @{u}`.chomp
	remoteName, branchName = tracking.split("/")
	url = `git remote get-url #{remoteName}`
	revision = `git rev-parse HEAD`.strip
	}

	File.open(TEST262_REVISION_PATH, "w+") {
	\| f \|
	puts "test262 remote url: " + url
	puts "test262 revision: " + revision
	f.puts "test262 remote url: " + url
	f.puts "test262 revision: " + revision
	}
	end

	def replaceResources(test262Path)
	harnessSource = File.join(test262Path, "harness")
	harnessDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "harness")
	testSource = File.join(test262Path, "test")
	testDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "test")

	FileUtils.rm_r harnessDestination.to_s
	FileUtils.cp_r harnessSource.to_s, harnessDestination.to_s

	FileUtils.rm_r testDestination.to_s
	FileUtils.cp_r testSource.to_s, testDestination.to_s
	end

	if ARGV.empty?
	usage
	end

	puts "Writing #{File.basename(TEST262_REVISION_PATH)}..."
	test262Path = Pathname.new(ARGV[0])
	printRevision(test262Path)

	puts "Enumerating tests..."
	tests = processFilesRecursively(test262Path)

	puts "Writing #{File.basename(TEST262_YAML_PATH)}..."
	printYAML(tests)

	if !$failures
	puts "Replacing test262 resources..."
	replaceResources(test262Path)
	end