Tool to mark jsc test skip/enable
https://bugs.webkit.org/show_bug.cgi?id=202063
Reviewed by Keith Miller.
* Scripts/run-javascriptcore-tests:
(runJSCStressTests):
* Scripts/run-jsc-stress-tests:
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@251161 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index bbd51bf..ea397bc 100644
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,14 @@
+2019-10-15 Zhifei Fang <zhifei_fang@apple.com>
+
+ Tool to mark jsc test skip/enable
+ https://bugs.webkit.org/show_bug.cgi?id=202063
+
+ Reviewed by Keith Miller.
+
+ * Scripts/run-javascriptcore-tests:
+ (runJSCStressTests):
+ * Scripts/run-jsc-stress-tests:
+
2019-10-15 Peng Liu <peng.liu6@apple.com>
[Picture-in-Picture Web API] Implement HTMLVideoElement.requestPictureInPicture() / Document.exitPictureInPicture()
diff --git a/Tools/Scripts/mark-jsc-stress-test b/Tools/Scripts/mark-jsc-stress-test
new file mode 100755
index 0000000..43e01b3
--- /dev/null
+++ b/Tools/Scripts/mark-jsc-stress-test
@@ -0,0 +1,171 @@
+#!/usr/bin/env python -u
+import os
+import sys
+import getopt
+import argparse
+import re
+import logging
+import json
+
+logger = logging.getLogger()
+handler = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+logger.setLevel(logging.INFO)
+logger.addHandler(handler)
+
+def iter_dir_recusive(path, callback):
+ if os.path.isfile(path) and os.path.splitext(path)[1] == '.js':
+ logger.info("Processing {} ...".format(path))
+ callback(path)
+ else:
+ for root, dirs, files in os.walk(path):
+ for name in files:
+ iter_dir_recusive(os.path.join(root, name), callback)
+ logger.info("Done.")
+
+
+class JSCTestModifier(object):
+ variables = [
+ "$hostOs", "$model", "$architecture"
+ ]
+ def __init__(self, test_pathes, conditions={}, match="all"):
+ self._conditions = conditions
+ self._test_pathes = test_pathes
+ self._match = match
+ self._skip_line_postfix = "# added by mark-jsc-stress-test.py"
+
+ def skip(self):
+ for path in self._test_pathes:
+ logger.info("Mark {} skip".format(path))
+ iter_dir_recusive(path, lambda file_path: self._skip_test_file(file_path))
+
+ def enable(self):
+ for path in self._test_pathes:
+ logger.info("Mark {} enable".format(path))
+ iter_dir_recusive(path, lambda file_path: self._enable_test_file(file_path))
+
+ def _generate_condition_op(self, value):
+ op = "=="
+ if ("!" == value[0]):
+ op = "!="
+ value = value[1:]
+ return op, value
+
+ # Condition grammer is like !A or B or C and D
+ # Translate it to ruby: $hostOs != A or $hostOs == B or $hostOs == C and $hostOs == D
+ def _parse_condition(self, variable, condition):
+ res = []
+ values = []
+ for word in re.split(r'\s+', condition):
+ if word == "or" or word == "and":
+ value = " ".join(values).strip()
+ values = []
+ res.append('{} {} "{}"'.format(variable, *self._generate_condition_op(value)))
+ res.append(word)
+ else:
+ values.append(word)
+ if values:
+ value = " ".join(values).strip()
+ res.append('{} {} "{}"'.format(variable, *self._generate_condition_op(value)))
+ return " ".join(res)
+
+ def _generate_skip_annotation_line(self):
+ skip_line_prefix = "//@ skip if"
+ skip_conditions = []
+ skip_line = "{} {} {}"
+ supported_variables = filter(lambda variable: variable in self._conditions, JSCTestModifier.variables)
+ condition_template = "{}" if len(supported_variables) == 1 else "({})"
+ for variable in supported_variables:
+ skip_conditions.append(condition_template.format(self._parse_condition(variable, self._conditions[variable])))
+ if not skip_conditions:
+ # No conditions, always skip
+ skip_conditions = ["true"]
+ if self._match == "any":
+ skip_line = skip_line.format(skip_line_prefix, " or ".join(skip_conditions), self._skip_line_postfix)
+ elif self._match == "all":
+ skip_line = skip_line.format(skip_line_prefix, " and ".join(skip_conditions), self._skip_line_postfix)
+ return skip_line
+
+ # This can only remove the skip annotation generated by this script
+ def _enable_test_file(self, test_file):
+ with open(test_file, 'r+') as f:
+ lines = f.readlines()
+ f.seek(0)
+ for line in lines:
+ if not self._skip_line_postfix in line:
+ f.write(line)
+ f.truncate()
+
+
+ def _skip_test_file(self, test_file):
+ # remove the exisiting skip line, so that we can apply the new one
+ self._enable_test_file(test_file)
+ skip_line = self._generate_skip_annotation_line()
+ with open(test_file, 'r+') as f:
+ original_content = f.read()
+ f.seek(0)
+ f.write("{}\n{}".format(skip_line, original_content))
+
+opensource_root = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../")
+jsc_test_search_path = [
+ os.path.join(opensource_root, "JSTests"),
+ os.path.join(opensource_root, "LayoutTests")
+]
+def main():
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest="action")
+ file_list_help = "Files/directories list; use ',' to separate each item. Example: a.js, b.js, c.js Use '-' if you are using --jsc-json-output argument"
+ parser_enable = subparsers.add_parser("enable", help="Enable the tests which are marked as skipped by this script")
+ parser_enable.add_argument("files", help=file_list_help)
+ parser_enable.add_argument("--jsc-json-output", help="Pass the json output of run-javascriptcore-tests to unskip all failed tests")
+
+ parser_skip = subparsers.add_parser("skip", help="Insert skip condition to given files/directories")
+ parser_skip.add_argument("files", help=file_list_help)
+ parser_skip.add_argument("--jsc-json-output", help="Pass the json output of run-javascriptcore-tests to skip all failed tests")
+ parser_skip.add_argument("--platform", "--host-os", help="Skip if host os matches given value, Examples: 'windows or linux' '!windows and !linux'")
+ parser_skip.add_argument("--model", help="Skip if hardware model matches given value, Examples: 'Apple Watch Series 3 or Apple Watch Series 4' '!Apple Watch Series 3 and !Apple Watch Series 4'")
+ parser_skip.add_argument("--architecture", help="Skip if architecture matches given value, Examples: 'arm or x86' '!arm and !x86'")
+ parser_skip.add_argument("--match", default="all", help="Match all or any above conditions")
+
+ args = vars(parser.parse_args())
+ conditions = {}
+ files = []
+ if not args["files"] and not args["--log-file"]:
+ logger.error("Please speicify a list of file, or use --log-file to give a JSC test log")
+ return 1
+ if args["files"] and not args["files"] == "-":
+ files += args["files"].split(",")
+ if args["jsc_json_output"]:
+ with open(args["jsc_json_output"]) as f:
+ jsc_json_output = json.load(f)
+ failures = jsc_json_output["stressTestFailures"]
+ failure_test_files_set = set()
+ for failure_test in failures:
+ path_parts = failure_test.split(os.path.sep)
+ if 'yaml' in path_parts[0]:
+ failure_test_path = os.path.join(*path_parts[1:])
+ else:
+ failure_test_path = failure_test
+ failure_test_file = os.path.splitext(failure_test_path)[0]
+ for search_path in jsc_test_search_path:
+ whole_path = os.path.join(search_path, failure_test_file)
+ if failure_test_file not in failure_test_files_set and os.path.isfile(whole_path):
+ files.append(whole_path)
+ failure_test_files_set.add(failure_test_file)
+
+ if "host_os" in args and args["host_os"]:
+ conditions["$hostOs"] = args["host_os"]
+ if "platform" in args and args["platform"]:
+ conditions["$hostOs"] = args["platform"]
+ if "architecture" in args and args["architecture"]:
+ conditions["$architecture"] = args["architecture"]
+ if "model" in args and args["model"]:
+ conditions["$model"] = args["model"]
+
+ modifer = JSCTestModifier(files, conditions, args["match"] if "match" in args else None)
+ getattr(modifer, args["action"])()
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/Tools/Scripts/run-javascriptcore-tests b/Tools/Scripts/run-javascriptcore-tests
index 8fcac48..f3bba60 100755
--- a/Tools/Scripts/run-javascriptcore-tests
+++ b/Tools/Scripts/run-javascriptcore-tests
@@ -90,6 +90,7 @@
my $createTarball = 0;
my $remoteHost = 0;
+my $model = 0;
my $failFast = 1;
my %jsonData = ();
my @testResults = ();
@@ -229,6 +230,7 @@
--tarball Create a tarball of the bundle produced by running the JSC stress tests.
--remote= Run the JSC stress tests on the specified remote host. Implies --tarball.
--remote-config-file= Same as remote, but read config from JSON file.
+ --model= Specify remote hardware model, this info used for determine what jsc tests should run on remote
--extra-tests= Path to a file containing extra tests
--child-processes= Specify the number of child processes.
--shell-runner Uses the shell-based test runner instead of the default make-based runner.
@@ -286,6 +288,7 @@
'json-output=s' => \$jsonFileName,
'tarball!' => \$createTarball,
'remote=s' => \$remoteHost,
+ 'model=s' => \$model,
'remote-config-file=s' => \$remoteConfigFile,
'child-processes=s' => \$childProcesses,
'shell-runner' => \$shellRunner,
@@ -536,6 +539,11 @@
push(@jscStressDriverCmd, "--remote-config-file");
push(@jscStressDriverCmd, $remoteConfigFile);
}
+
+ if ($model) {
+ push(@jscStressDriverCmd, "--model");
+ push(@jscStressDriverCmd, $model);
+ }
if ($childProcesses) {
push(@jscStressDriverCmd, "--child-processes");
diff --git a/Tools/Scripts/run-jsc-stress-tests b/Tools/Scripts/run-jsc-stress-tests
index 54dfc0b..85c7bcd 100755
--- a/Tools/Scripts/run-jsc-stress-tests
+++ b/Tools/Scripts/run-jsc-stress-tests
@@ -114,6 +114,7 @@
$remoteHosts = []
$architecture = nil
$hostOS = nil
+$model = nil
$filter = nil
$envVars = []
$mode = "full"
@@ -183,6 +184,7 @@
['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
['--remote', GetoptLong::REQUIRED_ARGUMENT],
['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
+ ['--model', GetoptLong::REQUIRED_ARGUMENT],
['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
['--filter', GetoptLong::REQUIRED_ARGUMENT],
['--verbose', '-v', GetoptLong::NO_ARGUMENT],
@@ -241,6 +243,8 @@
$architecture = arg
when '--os'
$hostOS = arg
+ when '--model'
+ $model = arg
when '--env-vars'
$envVars = arg.gsub(/\s+/, ' ').split(' ')
when '--quick'
