[ews-build] Parse and display layout test failures
https://bugs.webkit.org/show_bug.cgi?id=199709
Rubber-stamped by Jonathan Bedard.
* BuildSlaveSupport/ews-build/steps.py:
(RunWebKitTests.start): Initialize log_observer.
(RunWebKitTests._strip_python_logging_prefix): Copied from similar code in build.webkit.org buildbot config.
(RunWebKitTests._parseRunWebKitTestsOutput): Ditto.
(RunWebKitTests.commandComplete): Gather and parse the stdout and stderr logs.
(RunWebKitTests.evaluateResult): Analyze the results and decide build status.
(RunWebKitTests.getResultSummary): Update build and step summary.
* BuildSlaveSupport/ews-build/steps_unittest.py: Added and updated unit-tests.
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@247433 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Tools/BuildSlaveSupport/ews-build/steps.py b/Tools/BuildSlaveSupport/ews-build/steps.py
index d16b14a..5fb86d1 100644
--- a/Tools/BuildSlaveSupport/ews-build/steps.py
+++ b/Tools/BuildSlaveSupport/ews-build/steps.py
@@ -844,6 +844,10 @@
WithProperties('--%(configuration)s')]
def start(self):
+ self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
+ self.addLogObserver('stdio', self.log_observer)
+
+ self.incorrectLayoutLines = []
platform = self.getProperty('platform')
appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
additionalArguments = self.getProperty('additionalArguments')
@@ -855,9 +859,76 @@
self.setCommand(self.command + additionalArguments)
return shell.Test.start(self)
+ # FIXME: This will break if run-webkit-tests changes its default log formatter.
+ nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
+
+ def _strip_python_logging_prefix(self, line):
+ match_object = self.nrwt_log_message_regexp.match(line)
+ if match_object:
+ return match_object.group('message')
+ return line
+
+ def _parseRunWebKitTestsOutput(self, logText):
+ incorrectLayoutLines = []
+ expressions = [
+ ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
+ ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
+ ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
+ ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
+ ]
+ testFailures = {}
+
+ for line in logText.splitlines():
+ if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
+ incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
+ continue
+ for name, expression in expressions:
+ match = expression.search(line)
+
+ if match:
+ testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
+ break
+
+ # FIXME: Parse file names and put them in results
+
+ for name in testFailures:
+ incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
+
+ self.incorrectLayoutLines = incorrectLayoutLines
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+ logText = self.log_observer.getStdout() + self.log_observer.getStderr()
+ self._parseRunWebKitTestsOutput(logText)
+
+ def evaluateResult(self, cmd):
+ result = SUCCESS
+
+ if self.incorrectLayoutLines:
+ if len(self.incorrectLayoutLines) == 1:
+ line = self.incorrectLayoutLines[0]
+ if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
+ return WARNINGS
+
+ for line in self.incorrectLayoutLines:
+ if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
+ result = WARNINGS
+ else:
+ return FAILURE
+
+ # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
+ # This means that an exception was raised when running run-webkit-tests and
+ # was never handled.
+ if cmd.rc == 254:
+ return RETRY
+ if cmd.rc != 0:
+ return FAILURE
+
+ return result
+
def evaluateCommand(self, cmd):
- rc = super(RunWebKitTests, self).evaluateCommand(cmd)
- if rc == SUCCESS:
+ rc = self.evaluateResult(cmd)
+ if rc == SUCCESS or rc == WARNINGS:
message = 'Passed layout tests'
self.descriptionDone = message
self.build.results = SUCCESS
@@ -866,6 +937,15 @@
self.build.addStepsAfterCurrentStep([ArchiveTestResults(), UploadTestResults(), ExtractTestResults(), ReRunWebKitTests()])
return rc
+ def getResultSummary(self):
+ status = self.name
+
+ if self.results != SUCCESS and self.incorrectLayoutLines:
+ status = u' '.join(self.incorrectLayoutLines)
+ return {u'step': status}
+
+ return super(RunWebKitTests, self).getResultSummary()
+
class ReRunWebKitTests(RunWebKitTests):
name = 're-run-layout-tests'
diff --git a/Tools/BuildSlaveSupport/ews-build/steps_unittest.py b/Tools/BuildSlaveSupport/ews-build/steps_unittest.py
index aa77223..e9f6d89 100644
--- a/Tools/BuildSlaveSupport/ews-build/steps_unittest.py
+++ b/Tools/BuildSlaveSupport/ews-build/steps_unittest.py
@@ -975,6 +975,39 @@
self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
return self.runStep()
+ def test_warnings(self):
+ self.setupStep(RunWebKitTests())
+ self.setProperty('fullPlatform', 'ios-simulator')
+ self.setProperty('configuration', 'release')
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logfiles={'json': self.jsonFileName},
+ command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
+ )
+ + 0
+ + ExpectShell.log('stdio', stdout='''Unexpected flakiness: timeouts (2)
+ imported/blink/storage/indexeddb/blob-valid-before-commit.html [ Timeout Pass ]
+ storage/indexeddb/modern/deleteindex-2.html [ Timeout Pass ]'''),
+ )
+ self.expectOutcome(result=WARNINGS, state_string='2 flakes')
+ return self.runStep()
+
+ def test_unexpected_error(self):
+ self.setupStep(RunWebKitTests())
+ self.setProperty('fullPlatform', 'mac-highsierra')
+ self.setProperty('configuration', 'debug')
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logfiles={'json': self.jsonFileName},
+ command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--debug', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
+ )
+ + ExpectShell.log('stdio', stdout='Unexpected error.')
+ + 254,
+ )
+ self.expectOutcome(result=RETRY, state_string='layout-tests (retry)')
+ return self.runStep()
+
+
def test_failure(self):
self.setupStep(RunWebKitTests())
self.setProperty('fullPlatform', 'ios-simulator')
diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index 728c0fa..b9767cf 100644
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,19 @@
+2019-07-15 Aakash Jain <aakash_jain@apple.com>
+
+ [ews-build] Parse and display layout test failures
+ https://bugs.webkit.org/show_bug.cgi?id=199709
+
+ Rubber-stamped by Jonathan Bedard.
+
+ * BuildSlaveSupport/ews-build/steps.py:
+ (RunWebKitTests.start): Initialize log_observer.
+ (RunWebKitTests._strip_python_logging_prefix): Copied from similar code in build.webkit.org buildbot config.
+ (RunWebKitTests._parseRunWebKitTestsOutput): Ditto.
+ (RunWebKitTests.commandComplete): Gather and parse the stdout and stderr logs.
+ (RunWebKitTests.evaluateResult): Analyze the results and decide build status.
+ (RunWebKitTests.getResultSummary): Update build and step summary.
+ * BuildSlaveSupport/ews-build/steps_unittest.py: Added and updated unit-tests.
+
2019-07-15 Commit Queue <commit-queue@webkit.org>
Unreviewed, rolling out r247393.
