| # Copyright (C) 2012 Google Inc. All rights reserved. |
| # Copyright (C) 2020 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import logging |
| import unittest |
| |
| from webkitpy.common.host_mock import MockHost |
| from webkitpy.port.driver import DriverOutput |
| from webkitpy.port.test import TestPort |
| from webkitpy.performance_tests.perftest import PerfTest |
| from webkitpy.performance_tests.perftest import PerfTestMetric |
| from webkitpy.performance_tests.perftest import PerfTestFactory |
| from webkitpy.performance_tests.perftest import SingleProcessPerfTest |
| |
| from webkitcorepy import OutputCapture |
| |
| |
| class MockPort(TestPort): |
| def __init__(self, custom_run_test=None): |
| super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test) |
| |
| |
| class TestPerfTestMetric(unittest.TestCase): |
| def test_init_set_missing_unit(self): |
| self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', iterations=[1, 2, 3, 4, 5]).unit(), 'ms') |
| self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Malloc', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes') |
| self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'JSHeap', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes') |
| |
| def test_init_set_time_metric(self): |
| self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', 'ms').name(), 'Time') |
| self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', 'fps').name(), 'FrameRate') |
| self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', 'runs/s').name(), 'Runs') |
| |
| def test_has_values(self): |
| self.assertFalse(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time').has_values()) |
| self.assertTrue(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', iterations=[1]).has_values()) |
| |
| def test_append(self): |
| metric = PerfTestMetric(['some', 'test'], 'some/test.html', 'Time') |
| metric2 = PerfTestMetric(['some', 'test'], 'some/test.html', 'Time') |
| self.assertFalse(metric.has_values()) |
| self.assertFalse(metric2.has_values()) |
| |
| metric.append_group([1]) |
| self.assertTrue(metric.has_values()) |
| self.assertFalse(metric2.has_values()) |
| self.assertEqual(metric.grouped_iteration_values(), [[1]]) |
| self.assertEqual(metric.flattened_iteration_values(), [1]) |
| |
| metric.append_group([2]) |
| self.assertEqual(metric.grouped_iteration_values(), [[1], [2]]) |
| self.assertEqual(metric.flattened_iteration_values(), [1, 2]) |
| |
| metric2.append_group([3]) |
| self.assertTrue(metric2.has_values()) |
| self.assertEqual(metric.flattened_iteration_values(), [1, 2]) |
| self.assertEqual(metric2.flattened_iteration_values(), [3]) |
| |
| metric.append_group([4, 5]) |
| self.assertEqual(metric.grouped_iteration_values(), [[1], [2], [4, 5]]) |
| self.assertEqual(metric.flattened_iteration_values(), [1, 2, 4, 5]) |
| |
| |
| class TestPerfTest(unittest.TestCase): |
| def _assert_results_are_correct(self, test, output): |
| test.run_single = lambda driver, path, time_out_ms: output |
| self.assertTrue(test.run(10)) |
| subtests = test._metrics |
| self.assertEqual(list(map(lambda test: test['name'], subtests)), [None]) |
| metrics = subtests[0]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), metrics)), ['Time']) |
| self.assertEqual(metrics[0].flattened_iteration_values(), [1080, 1120, 1095, 1101, 1104] * 4) |
| |
| def test_parse_output(self): |
| output = DriverOutput(""" |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| """, image=None, image_hash=None, audio=None) |
| with OutputCapture(level=logging.INFO) as captured: |
| test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') |
| self._assert_results_are_correct(test, output) |
| |
| self.assertEqual(captured.stdout.getvalue(), '') |
| self.assertEqual(captured.stderr.getvalue(), '') |
| self.assertEqual(captured.root.log.getvalue(), """RESULT some-test: Time= 1100.0 ms |
| median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms |
| """) |
| |
| def test_parse_output_with_ignored_stdout(self): |
| output = DriverOutput(""" |
| main frame - has 1 onunload handler(s) |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| """, image=None, image_hash=None, audio=None) |
| with OutputCapture(level=logging.INFO) as captured: |
| test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') |
| self._assert_results_are_correct(test, output) |
| |
| self.assertEqual(captured.stdout.getvalue(), '') |
| self.assertEqual(captured.stderr.getvalue(), '') |
| self.assertEqual(captured.root.log.getvalue(), """RESULT some-test: Time= 1100.0 ms |
| median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms |
| """) |
| |
| def test_parse_output_with_ignored_stderr(self): |
| output = DriverOutput(":Time -> [1080, 1120, 1095, 1101, 1104] ms", image=None, image_hash=None, audio=None, error=""" |
| Jan 22 14:09:24 WebKitTestRunner[1296] <Error>: CGContextSetFillColorWithColor: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable. |
| Jan 22 14:09:24 WebKitTestRunner[1296] <Error>: CGContextSetStrokeColorWithColor: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable. |
| Jan 22 14:09:24 WebKitTestRunner[1296] <Error>: CGContextGetCompositeOperation: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable. |
| Jan 22 14:09:24 WebKitTestRunner[1296] <Error>: CGContextSetCompositeOperation: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable. |
| Jan 22 14:09:24 WebKitTestRunner[1296] <Error>: CGContextFillRects: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable. |
| """) |
| |
| class MockPortWithSierraName(MockPort): |
| def name(self): |
| return "mac-sierra" |
| |
| with OutputCapture(level=logging.INFO) as captured: |
| test = PerfTest(MockPortWithSierraName(), 'some-test', '/path/some-dir/some-test') |
| self._assert_results_are_correct(test, output) |
| |
| self.assertEqual(captured.stdout.getvalue(), '') |
| self.assertEqual(captured.stderr.getvalue(), '') |
| self.assertEqual(captured.root.log.getvalue(), """RESULT some-test: Time= 1100.0 ms |
| median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms |
| """) |
| |
| def _assert_failed_on_line(self, output_text, expected_log): |
| output = DriverOutput(output_text, image=None, image_hash=None, audio=None) |
| with OutputCapture(level=logging.INFO) as captured: |
| test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') |
| test.run_single = lambda driver, path, time_out_ms: output |
| self.assertFalse(test._run_with_driver(None, None)) |
| |
| self.assertEqual(captured.stdout.getvalue(), '') |
| self.assertEqual(captured.stderr.getvalue(), '') |
| self.assertEqual(captured.root.log.getvalue(), expected_log) |
| |
| def test_parse_output_with_running_five_times(self): |
| self._assert_failed_on_line(""" |
| Running 5 times |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| """, 'ERROR: Running 5 times\n') |
| |
| def test_parse_output_with_detailed_info(self): |
| self._assert_failed_on_line(""" |
| 1: 1080 ms |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| """, 'ERROR: 1: 1080 ms\n') |
| |
| def test_parse_output_with_statistics(self): |
| self._assert_failed_on_line(""" |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| mean: 105 ms |
| """, 'ERROR: mean: 105 ms\n') |
| |
| def test_parse_output_with_description(self): |
| output = DriverOutput(""" |
| Description: this is a test description. |
| |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| """, image=None, image_hash=None, audio=None) |
| test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') |
| self._assert_results_are_correct(test, output) |
| self.assertEqual(test.description(), 'this is a test description.') |
| |
| def test_parse_output_with_subtests(self): |
| output = DriverOutput(""" |
| Description: this is a test description. |
| some test:Time -> [1, 2, 3, 4, 5] ms |
| some other test = else:Time -> [6, 7, 8, 9, 10] ms |
| some other test = else:Malloc -> [11, 12, 13, 14, 15] bytes |
| Array Construction, []:Time -> [11, 12, 13, 14, 15] ms |
| Concat String:Time -> [15163, 15304, 15386, 15608, 15622] ms |
| jQuery - addClass:Time -> [2785, 2815, 2826, 2841, 2861] ms |
| Dojo - div:only-child:Time -> [7825, 7910, 7950, 7958, 7970] ms |
| Dojo - div:nth-child(2n+1):Time -> [3620, 3623, 3633, 3641, 3658] ms |
| Dojo - div > div:Time -> [10158, 10172, 10180, 10183, 10231] ms |
| Dojo - div ~ div:Time -> [6673, 6675, 6714, 6848, 6902] ms |
| |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| """, image=None, image_hash=None, audio=None) |
| with OutputCapture(level=logging.INFO) as captured: |
| test = PerfTest(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test') |
| test.run_single = lambda driver, path, time_out_ms: output |
| self.assertTrue(test.run(10)) |
| |
| subtests = test._metrics |
| self.assertEqual(list(map(lambda test: test['name'], subtests)), ['some test', 'some other test = else', |
| 'Array Construction, []', 'Concat String', 'jQuery - addClass', 'Dojo - div:only-child', |
| 'Dojo - div:nth-child(2n+1)', 'Dojo - div > div', 'Dojo - div ~ div', None]) |
| |
| some_test_metrics = subtests[0]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time']) |
| self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'some test']) |
| self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1, 2, 3, 4, 5] * 4) |
| |
| some_other_test_metrics = subtests[1]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), some_other_test_metrics)), ['Time', 'Malloc']) |
| self.assertEqual(some_other_test_metrics[0].path(), ['some-dir', 'some-test', 'some other test = else']) |
| self.assertEqual(some_other_test_metrics[0].flattened_iteration_values(), [6, 7, 8, 9, 10] * 4) |
| self.assertEqual(some_other_test_metrics[1].path(), ['some-dir', 'some-test', 'some other test = else']) |
| self.assertEqual(some_other_test_metrics[1].flattened_iteration_values(), [11, 12, 13, 14, 15] * 4) |
| |
| main_metrics = subtests[len(subtests) - 1]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), main_metrics)), ['Time']) |
| self.assertEqual(main_metrics[0].path(), ['some-dir', 'some-test']) |
| self.assertEqual(main_metrics[0].flattened_iteration_values(), [1080, 1120, 1095, 1101, 1104] * 4) |
| |
| self.assertEqual(captured.stdout.getvalue(), '') |
| self.assertEqual(captured.stderr.getvalue(), '') |
| self.assertEqual(captured.root.log.getvalue(), """DESCRIPTION: this is a test description. |
| RESULT some-dir: some-test: Time= 1100.0 ms |
| median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms |
| """) |
| |
| def test_parse_output_with_subtests_and_total(self): |
| output = DriverOutput(""" |
| :Time:Total -> [2324, 2328, 2345, 2314, 2312] ms |
| EmberJS-TodoMVC:Time:Total -> [1462, 1473, 1490, 1465, 1458] ms |
| EmberJS-TodoMVC/a:Time -> [1, 2, 3, 4, 5] ms |
| BackboneJS-TodoMVC:Time -> [862, 855, 855, 849, 854] ms |
| """, image=None, image_hash=None, audio=None) |
| with OutputCapture(level=logging.INFO) as captured: |
| test = PerfTest(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test') |
| test.run_single = lambda driver, path, time_out_ms: output |
| self.assertTrue(test.run(10)) |
| |
| subtests = test._metrics |
| self.assertEqual(list(map(lambda test: test['name'], subtests)), [None, 'EmberJS-TodoMVC', 'EmberJS-TodoMVC/a', 'BackboneJS-TodoMVC']) |
| |
| main_metrics = subtests[0]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), main_metrics)), ['Time']) |
| self.assertEqual(main_metrics[0].aggregator(), 'Total') |
| self.assertEqual(main_metrics[0].path(), ['some-dir', 'some-test']) |
| self.assertEqual(main_metrics[0].flattened_iteration_values(), [2324, 2328, 2345, 2314, 2312] * 4) |
| |
| some_test_metrics = subtests[1]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time']) |
| self.assertEqual(some_test_metrics[0].aggregator(), 'Total') |
| self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'EmberJS-TodoMVC']) |
| self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1462, 1473, 1490, 1465, 1458] * 4) |
| |
| some_test_metrics = subtests[2]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time']) |
| self.assertEqual(some_test_metrics[0].aggregator(), None) |
| self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'EmberJS-TodoMVC', 'a']) |
| self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1, 2, 3, 4, 5] * 4) |
| |
| some_test_metrics = subtests[3]['metrics'] |
| self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time']) |
| self.assertEqual(some_test_metrics[0].aggregator(), None) |
| self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'BackboneJS-TodoMVC']) |
| self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [862, 855, 855, 849, 854] * 4) |
| |
| self.assertEqual(captured.stdout.getvalue(), '') |
| self.assertEqual(captured.stderr.getvalue(), '') |
| self.assertEqual(captured.root.log.getvalue(), """RESULT some-dir: some-test: Time= 2324.6 ms |
| median= 2324.0 ms, stdev= 12.1326007105 ms, min= 2312.0 ms, max= 2345.0 ms |
| """) |
| |
| |
| class TestSingleProcessPerfTest(unittest.TestCase): |
| def test_use_only_one_process(self): |
| called = [0] |
| |
| def run_single(driver, path, time_out_ms): |
| called[0] += 1 |
| return DriverOutput(""" |
| Description: this is a test description. |
| :Time -> [1080, 1120, 1095, 1101, 1104] ms |
| """, image=None, image_hash=None, audio=None) |
| |
| test = SingleProcessPerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') |
| test.run_single = run_single |
| self.assertTrue(test.run(0)) |
| self.assertEqual(called[0], 1) |
| |
| |
| class TestPerfTestFactory(unittest.TestCase): |
| def test_regular_test(self): |
| test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test') |
| self.assertEqual(test.__class__, PerfTest) |
