# Copyright (C) 2012 Google Inc. All rights reserved.
# Copyright (C) 2020 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import logging
import unittest

from webkitpy.common.host_mock import MockHost
from webkitpy.port.driver import DriverOutput
from webkitpy.port.test import TestPort
from webkitpy.performance_tests.perftest import PerfTest
from webkitpy.performance_tests.perftest import PerfTestMetric
from webkitpy.performance_tests.perftest import PerfTestFactory
from webkitpy.performance_tests.perftest import SingleProcessPerfTest

from webkitcorepy import OutputCapture


class MockPort(TestPort):
    def __init__(self, custom_run_test=None):
        super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test)


class TestPerfTestMetric(unittest.TestCase):
    def test_init_set_missing_unit(self):
        self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', iterations=[1, 2, 3, 4, 5]).unit(), 'ms')
        self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Malloc', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes')
        self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'JSHeap', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes')

    def test_init_set_time_metric(self):
        self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', 'ms').name(), 'Time')
        self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', 'fps').name(), 'FrameRate')
        self.assertEqual(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', 'runs/s').name(), 'Runs')

    def test_has_values(self):
        self.assertFalse(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time').has_values())
        self.assertTrue(PerfTestMetric(['some', 'test'], 'some/test.html', 'Time', iterations=[1]).has_values())

    def test_append(self):
        metric = PerfTestMetric(['some', 'test'], 'some/test.html', 'Time')
        metric2 = PerfTestMetric(['some', 'test'], 'some/test.html', 'Time')
        self.assertFalse(metric.has_values())
        self.assertFalse(metric2.has_values())

        metric.append_group([1])
        self.assertTrue(metric.has_values())
        self.assertFalse(metric2.has_values())
        self.assertEqual(metric.grouped_iteration_values(), [[1]])
        self.assertEqual(metric.flattened_iteration_values(), [1])

        metric.append_group([2])
        self.assertEqual(metric.grouped_iteration_values(), [[1], [2]])
        self.assertEqual(metric.flattened_iteration_values(), [1, 2])

        metric2.append_group([3])
        self.assertTrue(metric2.has_values())
        self.assertEqual(metric.flattened_iteration_values(), [1, 2])
        self.assertEqual(metric2.flattened_iteration_values(), [3])

        metric.append_group([4, 5])
        self.assertEqual(metric.grouped_iteration_values(), [[1], [2], [4, 5]])
        self.assertEqual(metric.flattened_iteration_values(), [1, 2, 4, 5])


class TestPerfTest(unittest.TestCase):
    def _assert_results_are_correct(self, test, output):
        test.run_single = lambda driver, path, time_out_ms: output
        self.assertTrue(test.run(10))
        subtests = test._metrics
        self.assertEqual(list(map(lambda test: test['name'], subtests)), [None])
        metrics = subtests[0]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), metrics)), ['Time'])
        self.assertEqual(metrics[0].flattened_iteration_values(), [1080, 1120, 1095, 1101, 1104] * 4)

    def test_parse_output(self):
        output = DriverOutput("""
:Time -> [1080, 1120, 1095, 1101, 1104] ms
""", image=None, image_hash=None, audio=None)
        with OutputCapture(level=logging.INFO) as captured:
            test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
            self._assert_results_are_correct(test, output)

        self.assertEqual(captured.stdout.getvalue(), '')
        self.assertEqual(captured.stderr.getvalue(), '')
        self.assertEqual(captured.root.log.getvalue(), """RESULT some-test: Time= 1100.0 ms
median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms
""")

    def test_parse_output_with_ignored_stdout(self):
        output = DriverOutput("""
main frame - has 1 onunload handler(s)
:Time -> [1080, 1120, 1095, 1101, 1104] ms
""", image=None, image_hash=None, audio=None)
        with OutputCapture(level=logging.INFO) as captured:
            test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
            self._assert_results_are_correct(test, output)

        self.assertEqual(captured.stdout.getvalue(), '')
        self.assertEqual(captured.stderr.getvalue(), '')
        self.assertEqual(captured.root.log.getvalue(), """RESULT some-test: Time= 1100.0 ms
median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms
""")

    def test_parse_output_with_ignored_stderr(self):
        output = DriverOutput(":Time -> [1080, 1120, 1095, 1101, 1104] ms", image=None, image_hash=None, audio=None, error="""
Jan 22 14:09:24  WebKitTestRunner[1296] <Error>: CGContextSetFillColorWithColor: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable.
Jan 22 14:09:24  WebKitTestRunner[1296] <Error>: CGContextSetStrokeColorWithColor: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable.
Jan 22 14:09:24  WebKitTestRunner[1296] <Error>: CGContextGetCompositeOperation: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable.
Jan 22 14:09:24  WebKitTestRunner[1296] <Error>: CGContextSetCompositeOperation: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable.
Jan 22 14:09:24  WebKitTestRunner[1296] <Error>: CGContextFillRects: invalid context 0x0. If you want to see the backtrace, please set CG_CONTEXT_SHOW_BACKTRACE environmental variable.
""")

        class MockPortWithSierraName(MockPort):
            def name(self):
                return "mac-sierra"

        with OutputCapture(level=logging.INFO) as captured:
            test = PerfTest(MockPortWithSierraName(), 'some-test', '/path/some-dir/some-test')
            self._assert_results_are_correct(test, output)

        self.assertEqual(captured.stdout.getvalue(), '')
        self.assertEqual(captured.stderr.getvalue(), '')
        self.assertEqual(captured.root.log.getvalue(), """RESULT some-test: Time= 1100.0 ms
median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms
""")

    def _assert_failed_on_line(self, output_text, expected_log):
        output = DriverOutput(output_text, image=None, image_hash=None, audio=None)
        with OutputCapture(level=logging.INFO) as captured:
            test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
            test.run_single = lambda driver, path, time_out_ms: output
            self.assertFalse(test._run_with_driver(None, None))

        self.assertEqual(captured.stdout.getvalue(), '')
        self.assertEqual(captured.stderr.getvalue(), '')
        self.assertEqual(captured.root.log.getvalue(), expected_log)

    def test_parse_output_with_running_five_times(self):
        self._assert_failed_on_line("""
Running 5 times
:Time -> [1080, 1120, 1095, 1101, 1104] ms
""", 'ERROR: Running 5 times\n')

    def test_parse_output_with_detailed_info(self):
        self._assert_failed_on_line("""
    1: 1080 ms
:Time -> [1080, 1120, 1095, 1101, 1104] ms
""", 'ERROR:     1: 1080 ms\n')

    def test_parse_output_with_statistics(self):
        self._assert_failed_on_line("""
:Time -> [1080, 1120, 1095, 1101, 1104] ms
    mean: 105 ms
""", 'ERROR:     mean: 105 ms\n')

    def test_parse_output_with_description(self):
        output = DriverOutput("""
Description: this is a test description.

:Time -> [1080, 1120, 1095, 1101, 1104] ms
""", image=None, image_hash=None, audio=None)
        test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
        self._assert_results_are_correct(test, output)
        self.assertEqual(test.description(), 'this is a test description.')

    def test_parse_output_with_subtests(self):
        output = DriverOutput("""
Description: this is a test description.
some test:Time -> [1, 2, 3, 4, 5] ms
some other test = else:Time -> [6, 7, 8, 9, 10] ms
some other test = else:Malloc -> [11, 12, 13, 14, 15] bytes
Array Construction, []:Time -> [11, 12, 13, 14, 15] ms
Concat String:Time -> [15163, 15304, 15386, 15608, 15622] ms
jQuery - addClass:Time -> [2785, 2815, 2826, 2841, 2861] ms
Dojo - div:only-child:Time -> [7825, 7910, 7950, 7958, 7970] ms
Dojo - div:nth-child(2n+1):Time -> [3620, 3623, 3633, 3641, 3658] ms
Dojo - div > div:Time -> [10158, 10172, 10180, 10183, 10231] ms
Dojo - div ~ div:Time -> [6673, 6675, 6714, 6848, 6902] ms

:Time -> [1080, 1120, 1095, 1101, 1104] ms
""", image=None, image_hash=None, audio=None)
        with OutputCapture(level=logging.INFO) as captured:
            test = PerfTest(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
            test.run_single = lambda driver, path, time_out_ms: output
            self.assertTrue(test.run(10))

        subtests = test._metrics
        self.assertEqual(list(map(lambda test: test['name'], subtests)), ['some test', 'some other test = else',
            'Array Construction, []', 'Concat String', 'jQuery - addClass', 'Dojo - div:only-child',
            'Dojo - div:nth-child(2n+1)', 'Dojo - div > div', 'Dojo - div ~ div', None])

        some_test_metrics = subtests[0]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time'])
        self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'some test'])
        self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1, 2, 3, 4, 5] * 4)

        some_other_test_metrics = subtests[1]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), some_other_test_metrics)), ['Time', 'Malloc'])
        self.assertEqual(some_other_test_metrics[0].path(), ['some-dir', 'some-test', 'some other test = else'])
        self.assertEqual(some_other_test_metrics[0].flattened_iteration_values(), [6, 7, 8, 9, 10] * 4)
        self.assertEqual(some_other_test_metrics[1].path(), ['some-dir', 'some-test', 'some other test = else'])
        self.assertEqual(some_other_test_metrics[1].flattened_iteration_values(), [11, 12, 13, 14, 15] * 4)

        main_metrics = subtests[len(subtests) - 1]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), main_metrics)), ['Time'])
        self.assertEqual(main_metrics[0].path(), ['some-dir', 'some-test'])
        self.assertEqual(main_metrics[0].flattened_iteration_values(), [1080, 1120, 1095, 1101, 1104] * 4)

        self.assertEqual(captured.stdout.getvalue(), '')
        self.assertEqual(captured.stderr.getvalue(), '')
        self.assertEqual(captured.root.log.getvalue(), """DESCRIPTION: this is a test description.
RESULT some-dir: some-test: Time= 1100.0 ms
median= 1101.0 ms, stdev= 13.3140211016 ms, min= 1080.0 ms, max= 1120.0 ms
""")

    def test_parse_output_with_subtests_and_total(self):
        output = DriverOutput("""
:Time:Total -> [2324, 2328, 2345, 2314, 2312] ms
EmberJS-TodoMVC:Time:Total -> [1462, 1473, 1490, 1465, 1458] ms
EmberJS-TodoMVC/a:Time -> [1, 2, 3, 4, 5] ms
BackboneJS-TodoMVC:Time -> [862, 855, 855, 849, 854] ms
""", image=None, image_hash=None, audio=None)
        with OutputCapture(level=logging.INFO) as captured:
            test = PerfTest(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
            test.run_single = lambda driver, path, time_out_ms: output
            self.assertTrue(test.run(10))

        subtests = test._metrics
        self.assertEqual(list(map(lambda test: test['name'], subtests)), [None, 'EmberJS-TodoMVC', 'EmberJS-TodoMVC/a', 'BackboneJS-TodoMVC'])

        main_metrics = subtests[0]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), main_metrics)), ['Time'])
        self.assertEqual(main_metrics[0].aggregator(), 'Total')
        self.assertEqual(main_metrics[0].path(), ['some-dir', 'some-test'])
        self.assertEqual(main_metrics[0].flattened_iteration_values(), [2324, 2328, 2345, 2314, 2312] * 4)

        some_test_metrics = subtests[1]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time'])
        self.assertEqual(some_test_metrics[0].aggregator(), 'Total')
        self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'EmberJS-TodoMVC'])
        self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1462, 1473, 1490, 1465, 1458] * 4)

        some_test_metrics = subtests[2]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time'])
        self.assertEqual(some_test_metrics[0].aggregator(), None)
        self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'EmberJS-TodoMVC', 'a'])
        self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1, 2, 3, 4, 5] * 4)

        some_test_metrics = subtests[3]['metrics']
        self.assertEqual(list(map(lambda metric: metric.name(), some_test_metrics)), ['Time'])
        self.assertEqual(some_test_metrics[0].aggregator(), None)
        self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'BackboneJS-TodoMVC'])
        self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [862, 855, 855, 849, 854] * 4)

        self.assertEqual(captured.stdout.getvalue(), '')
        self.assertEqual(captured.stderr.getvalue(), '')
        self.assertEqual(captured.root.log.getvalue(), """RESULT some-dir: some-test: Time= 2324.6 ms
median= 2324.0 ms, stdev= 12.1326007105 ms, min= 2312.0 ms, max= 2345.0 ms
""")


class TestSingleProcessPerfTest(unittest.TestCase):
    def test_use_only_one_process(self):
        called = [0]

        def run_single(driver, path, time_out_ms):
            called[0] += 1
            return DriverOutput("""
Description: this is a test description.
:Time -> [1080, 1120, 1095, 1101, 1104] ms
""", image=None, image_hash=None, audio=None)

        test = SingleProcessPerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
        test.run_single = run_single
        self.assertTrue(test.run(0))
        self.assertEqual(called[0], 1)


class TestPerfTestFactory(unittest.TestCase):
    def test_regular_test(self):
        test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
        self.assertEqual(test.__class__, PerfTest)