# Copyright (C) 2012 Google Inc. All rights reserved.
# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import unittest

from webkitpy.common.host_mock import MockHost
from webkitpy.common.system.systemhost_mock import MockSystemHost
from webkitpy.layout_tests import run_webkit_tests
from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
from webkitpy.layout_tests.models import test_expectations
from webkitpy.layout_tests.models import test_failures
from webkitpy.layout_tests.models.test_run_results import TestRunResults
from webkitpy.layout_tests.models.test_input import TestInput
from webkitpy.layout_tests.models.test_results import TestResult
from webkitpy.port.test import TestPort


TestExpectations = test_expectations.TestExpectations


class FakePrinter(object):
    num_started = 0
    num_tests = 0

    def print_expected(self, run_results, get_tests_with_result_type):
        pass

    def print_workers_and_shards(self, num_workers, num_shards):
        pass

    def print_started_test(self, test_name):
        pass

    def print_finished_test(self, result, expected, exp_str, got_str):
        pass

    def write(self, msg):
        pass

    def write_update(self, msg):
        pass

    def flush(self):
        pass


class LayoutTestRunnerTests(unittest.TestCase):
    def _runner(self, port=None):
        # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
        options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
        options.child_processes = '1'

        host = MockHost()
        port = port or host.port_factory.get(options.platform, options=options)
        return LayoutTestRunner(options, port, FakePrinter(), port.results_directory(), lambda test_name: False)

    def _run_tests(self, runner, tests):
        test_inputs = [TestInput(test, 6000) for test in tests]
        expectations = TestExpectations(runner._port, tests)
        expectations.parse_all_expectations()
        runner.run_tests(expectations, test_inputs, set(),
            num_workers=1, needs_http=any('http' in test for test in tests), needs_websockets=any(['websocket' in test for test in tests]), needs_web_platform_test_server=any(['imported/w3c' in test for test in tests]), retrying=False)

    def test_interrupt_if_at_failure_limits(self):
        runner = self._runner()
        runner._options.exit_after_n_failures = None
        runner._options.exit_after_n_crashes_or_times = None
        test_names = ['passes/text.html', 'passes/image.html']
        runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names]

        expectations = TestExpectations(runner._port, test_names)
        expectations.parse_all_expectations()
        run_results = TestRunResults(expectations, len(test_names))
        run_results.unexpected_failures = 100
        run_results.unexpected_crashes = 50
        run_results.unexpected_timeouts = 50
        # No exception when the exit_after* options are None.
        runner._interrupt_if_at_failure_limits(run_results)

        # No exception when we haven't hit the limit yet.
        runner._options.exit_after_n_failures = 101
        runner._options.exit_after_n_crashes_or_timeouts = 101
        runner._interrupt_if_at_failure_limits(run_results)

        # Interrupt if we've exceeded either limit:
        runner._options.exit_after_n_crashes_or_timeouts = 10
        self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
        self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP)
        self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP)

        runner._options.exit_after_n_crashes_or_timeouts = None
        runner._options.exit_after_n_failures = 10
        exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)

    def test_update_summary_with_result(self):
        # Reftests expected to be image mismatch should be respected when pixel_tests=False.
        runner = self._runner()
        runner._options.pixel_tests = False
        runner._options.world_leaks = False
        test = 'failures/expected/reftest.html'
        leak_test = 'failures/expected/leak.html'
        expectations = TestExpectations(runner._port, tests=[test, leak_test])
        expectations.parse_all_expectations()
        runner._expectations = expectations

        run_results = TestRunResults(expectations, 1)
        result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!='])
        runner._update_summary_with_result(run_results, result)
        self.assertEqual(1, run_results.expected)
        self.assertEqual(0, run_results.unexpected)

        run_results = TestRunResults(expectations, 1)
        result = TestResult(test_name=test, failures=[], reftest_type=['=='])
        runner._update_summary_with_result(run_results, result)
        self.assertEqual(0, run_results.expected)
        self.assertEqual(1, run_results.unexpected)

        run_results = TestRunResults(expectations, 1)
        result = TestResult(test_name=leak_test, failures=[])
        runner._update_summary_with_result(run_results, result)
        self.assertEqual(1, run_results.expected)
        self.assertEqual(0, run_results.unexpected)

    def test_servers_started(self):

        def start_http_server():
            self.http_started = True

        def start_websocket_server():
            self.websocket_started = True

        def start_web_platform_test_server():
            self.web_platform_test_server_started = True

        def stop_http_server():
            self.http_stopped = True

        def stop_websocket_server():
            self.websocket_stopped = True

        def stop_web_platform_test_server():
            self.web_platform_test_server_stopped = True

        def is_http_server_running():
            return self.http_started and not self.http_stopped

        def is_websocket_server_running():
            return self.websocket_started and not self.websocket_stopped

        def is_wpt_server_running():
            return self.websocket_started and not self.web_platform_test_server_stopped

        host = MockHost()
        port = host.port_factory.get('test-mac-leopard')
        port.start_http_server = start_http_server
        port.start_websocket_server = start_websocket_server
        port.start_web_platform_test_server = start_web_platform_test_server
        port.stop_http_server = stop_http_server
        port.stop_websocket_server = stop_websocket_server
        port.stop_web_platform_test_server = stop_web_platform_test_server
        port.is_http_server_running = is_http_server_running
        port.is_websocket_server_running = is_websocket_server_running
        port.is_wpt_server_running = is_wpt_server_running

        self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
        self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
        runner = self._runner(port=port)
        runner._needs_http = True
        runner._needs_websockets = False
        runner._needs_web_platform_test_server = False
        runner.start_servers()
        self.assertEqual(self.http_started, True)
        self.assertEqual(self.websocket_started, False)
        self.assertEqual(self.web_platform_test_server_started, False)
        runner.stop_servers()
        self.assertEqual(self.http_stopped, True)
        self.assertEqual(self.websocket_stopped, False)
        self.assertEqual(self.web_platform_test_server_stopped, False)

        self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
        self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
        runner._needs_http = True
        runner._needs_websockets = True
        runner._needs_web_platform_test_server = False
        runner.start_servers()
        self.assertEqual(self.http_started, True)
        self.assertEqual(self.websocket_started, True)
        self.assertEqual(self.web_platform_test_server_started, False)
        runner.stop_servers()
        self.assertEqual(self.http_stopped, True)
        self.assertEqual(self.websocket_stopped, True)
        self.assertEqual(self.web_platform_test_server_stopped, False)

        self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
        self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
        runner._needs_http = False
        runner._needs_websockets = False
        runner._needs_web_platform_test_server = True
        runner.start_servers()
        self.assertEqual(self.http_started, False)
        self.assertEqual(self.websocket_started, False)
        self.assertEqual(self.web_platform_test_server_started, True)
        runner.stop_servers()
        self.assertEqual(self.http_stopped, False)
        self.assertEqual(self.websocket_stopped, False)
        self.assertEqual(self.web_platform_test_server_stopped, True)

        self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
        self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
        runner._needs_http = False
        runner._needs_websockets = False
        runner._needs_web_platform_test_server = False
        runner.start_servers()
        self.assertEqual(self.http_started, False)
        self.assertEqual(self.websocket_started, False)
        self.assertEqual(self.web_platform_test_server_started, False)
        runner.stop_servers()
        self.assertEqual(self.http_stopped, False)
        self.assertEqual(self.websocket_stopped, False)
        self.assertEqual(self.web_platform_test_server_stopped, False)


class SharderTests(unittest.TestCase):

    test_list = [
        "http/tests/websocket/tests/unicode.htm",
        "animations/keyframes.html",
        "http/tests/security/view-source-no-refresh.html",
        "http/tests/websocket/tests/websocket-protocol-ignored.html",
        "fast/css/display-none-inline-style-change-crash.html",
        "http/tests/xmlhttprequest/supported-xml-content-types.html",
        "dom/html/level2/html/HTMLAnchorElement03.html",
        "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
        "dom/html/level2/html/HTMLAnchorElement06.html",
    ]

    def get_test_input(self, test_file):
        return TestInput(test_file, needs_servers=(test_file.startswith('http')))

    def get_shards(self, num_workers, fully_parallel, test_list=None):
        port = TestPort(MockSystemHost())
        self.sharder = Sharder(port.split_test)
        test_list = test_list or self.test_list
        return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)

    def assert_shards(self, actual_shards, expected_shard_names):
        self.assertEqual(len(actual_shards), len(expected_shard_names))
        for i, shard in enumerate(actual_shards):
            expected_shard_name, expected_test_names = expected_shard_names[i]
            self.assertEqual(shard.name, expected_shard_name)
            self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
                              expected_test_names)

    def test_shard_by_dir(self):
        result = self.get_shards(num_workers=2, fully_parallel=False)

        self.assert_shards(result,
            [('animations', ['animations/keyframes.html']),
             ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
                                      'dom/html/level2/html/HTMLAnchorElement06.html']),
             ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
             ('http/tests/security', ['http/tests/security/view-source-no-refresh.html']),
             ('http/tests/websocket/tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']),
             ('http/tests/xmlhttprequest', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
             ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])

    def test_shard_every_file(self):
        result = self.get_shards(num_workers=2, fully_parallel=True)
        self.assert_shards(result,
            [('.', ['http/tests/websocket/tests/unicode.htm']),
             ('.', ['animations/keyframes.html']),
             ('.', ['http/tests/security/view-source-no-refresh.html']),
             ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
             ('.', ['fast/css/display-none-inline-style-change-crash.html']),
             ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
             ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
             ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
             ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])