# Copyright (C) 2018 Igalia S.L.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import unittest

import json
import os

from webkitpy.common.host_mock import MockHost
from webkitpy.common.test_expectations import TestExpectations
from webkitpy.common.webkit_finder import WebKitFinder


class MockTestExpectations(TestExpectations):

    def __init__(self, port, expectations, build_type='Release'):
        host = MockHost()
        port = host.port_factory.get(port)
        self._port_name = port.name()
        self._build_type = build_type
        self._expectations = json.loads(expectations)

    def is_skip(self, test, subtest):
        if test in self.skipped_tests():
            return True
        return subtest in self.skipped_subtests(test)


class ExpectationsTest(unittest.TestCase):

    BASIC = """
{
    "imported/w3c/webdriver/tests/test1.py": {
        "subtests": {
            "test1_one": {
                "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
            },
            "test1_two": {
                "expected": {
                    "all": {"status": ["FAIL"], "bug": "1234"},
                    "gtk": {"status": ["PASS"]}
                }
            }
        }
    },
    "imported/w3c/webdriver/tests/test2.py": {
        "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
    },
    "imported/w3c/webdriver/tests/test3.py": {
        "expected": {"all": {"status": ["FAIL"], "bug": "1234"}},
        "subtests": {
            "test3_one": {
                "expected": {"all": {"status": ["PASS"]}}
            }
        }
    }
}"""

    SKIP = """
{
    "imported/w3c/webdriver/tests/test1.py": {
        "expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
    },
    "imported/w3c/webdriver/tests/test2.py": {
        "expected": {"gtk": {"status": ["SKIP"], "bug": "1234"}}
    },
    "imported/w3c/webdriver/tests/test3.py": {
        "expected": {"all": {"status": ["SKIP"], "bug": "1234"}},
        "subtests": {
            "test3_one": {
                "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
            }
        }
    },
    "imported/w3c/webdriver/tests/test4.py": {
        "subtests": {
            "test4_one": {
                "expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
            }
        }
    }
}"""

    FLAKY = """
{
    "TestCookieManager": {
        "subtests": {
            "/webkit2/WebKitCookieManager/persistent-storage": {
                "expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "1234"}}
            }
        }
    },
    "TestWebKit": {
        "subtests": {
            "WebKit.MouseMoveAfterCrash": {
                "expected": {"all": {"status": ["CRASH", "PASS"], "bug": "1234"}}
            },
            "WebKit.WKConnection": {
                "expected": {"wpe": {"status": ["FAIL", "TIMEOUT"], "bug": "1234"}}
            }
        }
    }
}"""

    BUILD_TYPE = """
{
    "TestCookieManager": {
        "subtests": {
            "/webkit2/WebKitCookieManager/persistent-storage": {
                "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
            }
        }
    },
    "TestWebKit": {
        "subtests": {
            "WebKit.MouseMoveAfterCrash": {
                "expected": {"all@Debug": {"status": ["CRASH"], "bug": "1234"}}
            },
            "WebKit.WKConnection": {
                "expected": {"gtk@Release": {"status": ["FAIL"], "bug": "1234"},
                             "gtk@Debug": {"status": ["CRASH"], "bug": "1234"}}
            }
        }
    },
    "TestWebCore": {
        "expected": {"all@Debug": {"status": ["CRASH"]}},
        "subtests": {
            "ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL": {
                "expected": {"all": {"status": ["PASS"], "bug": "1234"}}
            },
            "FileMonitorTest.DetectChange": {
                "expected": {"all@Release": {"status": ["FAIL"], "bug": "1234"}}
            }
        }
    },
    "TestWebViewEditor": {
        "expected": {"all@Release": {"status": ["SKIP"]},
                     "wpe@Debug": {"status": ["SKIP"]}},
        "subtests": {
            "/webkit2/WebKitWebView/editable/editable": {
                "expected": {"gtk": {"status": ["FAIL"], "bug": "1234"}}
            }
        }
    }
}"""

    SLOW = """
{
    "TestCookieManager": {
        "expected": {"all": {"slow": true}},
        "subtests": {
            "/webkit2/WebKitCookieManager/persistent-storage": {
                "expected": {"wpe": {"status": ["FAIL"], "slow": false, "bug": "1234"}}
            }
        }
    },
    "TestWebKit": {
        "subtests": {
            "WebKit.MouseMoveAfterCrash": {
                "expected": {"all": {"status": ["FAIL"], "slow": true, "bug": "1234"}}
            },
            "WebKit.WKConnection": {
                "expected": {"gtk": {"status": ["CRASH"], "bug": "1234"}}
            }
        }
    }
}"""

    def assert_exp(self, test, subtest, result):
        self.assertIn(result, self.expectations.get_expectation(test, subtest))

    def assert_not_exp(self, test, subtest, result):
        self.assertNotIn(result, self.expectations.get_expectation(test, subtest))

    def assert_bad_exp(self, test):
        self.assertRaises(AssertionError, self.expectations.get_expectation(test))

    def assert_skip(self, test, subtest, result):
        self.assertEqual(self.expectations.is_skip(test, subtest), result)

    def assert_slow(self, test, subtest, result):
        self.assertEqual(self.expectations.is_slow(test, subtest), result)

    def test_basic(self):
        self.expectations = MockTestExpectations('gtk', self.BASIC)
        self.assert_exp('imported/w3c/webdriver/tests/test5.py', 'test5_two', 'PASS')

        self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_five', 'PASS')
        self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_one', 'FAIL')
        self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'PASS')

        self.assert_exp('imported/w3c/webdriver/tests/test2.py', 'test2_one', 'FAIL')

        self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_two', 'FAIL')
        self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_one', 'PASS')

        self.expectations = MockTestExpectations('wpe', self.BASIC)
        self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'FAIL')

    def test_skip(self):
        self.expectations = MockTestExpectations('gtk', self.SKIP)
        self.assert_skip('imported/w3c/webdriver/tests/test5.py', None, False)
        self.assert_skip('imported/w3c/webdriver/tests/test1.py', None, True)
        self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, True)
        self.assert_skip('imported/w3c/webdriver/tests/test3.py', None, True)
        self.assert_skip('imported/w3c/webdriver/tests/test3.py', 'test3_one', True)
        self.assert_skip('imported/w3c/webdriver/tests/test4.py', None, False)
        self.assert_skip('imported/w3c/webdriver/tests/test4.py', 'test4_one', True)
        self.assert_exp('imported/w3c/webdriver/tests/test4.py', 'test4_one', 'SKIP')

        self.expectations = MockTestExpectations('wpe', self.SKIP)
        self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, False)

    def test_flaky(self):
        self.expectations = MockTestExpectations('gtk', self.FLAKY)
        self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
        self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
        self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
        self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
        self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'PASS')
        self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
        self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')

        self.expectations = MockTestExpectations('wpe', self.FLAKY)
        self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
        self.assert_not_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
        self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
        self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
        self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
        self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')

    def test_build_type(self):
        self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Debug')
        self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
        self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
        self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'CRASH')
        self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
        self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'CRASH')
        self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'CRASH')
        self.assert_skip('TestWebViewEditor', None, False)
        self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', False)
        self.assert_exp('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', 'FAIL')

        self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Release')
        self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
        self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
        self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
        self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
        self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'FAIL')
        self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'PASS')
        self.assert_skip('TestWebViewEditor', None, True)
        self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)

        self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Release')
        self.assert_skip('TestWebViewEditor', None, True)
        self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)

        self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Debug')
        self.assert_skip('TestWebViewEditor', None, True)
        self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)

    def test_slow(self):
        self.expectations = MockTestExpectations('gtk', self.SLOW)
        self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/basic', True)
        self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', True)
        self.assert_slow('TestWebKit', 'WebKit.WKView', False)
        self.assert_slow('TestWebKit', 'WebKit.MouseMoveAfterCrash', True)
        self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'FAIL')
        self.assert_slow('TestWebKit', 'WebKit.WKConnection', False)

        self.expectations = MockTestExpectations('wpe', self.SLOW)
        self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/basic', True)
        self.assert_slow('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', False)
        self.assert_slow('TestWebKit', 'WebKit.WKView', False)
        self.assert_slow('TestWebKit', 'WebKit.MouseMoveAfterCrash', True)
        self.assert_slow('TestWebKit', 'WebKit.WKConnection', False)