blob: 25c82b986e100ff6395d9336cbdedc92960e50d2 [file] [log] [blame]
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import json
import unittest
from webkitpy.common.net import resultsjsonparser_unittest
from webkitpy.common.host_mock import MockHost
from webkitpy.layout_tests.layout_package.json_results_generator import strip_json_wrapper
from webkitpy.port.base import Port
from webkitpy.tool.commands.rebaselineserver import TestConfig, RebaselineServer
from webkitpy.tool.servers import rebaselineserver
class RebaselineTestTest(unittest.TestCase):
def test_text_rebaseline_update(self):
self._assertRebaseline(
test_files=(
'fast/text-expected.txt',
'platform/mac/fast/text-expected.txt',
),
results_files=(
'fast/text-actual.txt',
),
test_name='fast/text.html',
baseline_target='mac',
baseline_move_to='none',
expected_success=True,
expected_log=[
'Rebaselining fast/text...',
' Updating baselines for mac',
' Updated text-expected.txt',
])
def test_text_rebaseline_new(self):
self._assertRebaseline(
test_files=(
'fast/text-expected.txt',
),
results_files=(
'fast/text-actual.txt',
),
test_name='fast/text.html',
baseline_target='mac',
baseline_move_to='none',
expected_success=True,
expected_log=[
'Rebaselining fast/text...',
' Updating baselines for mac',
' Updated text-expected.txt',
])
def test_text_rebaseline_move_no_op_1(self):
self._assertRebaseline(
test_files=(
'fast/text-expected.txt',
'platform/win/fast/text-expected.txt',
),
results_files=(
'fast/text-actual.txt',
),
test_name='fast/text.html',
baseline_target='mac',
baseline_move_to='mac-leopard',
expected_success=True,
expected_log=[
'Rebaselining fast/text...',
' Updating baselines for mac',
' Updated text-expected.txt',
])
def test_text_rebaseline_move_no_op_2(self):
self._assertRebaseline(
test_files=(
'fast/text-expected.txt',
'platform/mac/fast/text-expected.checksum',
),
results_files=(
'fast/text-actual.txt',
),
test_name='fast/text.html',
baseline_target='mac',
baseline_move_to='mac-leopard',
expected_success=True,
expected_log=[
'Rebaselining fast/text...',
' Moving current mac baselines to mac-leopard',
' No current baselines to move',
' Updating baselines for mac',
' Updated text-expected.txt',
])
def test_text_rebaseline_move(self):
self._assertRebaseline(
test_files=(
'fast/text-expected.txt',
'platform/mac/fast/text-expected.txt',
),
results_files=(
'fast/text-actual.txt',
),
test_name='fast/text.html',
baseline_target='mac',
baseline_move_to='mac-leopard',
expected_success=True,
expected_log=[
'Rebaselining fast/text...',
' Moving current mac baselines to mac-leopard',
' Moved text-expected.txt',
' Updating baselines for mac',
' Updated text-expected.txt',
])
def test_text_rebaseline_move_only_images(self):
self._assertRebaseline(
test_files=(
'fast/image-expected.txt',
'platform/mac/fast/image-expected.txt',
'platform/mac/fast/image-expected.png',
'platform/mac/fast/image-expected.checksum',
),
results_files=(
'fast/image-actual.png',
'fast/image-actual.checksum',
),
test_name='fast/image.html',
baseline_target='mac',
baseline_move_to='mac-leopard',
expected_success=True,
expected_log=[
'Rebaselining fast/image...',
' Moving current mac baselines to mac-leopard',
' Moved image-expected.checksum',
' Moved image-expected.png',
' Updating baselines for mac',
' Updated image-expected.checksum',
' Updated image-expected.png',
])
def test_text_rebaseline_move_already_exist(self):
self._assertRebaseline(
test_files=(
'fast/text-expected.txt',
'platform/mac-leopard/fast/text-expected.txt',
'platform/mac/fast/text-expected.txt',
),
results_files=(
'fast/text-actual.txt',
),
test_name='fast/text.html',
baseline_target='mac',
baseline_move_to='mac-leopard',
expected_success=False,
expected_log=[
'Rebaselining fast/text...',
' Moving current mac baselines to mac-leopard',
' Already had baselines in mac-leopard, could not move existing mac ones',
])
def test_image_rebaseline(self):
self._assertRebaseline(
test_files=(
'fast/image-expected.txt',
'platform/mac/fast/image-expected.png',
'platform/mac/fast/image-expected.checksum',
),
results_files=(
'fast/image-actual.png',
'fast/image-actual.checksum',
),
test_name='fast/image.html',
baseline_target='mac',
baseline_move_to='none',
expected_success=True,
expected_log=[
'Rebaselining fast/image...',
' Updating baselines for mac',
' Updated image-expected.checksum',
' Updated image-expected.png',
])
def test_gather_baselines(self):
example_json = resultsjsonparser_unittest.ParsedJSONResultsTest._example_full_results_json
results_json = json.loads(strip_json_wrapper(example_json))
server = RebaselineServer()
server._test_config = get_test_config()
server._gather_baselines(results_json)
self.assertEqual(results_json['tests']['svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html']['state'], 'needs_rebaseline')
self.assertNotIn('prototype-chocolate.html', results_json['tests'])
def _assertRebaseline(self, test_files, results_files, test_name, baseline_target, baseline_move_to, expected_success, expected_log):
log = []
test_config = get_test_config(test_files, results_files)
success = rebaselineserver._rebaseline_test(
test_name,
baseline_target,
baseline_move_to,
test_config,
log=lambda l: log.append(l))
self.assertEqual(expected_log, log)
self.assertEqual(expected_success, success)
class GetActualResultFilesTest(unittest.TestCase):
def test(self):
test_config = get_test_config(result_files=(
'fast/text-actual.txt',
'fast2/text-actual.txt',
'fast/text2-actual.txt',
'fast/text-notactual.txt',
))
self.assertItemsEqual(
('text-actual.txt',),
rebaselineserver._get_actual_result_files(
'fast/text.html', test_config))
class GetBaselinesTest(unittest.TestCase):
def test_no_baselines(self):
self._assertBaselines(
test_files=(),
test_name='fast/missing.html',
expected_baselines={})
def test_text_baselines(self):
self._assertBaselines(
test_files=(
'fast/text-expected.txt',
'platform/mac/fast/text-expected.txt',
),
test_name='fast/text.html',
expected_baselines={
'mac': {'.txt': True},
'base': {'.txt': False},
})
def test_image_and_text_baselines(self):
self._assertBaselines(
test_files=(
'fast/image-expected.txt',
'platform/mac/fast/image-expected.png',
'platform/mac/fast/image-expected.checksum',
'platform/win/fast/image-expected.png',
'platform/win/fast/image-expected.checksum',
),
test_name='fast/image.html',
expected_baselines={
'base': {'.txt': True},
'mac': {'.checksum': True, '.png': True},
'win': {'.checksum': False, '.png': False},
})
def test_extra_baselines(self):
self._assertBaselines(
test_files=(
'fast/text-expected.txt',
'platform/nosuchplatform/fast/text-expected.txt',
),
test_name='fast/text.html',
expected_baselines={'base': {'.txt': True}})
def _assertBaselines(self, test_files, test_name, expected_baselines):
actual_baselines = rebaselineserver.get_test_baselines(test_name, get_test_config(test_files))
self.assertEqual(expected_baselines, actual_baselines)
def get_test_config(test_files=[], result_files=[]):
# We could grab this from port.layout_tests_dir(), but instantiating a fully mocked port is a pain.
layout_tests_directory = "/mock-checkout/LayoutTests"
results_directory = '/WebKitBuild/Debug/layout-test-results'
host = MockHost()
for file in test_files:
host.filesystem.write_binary_file(host.filesystem.join(layout_tests_directory, file), '')
for file in result_files:
host.filesystem.write_binary_file(host.filesystem.join(results_directory, file), '')
class TestMacPort(Port):
port_name = "mac"
return TestConfig(
TestMacPort(host, 'mac'),
layout_tests_directory,
results_directory,
('mac', 'mac-leopard', 'win', 'linux'),
host.scm(),
host)