| OLD | NEW |
| (Empty) |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | |
| 2 # | |
| 3 # Redistribution and use in source and binary forms, with or without | |
| 4 # modification, are permitted provided that the following conditions are | |
| 5 # met: | |
| 6 # | |
| 7 # * Redistributions of source code must retain the above copyright | |
| 8 # notice, this list of conditions and the following disclaimer. | |
| 9 # * Redistributions in binary form must reproduce the above | |
| 10 # copyright notice, this list of conditions and the following disclaimer | |
| 11 # in the documentation and/or other materials provided with the | |
| 12 # distribution. | |
| 13 # * Neither the name of Google Inc. nor the names of its | |
| 14 # contributors may be used to endorse or promote products derived from | |
| 15 # this software without specific prior written permission. | |
| 16 # | |
| 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 28 | |
| 29 import json | |
| 30 import unittest | |
| 31 | |
| 32 from webkitpy.common.net import layouttestresults_unittest | |
| 33 from webkitpy.common.host_mock import MockHost | |
| 34 from webkitpy.layout_tests.layout_package.json_results_generator import strip_js
on_wrapper | |
| 35 from webkitpy.layout_tests.port.base import Port | |
| 36 from webkitpy.tool.commands.rebaselineserver import TestConfig, RebaselineServer | |
| 37 from webkitpy.tool.servers import rebaselineserver | |
| 38 | |
| 39 | |
| 40 class RebaselineTestTest(unittest.TestCase): | |
| 41 def test_text_rebaseline_update(self): | |
| 42 self._assertRebaseline( | |
| 43 test_files=( | |
| 44 'fast/text-expected.txt', | |
| 45 'platform/mac/fast/text-expected.txt', | |
| 46 ), | |
| 47 results_files=( | |
| 48 'fast/text-actual.txt', | |
| 49 ), | |
| 50 test_name='fast/text.html', | |
| 51 baseline_target='mac', | |
| 52 baseline_move_to='none', | |
| 53 expected_success=True, | |
| 54 expected_log=[ | |
| 55 'Rebaselining fast/text...', | |
| 56 ' Updating baselines for mac', | |
| 57 ' Updated text-expected.txt', | |
| 58 ]) | |
| 59 | |
| 60 def test_text_rebaseline_new(self): | |
| 61 self._assertRebaseline( | |
| 62 test_files=( | |
| 63 'fast/text-expected.txt', | |
| 64 ), | |
| 65 results_files=( | |
| 66 'fast/text-actual.txt', | |
| 67 ), | |
| 68 test_name='fast/text.html', | |
| 69 baseline_target='mac', | |
| 70 baseline_move_to='none', | |
| 71 expected_success=True, | |
| 72 expected_log=[ | |
| 73 'Rebaselining fast/text...', | |
| 74 ' Updating baselines for mac', | |
| 75 ' Updated text-expected.txt', | |
| 76 ]) | |
| 77 | |
| 78 def test_text_rebaseline_move_no_op_1(self): | |
| 79 self._assertRebaseline( | |
| 80 test_files=( | |
| 81 'fast/text-expected.txt', | |
| 82 'platform/win/fast/text-expected.txt', | |
| 83 ), | |
| 84 results_files=( | |
| 85 'fast/text-actual.txt', | |
| 86 ), | |
| 87 test_name='fast/text.html', | |
| 88 baseline_target='mac', | |
| 89 baseline_move_to='mac-leopard', | |
| 90 expected_success=True, | |
| 91 expected_log=[ | |
| 92 'Rebaselining fast/text...', | |
| 93 ' Updating baselines for mac', | |
| 94 ' Updated text-expected.txt', | |
| 95 ]) | |
| 96 | |
| 97 def test_text_rebaseline_move_no_op_2(self): | |
| 98 self._assertRebaseline( | |
| 99 test_files=( | |
| 100 'fast/text-expected.txt', | |
| 101 'platform/mac/fast/text-expected.checksum', | |
| 102 ), | |
| 103 results_files=( | |
| 104 'fast/text-actual.txt', | |
| 105 ), | |
| 106 test_name='fast/text.html', | |
| 107 baseline_target='mac', | |
| 108 baseline_move_to='mac-leopard', | |
| 109 expected_success=True, | |
| 110 expected_log=[ | |
| 111 'Rebaselining fast/text...', | |
| 112 ' Moving current mac baselines to mac-leopard', | |
| 113 ' No current baselines to move', | |
| 114 ' Updating baselines for mac', | |
| 115 ' Updated text-expected.txt', | |
| 116 ]) | |
| 117 | |
| 118 def test_text_rebaseline_move(self): | |
| 119 self._assertRebaseline( | |
| 120 test_files=( | |
| 121 'fast/text-expected.txt', | |
| 122 'platform/mac/fast/text-expected.txt', | |
| 123 ), | |
| 124 results_files=( | |
| 125 'fast/text-actual.txt', | |
| 126 ), | |
| 127 test_name='fast/text.html', | |
| 128 baseline_target='mac', | |
| 129 baseline_move_to='mac-leopard', | |
| 130 expected_success=True, | |
| 131 expected_log=[ | |
| 132 'Rebaselining fast/text...', | |
| 133 ' Moving current mac baselines to mac-leopard', | |
| 134 ' Moved text-expected.txt', | |
| 135 ' Updating baselines for mac', | |
| 136 ' Updated text-expected.txt', | |
| 137 ]) | |
| 138 | |
| 139 def test_text_rebaseline_move_only_images(self): | |
| 140 self._assertRebaseline( | |
| 141 test_files=( | |
| 142 'fast/image-expected.txt', | |
| 143 'platform/mac/fast/image-expected.txt', | |
| 144 'platform/mac/fast/image-expected.png', | |
| 145 'platform/mac/fast/image-expected.checksum', | |
| 146 ), | |
| 147 results_files=( | |
| 148 'fast/image-actual.png', | |
| 149 'fast/image-actual.checksum', | |
| 150 ), | |
| 151 test_name='fast/image.html', | |
| 152 baseline_target='mac', | |
| 153 baseline_move_to='mac-leopard', | |
| 154 expected_success=True, | |
| 155 expected_log=[ | |
| 156 'Rebaselining fast/image...', | |
| 157 ' Moving current mac baselines to mac-leopard', | |
| 158 ' Moved image-expected.checksum', | |
| 159 ' Moved image-expected.png', | |
| 160 ' Updating baselines for mac', | |
| 161 ' Updated image-expected.checksum', | |
| 162 ' Updated image-expected.png', | |
| 163 ]) | |
| 164 | |
| 165 def test_text_rebaseline_move_already_exist(self): | |
| 166 self._assertRebaseline( | |
| 167 test_files=( | |
| 168 'fast/text-expected.txt', | |
| 169 'platform/mac-leopard/fast/text-expected.txt', | |
| 170 'platform/mac/fast/text-expected.txt', | |
| 171 ), | |
| 172 results_files=( | |
| 173 'fast/text-actual.txt', | |
| 174 ), | |
| 175 test_name='fast/text.html', | |
| 176 baseline_target='mac', | |
| 177 baseline_move_to='mac-leopard', | |
| 178 expected_success=False, | |
| 179 expected_log=[ | |
| 180 'Rebaselining fast/text...', | |
| 181 ' Moving current mac baselines to mac-leopard', | |
| 182 ' Already had baselines in mac-leopard, could not move existi
ng mac ones', | |
| 183 ]) | |
| 184 | |
| 185 def test_image_rebaseline(self): | |
| 186 self._assertRebaseline( | |
| 187 test_files=( | |
| 188 'fast/image-expected.txt', | |
| 189 'platform/mac/fast/image-expected.png', | |
| 190 'platform/mac/fast/image-expected.checksum', | |
| 191 ), | |
| 192 results_files=( | |
| 193 'fast/image-actual.png', | |
| 194 'fast/image-actual.checksum', | |
| 195 ), | |
| 196 test_name='fast/image.html', | |
| 197 baseline_target='mac', | |
| 198 baseline_move_to='none', | |
| 199 expected_success=True, | |
| 200 expected_log=[ | |
| 201 'Rebaselining fast/image...', | |
| 202 ' Updating baselines for mac', | |
| 203 ' Updated image-expected.checksum', | |
| 204 ' Updated image-expected.png', | |
| 205 ]) | |
| 206 | |
| 207 def test_gather_baselines(self): | |
| 208 example_json = layouttestresults_unittest.LayoutTestResultsTest.example_
full_results_json | |
| 209 results_json = json.loads(strip_json_wrapper(example_json)) | |
| 210 server = RebaselineServer() | |
| 211 server._test_config = get_test_config() | |
| 212 server._gather_baselines(results_json) | |
| 213 self.assertEqual(results_json['tests']['svg/dynamic-updates/SVGFEDropSha
dowElement-dom-stdDeviation-attr.html']['state'], 'needs_rebaseline') | |
| 214 self.assertNotIn('prototype-chocolate.html', results_json['tests']) | |
| 215 | |
| 216 def _assertRebaseline(self, test_files, results_files, test_name, baseline_t
arget, baseline_move_to, expected_success, expected_log): | |
| 217 log = [] | |
| 218 test_config = get_test_config(test_files, results_files) | |
| 219 success = rebaselineserver._rebaseline_test( | |
| 220 test_name, | |
| 221 baseline_target, | |
| 222 baseline_move_to, | |
| 223 test_config, | |
| 224 log=lambda l: log.append(l)) | |
| 225 self.assertEqual(expected_log, log) | |
| 226 self.assertEqual(expected_success, success) | |
| 227 | |
| 228 | |
| 229 class GetActualResultFilesTest(unittest.TestCase): | |
| 230 def test(self): | |
| 231 test_config = get_test_config(result_files=( | |
| 232 'fast/text-actual.txt', | |
| 233 'fast2/text-actual.txt', | |
| 234 'fast/text2-actual.txt', | |
| 235 'fast/text-notactual.txt', | |
| 236 )) | |
| 237 self.assertItemsEqual( | |
| 238 ('text-actual.txt',), | |
| 239 rebaselineserver._get_actual_result_files( | |
| 240 'fast/text.html', test_config)) | |
| 241 | |
| 242 | |
| 243 class GetBaselinesTest(unittest.TestCase): | |
| 244 def test_no_baselines(self): | |
| 245 self._assertBaselines( | |
| 246 test_files=(), | |
| 247 test_name='fast/missing.html', | |
| 248 expected_baselines={}) | |
| 249 | |
| 250 def test_text_baselines(self): | |
| 251 self._assertBaselines( | |
| 252 test_files=( | |
| 253 'fast/text-expected.txt', | |
| 254 'platform/mac/fast/text-expected.txt', | |
| 255 ), | |
| 256 test_name='fast/text.html', | |
| 257 expected_baselines={ | |
| 258 'mac': {'.txt': True}, | |
| 259 'base': {'.txt': False}, | |
| 260 }) | |
| 261 | |
| 262 def test_image_and_text_baselines(self): | |
| 263 self._assertBaselines( | |
| 264 test_files=( | |
| 265 'fast/image-expected.txt', | |
| 266 'platform/mac/fast/image-expected.png', | |
| 267 'platform/mac/fast/image-expected.checksum', | |
| 268 'platform/win/fast/image-expected.png', | |
| 269 'platform/win/fast/image-expected.checksum', | |
| 270 ), | |
| 271 test_name='fast/image.html', | |
| 272 expected_baselines={ | |
| 273 'base': {'.txt': True}, | |
| 274 'mac': {'.checksum': True, '.png': True}, | |
| 275 'win': {'.checksum': False, '.png': False}, | |
| 276 }) | |
| 277 | |
| 278 def test_extra_baselines(self): | |
| 279 self._assertBaselines( | |
| 280 test_files=( | |
| 281 'fast/text-expected.txt', | |
| 282 'platform/nosuchplatform/fast/text-expected.txt', | |
| 283 ), | |
| 284 test_name='fast/text.html', | |
| 285 expected_baselines={'base': {'.txt': True}}) | |
| 286 | |
| 287 def _assertBaselines(self, test_files, test_name, expected_baselines): | |
| 288 actual_baselines = rebaselineserver.get_test_baselines(test_name, get_te
st_config(test_files)) | |
| 289 self.assertEqual(expected_baselines, actual_baselines) | |
| 290 | |
| 291 | |
| 292 def get_test_config(test_files=[], result_files=[]): | |
| 293 host = MockHost() | |
| 294 port = host.port_factory.get() | |
| 295 layout_tests_directory = port.layout_tests_dir() | |
| 296 results_directory = port.results_directory() | |
| 297 | |
| 298 for file in test_files: | |
| 299 host.filesystem.write_binary_file(host.filesystem.join(layout_tests_dire
ctory, file), '') | |
| 300 for file in result_files: | |
| 301 host.filesystem.write_binary_file(host.filesystem.join(results_directory
, file), '') | |
| 302 | |
| 303 class TestMacPort(Port): | |
| 304 port_name = "mac" | |
| 305 FALLBACK_PATHS = {'': ['mac']} | |
| 306 | |
| 307 return TestConfig( | |
| 308 TestMacPort(host, 'mac'), | |
| 309 layout_tests_directory, | |
| 310 results_directory, | |
| 311 ('mac', 'mac-leopard', 'win', 'linux'), | |
| 312 host.filesystem, | |
| 313 host.scm()) | |
| OLD | NEW |