| OLD | NEW |
| 1 # Copyright (C) 2011 Google Inc. All rights reserved. | 1 # Copyright (C) 2011 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 25 matching lines...) Expand all Loading... |
| 36 from webkitpy.layout_tests.port.driver import DeviceFailure, DriverInput, Driver
Output | 36 from webkitpy.layout_tests.port.driver import DeviceFailure, DriverInput, Driver
Output |
| 37 from webkitpy.layout_tests.models import test_expectations | 37 from webkitpy.layout_tests.models import test_expectations |
| 38 from webkitpy.layout_tests.models import test_failures | 38 from webkitpy.layout_tests.models import test_failures |
| 39 from webkitpy.layout_tests.models.test_results import TestResult | 39 from webkitpy.layout_tests.models.test_results import TestResult |
| 40 from webkitpy.layout_tests.models import testharness_results | 40 from webkitpy.layout_tests.models import testharness_results |
| 41 | 41 |
| 42 | 42 |
| 43 _log = logging.getLogger(__name__) | 43 _log = logging.getLogger(__name__) |
| 44 | 44 |
| 45 | 45 |
| 46 def run_single_test( | 46 def run_single_test(port, options, results_directory, worker_name, driver, test_
input, stop_when_done): |
| 47 port, options, results_directory, worker_name, primary_driver, | 47 runner = SingleTestRunner(port, options, results_directory, worker_name, dri
ver, test_input, stop_when_done) |
| 48 secondary_driver, test_input, stop_when_done): | |
| 49 runner = SingleTestRunner( | |
| 50 port, options, results_directory, worker_name, primary_driver, | |
| 51 secondary_driver, test_input, stop_when_done) | |
| 52 try: | 48 try: |
| 53 return runner.run() | 49 return runner.run() |
| 54 except DeviceFailure as e: | 50 except DeviceFailure as e: |
| 55 _log.error("device failed: %s", str(e)) | 51 _log.error("device failed: %s", str(e)) |
| 56 return TestResult(test_input.test_name, device_failed=True) | 52 return TestResult(test_input.test_name, device_failed=True) |
| 57 | 53 |
| 58 | 54 |
| 59 class SingleTestRunner(object): | 55 class SingleTestRunner(object): |
| 60 (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platfor
m', 'version', 'update') | 56 (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platfor
m', 'version', 'update') |
| 61 | 57 |
| 62 def __init__(self, port, options, results_directory, worker_name, | 58 def __init__(self, port, options, results_directory, worker_name, driver, te
st_input, stop_when_done): |
| 63 primary_driver, secondary_driver, test_input, stop_when_done): | |
| 64 self._port = port | 59 self._port = port |
| 65 self._filesystem = port.host.filesystem | 60 self._filesystem = port.host.filesystem |
| 66 self._options = options | 61 self._options = options |
| 67 self._results_directory = results_directory | 62 self._results_directory = results_directory |
| 68 self._driver = primary_driver | 63 self._driver = driver |
| 69 self._reference_driver = primary_driver | |
| 70 self._timeout = test_input.timeout | 64 self._timeout = test_input.timeout |
| 71 self._worker_name = worker_name | 65 self._worker_name = worker_name |
| 72 self._test_name = test_input.test_name | 66 self._test_name = test_input.test_name |
| 73 self._should_run_pixel_test = test_input.should_run_pixel_test | 67 self._should_run_pixel_test = test_input.should_run_pixel_test |
| 74 self._reference_files = test_input.reference_files | 68 self._reference_files = test_input.reference_files |
| 75 self._should_add_missing_baselines = test_input.should_add_missing_basel
ines | 69 self._should_add_missing_baselines = test_input.should_add_missing_basel
ines |
| 76 self._stop_when_done = stop_when_done | 70 self._stop_when_done = stop_when_done |
| 77 | 71 |
| 78 # If this is a virtual test that uses the default flags instead of the | |
| 79 # virtual flags for it's references, run it on the secondary driver so | |
| 80 # that the primary driver does not need to be restarted. | |
| 81 if (self._port.is_virtual_test(self._test_name) and | |
| 82 not self._port.lookup_virtual_reference_args(self._test_name)): | |
| 83 self._reference_driver = secondary_driver | |
| 84 | |
| 85 if self._reference_files: | 72 if self._reference_files: |
| 86 # Detect and report a test which has a wrong combination of expectat
ion files. | 73 # Detect and report a test which has a wrong combination of expectat
ion files. |
| 87 # For example, if 'foo.html' has two expectation files, 'foo-expecte
d.html' and | 74 # For example, if 'foo.html' has two expectation files, 'foo-expecte
d.html' and |
| 88 # 'foo-expected.txt', we should warn users. One test file must be us
ed exclusively | 75 # 'foo-expected.txt', we should warn users. One test file must be us
ed exclusively |
| 89 # in either layout tests or reftests, but not in both. | 76 # in either layout tests or reftests, but not in both. |
| 90 for suffix in ('.txt', '.png', '.wav'): | 77 for suffix in ('.txt', '.png', '.wav'): |
| 91 expected_filename = self._port.expected_filename(self._test_name
, suffix) | 78 expected_filename = self._port.expected_filename(self._test_name
, suffix) |
| 92 if self._filesystem.exists(expected_filename): | 79 if self._filesystem.exists(expected_filename): |
| 93 _log.error('%s is a reftest, but has an unused expectation f
ile. Please remove %s.', | 80 _log.error('%s is a reftest, but has an unused expectation f
ile. Please remove %s.', |
| 94 self._test_name, expected_filename) | 81 self._test_name, expected_filename) |
| (...skipping 283 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 378 putAllMismatchBeforeMatch = sorted | 365 putAllMismatchBeforeMatch = sorted |
| 379 reference_test_names = [] | 366 reference_test_names = [] |
| 380 for expectation, reference_filename in putAllMismatchBeforeMatch(self._r
eference_files): | 367 for expectation, reference_filename in putAllMismatchBeforeMatch(self._r
eference_files): |
| 381 if self._port.lookup_virtual_test_base(self._test_name): | 368 if self._port.lookup_virtual_test_base(self._test_name): |
| 382 args = self._port.lookup_virtual_reference_args(self._test_name) | 369 args = self._port.lookup_virtual_reference_args(self._test_name) |
| 383 else: | 370 else: |
| 384 args = self._port.lookup_physical_reference_args(self._test_name
) | 371 args = self._port.lookup_physical_reference_args(self._test_name
) |
| 385 reference_test_name = self._port.relative_test_filename(reference_fi
lename) | 372 reference_test_name = self._port.relative_test_filename(reference_fi
lename) |
| 386 reference_test_names.append(reference_test_name) | 373 reference_test_names.append(reference_test_name) |
| 387 driver_input = DriverInput(reference_test_name, self._timeout, image
_hash=None, should_run_pixel_test=True, args=args) | 374 driver_input = DriverInput(reference_test_name, self._timeout, image
_hash=None, should_run_pixel_test=True, args=args) |
| 388 reference_output = self._reference_driver.run_test(driver_input, sel
f._stop_when_done) | 375 reference_output = self._driver.run_test(driver_input, self._stop_wh
en_done) |
| 389 test_result = self._compare_output_with_reference(reference_output,
test_output, reference_filename, expectation == '!=') | 376 test_result = self._compare_output_with_reference(reference_output,
test_output, reference_filename, expectation == '!=') |
| 390 | 377 |
| 391 if (expectation == '!=' and test_result.failures) or (expectation ==
'==' and not test_result.failures): | 378 if (expectation == '!=' and test_result.failures) or (expectation ==
'==' and not test_result.failures): |
| 392 break | 379 break |
| 393 total_test_time += test_result.test_run_time | 380 total_test_time += test_result.test_run_time |
| 394 | 381 |
| 395 assert(reference_output) | 382 assert(reference_output) |
| 396 test_result_writer.write_test_result(self._filesystem, self._port, self.
_results_directory, self._test_name, test_output, reference_output, test_result.
failures) | 383 test_result_writer.write_test_result(self._filesystem, self._port, self.
_results_directory, self._test_name, test_output, reference_output, test_result.
failures) |
| 397 | 384 |
| 398 # FIXME: We don't really deal with a mix of reftest types properly. We p
ass in a set() to reftest_type | 385 # FIXME: We don't really deal with a mix of reftest types properly. We p
ass in a set() to reftest_type |
| (...skipping 30 matching lines...) Expand all Loading... |
| 429 elif reference_driver_output.image_hash != actual_driver_output.image_ha
sh: | 416 elif reference_driver_output.image_hash != actual_driver_output.image_ha
sh: |
| 430 diff, err_str = self._port.diff_image(reference_driver_output.image,
actual_driver_output.image) | 417 diff, err_str = self._port.diff_image(reference_driver_output.image,
actual_driver_output.image) |
| 431 if diff: | 418 if diff: |
| 432 failures.append(test_failures.FailureReftestMismatch(reference_f
ilename)) | 419 failures.append(test_failures.FailureReftestMismatch(reference_f
ilename)) |
| 433 elif err_str: | 420 elif err_str: |
| 434 _log.error(err_str) | 421 _log.error(err_str) |
| 435 else: | 422 else: |
| 436 _log.warning(" %s -> ref test hashes didn't match but diff pass
ed" % self._test_name) | 423 _log.warning(" %s -> ref test hashes didn't match but diff pass
ed" % self._test_name) |
| 437 | 424 |
| 438 return TestResult(self._test_name, failures, total_test_time, has_stderr
, pid=actual_driver_output.pid) | 425 return TestResult(self._test_name, failures, total_test_time, has_stderr
, pid=actual_driver_output.pid) |
| OLD | NEW |