| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 """ | 3 """ |
| 4 Copyright 2013 Google Inc. | 4 Copyright 2013 Google Inc. |
| 5 | 5 |
| 6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
| 7 found in the LICENSE file. | 7 found in the LICENSE file. |
| 8 | 8 |
| 9 Test results.py | 9 Test results.py |
| 10 | 10 |
| 11 TODO(epoger): Create a command to update the expected results (in | 11 TODO(epoger): Create a command to update the expected results (in |
| 12 OUTPUT_DIR_EXPECTED) when appropriate. For now, you should: | 12 self._output_dir_expected) when appropriate. For now, you should: |
| 13 1. examine the results in OUTPUT_DIR_ACTUAL and make sure they are ok | 13 1. examine the results in self._output_dir_actual and make sure they are ok |
| 14 2. rm -rf OUTPUT_DIR_EXPECTED | 14 2. rm -rf self._output_dir_expected |
| 15 3. mv OUTPUT_DIR_ACTUAL OUTPUT_DIR_EXPECTED | 15 3. mv self._output_dir_actual self._output_dir_expected |
| 16 Although, if you're using an SVN checkout, this will blow away .svn directories | 16 Although, if you're using an SVN checkout, this will blow away .svn directories |
| 17 within OUTPUT_DIR_EXPECTED, which wouldn't be good... | 17 within self._output_dir_expected, which wouldn't be good... |
| 18 | 18 |
| 19 """ | 19 """ |
| 20 | 20 |
| 21 import filecmp | |
| 22 import os | 21 import os |
| 23 import shutil | |
| 24 import sys | 22 import sys |
| 25 import tempfile | |
| 26 import unittest | |
| 27 | 23 |
| 28 # Imports from within Skia | 24 # Imports from within Skia |
| 25 import base_unittest |
| 29 import results | 26 import results |
| 30 import gm_json # must import results first, so that gm_json will be in sys.path | 27 import gm_json # must import results first, so that gm_json will be in sys.path |
| 31 | 28 |
| 32 PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) | |
| 33 INPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'inputs') | |
| 34 OUTPUT_DIR_ACTUAL = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual') | |
| 35 OUTPUT_DIR_EXPECTED = os.path.join(PARENT_DIR, 'tests', 'outputs', 'expected') | |
| 36 | 29 |
| 37 | 30 class ResultsTest(base_unittest.TestCase): |
| 38 class ResultsTest(unittest.TestCase): | |
| 39 | |
| 40 def setUp(self): | |
| 41 self._temp_dir = tempfile.mkdtemp() | |
| 42 self._output_dir_actual = os.path.join(OUTPUT_DIR_ACTUAL, self.id()) | |
| 43 self._output_dir_expected = os.path.join(OUTPUT_DIR_EXPECTED, self.id()) | |
| 44 create_empty_dir(self._output_dir_actual) | |
| 45 | |
| 46 def tearDown(self): | |
| 47 shutil.rmtree(self._temp_dir) | |
| 48 different_files = find_different_files(self._output_dir_actual, | |
| 49 self._output_dir_expected) | |
| 50 # Maybe we should move this assert elsewhere? It's unusual to see an | |
| 51 # assert within tearDown(), but my thinking was: | |
| 52 # 1. Every test case will have some collection of output files that need to | |
| 53 # be validated. | |
| 54 # 2. So put that validation within tearDown(), which will be called after | |
| 55 # every test case! | |
| 56 # | |
| 57 # I have confirmed that the test really does fail if this assert is | |
| 58 # triggered. | |
| 59 # | |
| 60 # Ravi notes: if somebody later comes along and adds cleanup code below the | |
| 61 # assert, then if tests fail, the artifacts will not be cleaned up. | |
| 62 assert (not different_files), \ | |
| 63 ('found differing files between actual dir %s and expected dir %s: %s' % | |
| 64 (self._output_dir_actual, self._output_dir_expected, different_files)) | |
| 65 | |
| 66 def shortDescription(self): | |
| 67 """Tell unittest framework to not print docstrings for test cases.""" | |
| 68 return None | |
| 69 | 31 |
| 70 def test_gm(self): | 32 def test_gm(self): |
| 71 """Process results of a GM run with the Results object.""" | 33 """Process results of a GM run with the Results object.""" |
| 72 results_obj = results.Results( | 34 results_obj = results.Results( |
| 73 actuals_root=os.path.join(INPUT_DIR, 'gm-actuals'), | 35 actuals_root=os.path.join(self._input_dir, 'gm-actuals'), |
| 74 expected_root=os.path.join(INPUT_DIR, 'gm-expectations'), | 36 expected_root=os.path.join(self._input_dir, 'gm-expectations'), |
| 75 generated_images_root=self._temp_dir) | 37 generated_images_root=self._temp_dir) |
| 76 gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL), | 38 gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL), |
| 77 os.path.join(self._output_dir_actual, 'gm.json')) | 39 os.path.join(self._output_dir_actual, 'gm.json')) |
| 78 | 40 |
| 79 | 41 |
| 80 # TODO(epoger): create_empty_dir(), find_different_files(), etc. should be | |
| 81 # extracted from this file to some common location, where they can be shared | |
| 82 # with download_test.py and other users. | |
| 83 | |
| 84 def create_empty_dir(path): | |
| 85 """Create an empty directory at the given path.""" | |
| 86 if os.path.isdir(path): | |
| 87 shutil.rmtree(path) | |
| 88 elif os.path.lexists(path): | |
| 89 os.remove(path) | |
| 90 os.makedirs(path) | |
| 91 | |
| 92 | |
| 93 def find_different_files(dir1, dir2, ignore_subtree_names=None): | |
| 94 """Returns a list of any files that differ between the directory trees rooted | |
| 95 at dir1 and dir2. | |
| 96 | |
| 97 Args: | |
| 98 dir1: root of a directory tree; if nonexistent, will raise OSError | |
| 99 dir2: root of another directory tree; if nonexistent, will raise OSError | |
| 100 ignore_subtree_names: list of subtree directory names to ignore; | |
| 101 defaults to ['.svn'], so all SVN files are ignores | |
| 102 | |
| 103 TODO(epoger): include the dirname within each filename (not just the | |
| 104 basename), to make it easier to locate any differences | |
| 105 """ | |
| 106 differing_files = [] | |
| 107 if ignore_subtree_names is None: | |
| 108 ignore_subtree_names = ['.svn'] | |
| 109 dircmp = filecmp.dircmp(dir1, dir2, ignore=ignore_subtree_names) | |
| 110 differing_files.extend(dircmp.left_only) | |
| 111 differing_files.extend(dircmp.right_only) | |
| 112 differing_files.extend(dircmp.common_funny) | |
| 113 differing_files.extend(dircmp.diff_files) | |
| 114 differing_files.extend(dircmp.funny_files) | |
| 115 for common_dir in dircmp.common_dirs: | |
| 116 differing_files.extend(find_different_files( | |
| 117 os.path.join(dir1, common_dir), os.path.join(dir2, common_dir))) | |
| 118 return differing_files | |
| 119 | |
| 120 | |
| 121 def main(): | 42 def main(): |
| 122 suite = unittest.TestLoader().loadTestsFromTestCase(ResultsTest) | 43 base_unittest.main(ResultsTest) |
| 123 unittest.TextTestRunner(verbosity=2).run(suite) | |
| 124 | 44 |
| 125 | 45 |
| 126 if __name__ == '__main__': | 46 if __name__ == '__main__': |
| 127 main() | 47 main() |
| OLD | NEW |