OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 """ | 3 """ |
4 Copyright 2013 Google Inc. | 4 Copyright 2013 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 | 8 |
9 Test results.py | 9 Test results.py |
10 | 10 |
11 TODO(epoger): Create a command to update the expected results (in | 11 TODO(epoger): Create a command to update the expected results (in |
12 OUTPUT_DIR_EXPECTED) when appropriate. For now, you should: | 12 self._output_dir_expected) when appropriate. For now, you should: |
13 1. examine the results in OUTPUT_DIR_ACTUAL and make sure they are ok | 13 1. examine the results in self._output_dir_actual and make sure they are ok |
14 2. rm -rf OUTPUT_DIR_EXPECTED | 14 2. rm -rf self._output_dir_expected |
15 3. mv OUTPUT_DIR_ACTUAL OUTPUT_DIR_EXPECTED | 15 3. mv self._output_dir_actual self._output_dir_expected |
16 Although, if you're using an SVN checkout, this will blow away .svn directories | 16 Although, if you're using an SVN checkout, this will blow away .svn directories |
17 within OUTPUT_DIR_EXPECTED, which wouldn't be good... | 17 within self._output_dir_expected, which wouldn't be good... |
18 | 18 |
19 """ | 19 """ |
20 | 20 |
21 import filecmp | |
22 import os | 21 import os |
23 import shutil | |
24 import sys | 22 import sys |
25 import tempfile | |
26 import unittest | |
27 | 23 |
28 # Imports from within Skia | 24 # Imports from within Skia |
| 25 import base_unittest |
29 import results | 26 import results |
30 import gm_json # must import results first, so that gm_json will be in sys.path | 27 import gm_json # must import results first, so that gm_json will be in sys.path |
31 | 28 |
32 PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) | |
33 INPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'inputs') | |
34 OUTPUT_DIR_ACTUAL = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual') | |
35 OUTPUT_DIR_EXPECTED = os.path.join(PARENT_DIR, 'tests', 'outputs', 'expected') | |
36 | 29 |
37 | 30 class ResultsTest(base_unittest.TestCase): |
38 class ResultsTest(unittest.TestCase): | |
39 | |
40 def setUp(self): | |
41 self._temp_dir = tempfile.mkdtemp() | |
42 self._output_dir_actual = os.path.join(OUTPUT_DIR_ACTUAL, self.id()) | |
43 self._output_dir_expected = os.path.join(OUTPUT_DIR_EXPECTED, self.id()) | |
44 create_empty_dir(self._output_dir_actual) | |
45 | |
46 def tearDown(self): | |
47 shutil.rmtree(self._temp_dir) | |
48 different_files = find_different_files(self._output_dir_actual, | |
49 self._output_dir_expected) | |
50 # Maybe we should move this assert elsewhere? It's unusual to see an | |
51 # assert within tearDown(), but my thinking was: | |
52 # 1. Every test case will have some collection of output files that need to | |
53 # be validated. | |
54 # 2. So put that validation within tearDown(), which will be called after | |
55 # every test case! | |
56 # | |
57 # I have confirmed that the test really does fail if this assert is | |
58 # triggered. | |
59 # | |
60 # Ravi notes: if somebody later comes along and adds cleanup code below the | |
61 # assert, then if tests fail, the artifacts will not be cleaned up. | |
62 assert (not different_files), \ | |
63 ('found differing files between actual dir %s and expected dir %s: %s' % | |
64 (self._output_dir_actual, self._output_dir_expected, different_files)) | |
65 | |
66 def shortDescription(self): | |
67 """Tell unittest framework to not print docstrings for test cases.""" | |
68 return None | |
69 | 31 |
70 def test_gm(self): | 32 def test_gm(self): |
71 """Process results of a GM run with the Results object.""" | 33 """Process results of a GM run with the Results object.""" |
72 results_obj = results.Results( | 34 results_obj = results.Results( |
73 actuals_root=os.path.join(INPUT_DIR, 'gm-actuals'), | 35 actuals_root=os.path.join(self._input_dir, 'gm-actuals'), |
74 expected_root=os.path.join(INPUT_DIR, 'gm-expectations'), | 36 expected_root=os.path.join(self._input_dir, 'gm-expectations'), |
75 generated_images_root=self._temp_dir) | 37 generated_images_root=self._temp_dir) |
76 gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL), | 38 gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL), |
77 os.path.join(self._output_dir_actual, 'gm.json')) | 39 os.path.join(self._output_dir_actual, 'gm.json')) |
78 | 40 |
79 | 41 |
80 # TODO(epoger): create_empty_dir(), find_different_files(), etc. should be | |
81 # extracted from this file to some common location, where they can be shared | |
82 # with download_test.py and other users. | |
83 | |
84 def create_empty_dir(path): | |
85 """Create an empty directory at the given path.""" | |
86 if os.path.isdir(path): | |
87 shutil.rmtree(path) | |
88 elif os.path.lexists(path): | |
89 os.remove(path) | |
90 os.makedirs(path) | |
91 | |
92 | |
93 def find_different_files(dir1, dir2, ignore_subtree_names=None): | |
94 """Returns a list of any files that differ between the directory trees rooted | |
95 at dir1 and dir2. | |
96 | |
97 Args: | |
98 dir1: root of a directory tree; if nonexistent, will raise OSError | |
99 dir2: root of another directory tree; if nonexistent, will raise OSError | |
100 ignore_subtree_names: list of subtree directory names to ignore; | |
101 defaults to ['.svn'], so all SVN files are ignores | |
102 | |
103 TODO(epoger): include the dirname within each filename (not just the | |
104 basename), to make it easier to locate any differences | |
105 """ | |
106 differing_files = [] | |
107 if ignore_subtree_names is None: | |
108 ignore_subtree_names = ['.svn'] | |
109 dircmp = filecmp.dircmp(dir1, dir2, ignore=ignore_subtree_names) | |
110 differing_files.extend(dircmp.left_only) | |
111 differing_files.extend(dircmp.right_only) | |
112 differing_files.extend(dircmp.common_funny) | |
113 differing_files.extend(dircmp.diff_files) | |
114 differing_files.extend(dircmp.funny_files) | |
115 for common_dir in dircmp.common_dirs: | |
116 differing_files.extend(find_different_files( | |
117 os.path.join(dir1, common_dir), os.path.join(dir2, common_dir))) | |
118 return differing_files | |
119 | |
120 | |
121 def main(): | 42 def main(): |
122 suite = unittest.TestLoader().loadTestsFromTestCase(ResultsTest) | 43 base_unittest.main(ResultsTest) |
123 unittest.TextTestRunner(verbosity=2).run(suite) | |
124 | 44 |
125 | 45 |
126 if __name__ == '__main__': | 46 if __name__ == '__main__': |
127 main() | 47 main() |
OLD | NEW |