| OLD | NEW | 
|    1 #!/usr/bin/python |    1 #!/usr/bin/python | 
|    2  |    2  | 
|    3 """ |    3 """ | 
|    4 Copyright 2013 Google Inc. |    4 Copyright 2013 Google Inc. | 
|    5  |    5  | 
|    6 Use of this source code is governed by a BSD-style license that can be |    6 Use of this source code is governed by a BSD-style license that can be | 
|    7 found in the LICENSE file. |    7 found in the LICENSE file. | 
|    8  |    8  | 
|    9 Test results.py |    9 Test results.py | 
|   10  |   10  | 
|   11 TODO(epoger): Create a command to update the expected results (in |   11 TODO(epoger): Create a command to update the expected results (in | 
|   12 OUTPUT_DIR_EXPECTED) when appropriate.  For now, you should: |   12 self._output_dir_expected) when appropriate.  For now, you should: | 
|   13 1. examine the results in OUTPUT_DIR_ACTUAL and make sure they are ok |   13 1. examine the results in self._output_dir_actual and make sure they are ok | 
|   14 2. rm -rf OUTPUT_DIR_EXPECTED |   14 2. rm -rf self._output_dir_expected | 
|   15 3. mv OUTPUT_DIR_ACTUAL OUTPUT_DIR_EXPECTED |   15 3. mv self._output_dir_actual self._output_dir_expected | 
|   16 Although, if you're using an SVN checkout, this will blow away .svn directories |   16 Although, if you're using an SVN checkout, this will blow away .svn directories | 
|   17 within OUTPUT_DIR_EXPECTED, which wouldn't be good... |   17 within self._output_dir_expected, which wouldn't be good... | 
|   18  |   18  | 
|   19 """ |   19 """ | 
|   20  |   20  | 
|   21 import filecmp |  | 
|   22 import os |   21 import os | 
|   23 import shutil |  | 
|   24 import sys |   22 import sys | 
|   25 import tempfile |  | 
|   26 import unittest |  | 
|   27  |   23  | 
|   28 # Imports from within Skia |   24 # Imports from within Skia | 
 |   25 import base_unittest | 
|   29 import results |   26 import results | 
|   30 import gm_json  # must import results first, so that gm_json will be in sys.path |   27 import gm_json  # must import results first, so that gm_json will be in sys.path | 
|   31  |   28  | 
|   32 PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) |  | 
|   33 INPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'inputs') |  | 
|   34 OUTPUT_DIR_ACTUAL   = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual') |  | 
|   35 OUTPUT_DIR_EXPECTED = os.path.join(PARENT_DIR, 'tests', 'outputs', 'expected') |  | 
|   36  |   29  | 
|   37  |   30 class ResultsTest(base_unittest.TestCase): | 
|   38 class ResultsTest(unittest.TestCase): |  | 
|   39  |  | 
|   40   def setUp(self): |  | 
|   41     self._temp_dir = tempfile.mkdtemp() |  | 
|   42     self._output_dir_actual   = os.path.join(OUTPUT_DIR_ACTUAL, self.id()) |  | 
|   43     self._output_dir_expected = os.path.join(OUTPUT_DIR_EXPECTED, self.id()) |  | 
|   44     create_empty_dir(self._output_dir_actual) |  | 
|   45  |  | 
|   46   def tearDown(self): |  | 
|   47     shutil.rmtree(self._temp_dir) |  | 
|   48     different_files = find_different_files(self._output_dir_actual, |  | 
|   49                                            self._output_dir_expected) |  | 
|   50     # Maybe we should move this assert elsewhere?  It's unusual to see an |  | 
|   51     # assert within tearDown(), but my thinking was: |  | 
|   52     # 1. Every test case will have some collection of output files that need to |  | 
|   53     #    be validated. |  | 
|   54     # 2. So put that validation within tearDown(), which will be called after |  | 
|   55     #    every test case! |  | 
|   56     # |  | 
|   57     # I have confirmed that the test really does fail if this assert is |  | 
|   58     # triggered. |  | 
|   59     # |  | 
|   60     # Ravi notes: if somebody later comes along and adds cleanup code below the |  | 
|   61     # assert, then if tests fail, the artifacts will not be cleaned up. |  | 
|   62     assert (not different_files), \ |  | 
|   63       ('found differing files between actual dir %s and expected dir %s: %s' % |  | 
|   64        (self._output_dir_actual, self._output_dir_expected, different_files)) |  | 
|   65  |  | 
|   66   def shortDescription(self): |  | 
|   67     """Tell unittest framework to not print docstrings for test cases.""" |  | 
|   68     return None |  | 
|   69  |   31  | 
|   70   def test_gm(self): |   32   def test_gm(self): | 
|   71     """Process results of a GM run with the Results object.""" |   33     """Process results of a GM run with the Results object.""" | 
|   72     results_obj = results.Results( |   34     results_obj = results.Results( | 
|   73         actuals_root=os.path.join(INPUT_DIR, 'gm-actuals'), |   35         actuals_root=os.path.join(self._input_dir, 'gm-actuals'), | 
|   74         expected_root=os.path.join(INPUT_DIR, 'gm-expectations'), |   36         expected_root=os.path.join(self._input_dir, 'gm-expectations'), | 
|   75         generated_images_root=self._temp_dir) |   37         generated_images_root=self._temp_dir) | 
|   76     gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL), |   38     gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL), | 
|   77                         os.path.join(self._output_dir_actual, 'gm.json')) |   39                         os.path.join(self._output_dir_actual, 'gm.json')) | 
|   78  |   40  | 
|   79  |   41  | 
|   80 # TODO(epoger): create_empty_dir(), find_different_files(), etc. should be |  | 
|   81 # extracted from this file to some common location, where they can be shared |  | 
|   82 # with download_test.py and other users. |  | 
|   83  |  | 
|   84 def create_empty_dir(path): |  | 
|   85   """Create an empty directory at the given path.""" |  | 
|   86   if os.path.isdir(path): |  | 
|   87     shutil.rmtree(path) |  | 
|   88   elif os.path.lexists(path): |  | 
|   89     os.remove(path) |  | 
|   90   os.makedirs(path) |  | 
|   91  |  | 
|   92  |  | 
|   93 def find_different_files(dir1, dir2, ignore_subtree_names=None): |  | 
|   94   """Returns a list of any files that differ between the directory trees rooted |  | 
|   95   at dir1 and dir2. |  | 
|   96  |  | 
|   97   Args: |  | 
|   98     dir1: root of a directory tree; if nonexistent, will raise OSError |  | 
|   99     dir2: root of another directory tree; if nonexistent, will raise OSError |  | 
|  100     ignore_subtree_names: list of subtree directory names to ignore; |  | 
|  101           defaults to ['.svn'], so all SVN files are ignores |  | 
|  102  |  | 
|  103   TODO(epoger): include the dirname within each filename (not just the |  | 
|  104   basename), to make it easier to locate any differences |  | 
|  105   """ |  | 
|  106   differing_files = [] |  | 
|  107   if ignore_subtree_names is None: |  | 
|  108     ignore_subtree_names = ['.svn'] |  | 
|  109   dircmp = filecmp.dircmp(dir1, dir2, ignore=ignore_subtree_names) |  | 
|  110   differing_files.extend(dircmp.left_only) |  | 
|  111   differing_files.extend(dircmp.right_only) |  | 
|  112   differing_files.extend(dircmp.common_funny) |  | 
|  113   differing_files.extend(dircmp.diff_files) |  | 
|  114   differing_files.extend(dircmp.funny_files) |  | 
|  115   for common_dir in dircmp.common_dirs: |  | 
|  116     differing_files.extend(find_different_files( |  | 
|  117         os.path.join(dir1, common_dir), os.path.join(dir2, common_dir))) |  | 
|  118   return differing_files |  | 
|  119  |  | 
|  120  |  | 
|  121 def main(): |   42 def main(): | 
|  122   suite = unittest.TestLoader().loadTestsFromTestCase(ResultsTest) |   43   base_unittest.main(ResultsTest) | 
|  123   unittest.TextTestRunner(verbosity=2).run(suite) |  | 
|  124  |   44  | 
|  125  |   45  | 
|  126 if __name__ == '__main__': |   46 if __name__ == '__main__': | 
|  127   main() |   47   main() | 
| OLD | NEW |