render_pictures: add --mismatchPath flag

tools/tests/render_pictures_test.py

 #!/usr/bin/python
 
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Test the render_pictures binary.
 """
@@ skipping 138 matching lines @@
     output_list.append(output_dict)
   return output_list
 
 
 class RenderPicturesTest(base_unittest.TestCase):
 
   def setUp(self):
     self.maxDiff = MAX_DIFF_LENGTH
     self._expectations_dir = tempfile.mkdtemp()
     self._input_skp_dir = tempfile.mkdtemp()
-    self._temp_dir = tempfile.mkdtemp()
+    # All output of render_pictures binary will go into this directory.
+    self._output_dir = tempfile.mkdtemp()
 
   def tearDown(self):
     shutil.rmtree(self._expectations_dir)
     shutil.rmtree(self._input_skp_dir)
-    shutil.rmtree(self._temp_dir)
+    shutil.rmtree(self._output_dir)
 
   def test_tiled_whole_image(self):
     """Run render_pictures with tiles and --writeWholeImage flag.
 
     TODO(epoger): This test generates undesired results!  The JSON summary
     includes both whole-image and tiled-images (as it should), but only
     whole-images are written out to disk.  See http://skbug.com/2463
+    Once I fix that, I should add a similar test that exercises mismatchPath.
 
     TODO(epoger): I noticed that when this is run without --writePath being
     specified, this test writes red_skp.png and green_skp.png to the current
     directory.  We should fix that... if --writePath is not specified, this
     probably shouldn't write out red_skp.png and green_skp.png at all!
     See http://skbug.com/2464
     """
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    write_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'writePath'))
     self._generate_skps()
     expectations_path = self._create_expectations()
     self._run_render_pictures([
         '-r', self._input_skp_dir,
         '--bbh', 'grid', '256', '256',
         '--mode', 'tile', '256', '256',
         '--readJsonSummaryPath', expectations_path,
         '--writeJsonSummaryPath', output_json_path,
-        '--writePath', self._temp_dir,
+        '--writePath', write_path_dir,
         '--writeWholeImage'])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 "tiled-images": RED_TILES,
                 "whole-image": RED_WHOLEIMAGE,
             },
             "green.skp": {
                 "tiled-images": GREEN_TILES,
                 "whole-image": GREEN_WHOLEIMAGE,
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(
-        self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json'])
+        write_path_dir, ['red_skp.png', 'green_skp.png'])
 
   def test_missing_tile_and_whole_image(self):
     """test_tiled_whole_image, but missing expectations for some images.
     """
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    write_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'writePath'))
     self._generate_skps()
     expectations_path = self._create_expectations(missing_some_images=True)
     self._run_render_pictures([
         '-r', self._input_skp_dir,
         '--bbh', 'grid', '256', '256',
         '--mode', 'tile', '256', '256',
         '--readJsonSummaryPath', expectations_path,
         '--writeJsonSummaryPath', output_json_path,
-        '--writePath', self._temp_dir,
+        '--writePath', write_path_dir,
         '--writeWholeImage'])
     modified_red_tiles = copy.deepcopy(RED_TILES)
     modified_red_tiles[5]['comparisonResult'] = 'no-comparison'
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 "tiled-images": modified_red_tiles,
                 "whole-image": modified_dict(
                     RED_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
             },
             "green.skp": {
                 "tiled-images": GREEN_TILES,
                 "whole-image": GREEN_WHOLEIMAGE,
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
 
   def test_untiled(self):
     """Run without tiles."""
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    write_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'writePath'))
     self._generate_skps()
     expectations_path = self._create_expectations()
     self._run_render_pictures([
         '-r', self._input_skp_dir,
         '--readJsonSummaryPath', expectations_path,
-        '--writePath', self._temp_dir,
+        '--writePath', write_path_dir,
         '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 "whole-image": RED_WHOLEIMAGE,
             },
             "green.skp": {
                 "whole-image": GREEN_WHOLEIMAGE,
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(
-        self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json'])
+        write_path_dir, ['red_skp.png', 'green_skp.png'])
 
   def test_untiled_writeChecksumBasedFilenames(self):
     """Same as test_untiled, but with --writeChecksumBasedFilenames."""
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    write_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'writePath'))
     self._generate_skps()
     self._run_render_pictures(['-r', self._input_skp_dir,
                                '--writeChecksumBasedFilenames',
-                               '--writePath', self._temp_dir,
+                               '--writePath', write_path_dir,
                                '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 # Manually verified: 640x400 red rectangle with black border
                 "whole-image": {
                     "checksumAlgorithm" : "bitmap-64bitMD5",
                     "checksumValue" : 11092453015575919668,
                     "comparisonResult" : "no-comparison",
                     "filepath" : "red_skp/bitmap-64bitMD5_11092453015575919668.png",
                 },
             },
             "green.skp": {
                 # Manually verified: 640x400 green rectangle with black border
                 "whole-image": {
                     "checksumAlgorithm" : "bitmap-64bitMD5",
                     "checksumValue" : 8891695120562235492,
                     "comparisonResult" : "no-comparison",
                     "filepath" : "green_skp/bitmap-64bitMD5_8891695120562235492.png",
                 },
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
-    self._assert_directory_contents(self._temp_dir, [
-        'red_skp', 'green_skp', 'actuals.json'])
+    self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp'])
     self._assert_directory_contents(
-        os.path.join(self._temp_dir, 'red_skp'),
+        os.path.join(write_path_dir, 'red_skp'),
         ['bitmap-64bitMD5_11092453015575919668.png'])
     self._assert_directory_contents(
-        os.path.join(self._temp_dir, 'green_skp'),
+        os.path.join(write_path_dir, 'green_skp'),
         ['bitmap-64bitMD5_8891695120562235492.png'])
 
   def test_untiled_validate(self):
     """Same as test_untiled, but with --validate."""
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    write_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'writePath'))
     self._generate_skps()
     expectations_path = self._create_expectations()
     self._run_render_pictures([
         '-r', self._input_skp_dir,
         '--readJsonSummaryPath', expectations_path,
         '--validate',
-        '--writePath', self._temp_dir,
+        '--writePath', write_path_dir,
         '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 "whole-image": RED_WHOLEIMAGE,
             },
             "green.skp": {
                 "whole-image": GREEN_WHOLEIMAGE,
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(
-        self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json'])
+        write_path_dir, ['red_skp.png', 'green_skp.png'])
 
   def test_untiled_without_writePath(self):
     """Same as test_untiled, but without --writePath."""
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
     self._generate_skps()
     expectations_path = self._create_expectations()
     self._run_render_pictures([
         '-r', self._input_skp_dir,
         '--readJsonSummaryPath', expectations_path,
         '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 "whole-image": RED_WHOLEIMAGE,
             },
             "green.skp": {
                 "whole-image": GREEN_WHOLEIMAGE,
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
 
   def test_tiled(self):
     """Generate individual tiles."""
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    write_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'writePath'))
     self._generate_skps()
     expectations_path = self._create_expectations()
     self._run_render_pictures([
         '-r', self._input_skp_dir,
         '--bbh', 'grid', '256', '256',
         '--mode', 'tile', '256', '256',
         '--readJsonSummaryPath', expectations_path,
-        '--writePath', self._temp_dir,
+        '--writePath', write_path_dir,
         '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 "tiled-images": RED_TILES,
             },
             "green.skp": {
                 "tiled-images": GREEN_TILES,
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(
-        self._temp_dir,
+        write_path_dir,
         ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png',
          'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png',
          'green_skp-tile0.png', 'green_skp-tile1.png', 'green_skp-tile2.png',
          'green_skp-tile3.png', 'green_skp-tile4.png', 'green_skp-tile5.png',
-         'actuals.json'])
+        ])
+
+  def test_tiled_mismatches(self):
+    """Same as test_tiled, but only write out mismatching images."""
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    mismatch_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'mismatchPath'))
+    self._generate_skps()
+    expectations_path = self._create_expectations()
+    self._run_render_pictures([
+        '-r', self._input_skp_dir,
+        '--bbh', 'grid', '256', '256',
+        '--mode', 'tile', '256', '256',
+        '--readJsonSummaryPath', expectations_path,
+        '--mismatchPath', mismatch_path_dir,
+        '--writeJsonSummaryPath', output_json_path])
+    expected_summary_dict = {
+        "header" : EXPECTED_HEADER_CONTENTS,
+        "actual-results" : {
+            "red.skp": {
+                "tiled-images": RED_TILES,
+            },
+            "green.skp": {
+                "tiled-images": GREEN_TILES,
+            }
+        }
+    }
+    self._assert_json_contents(output_json_path, expected_summary_dict)
+    self._assert_directory_contents(
+        mismatch_path_dir,
+        ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png',
+         'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png',
+        ])
 
   def test_tiled_writeChecksumBasedFilenames(self):
     """Same as test_tiled, but with --writeChecksumBasedFilenames."""
-    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    output_json_path = os.path.join(self._output_dir, 'actuals.json')
+    write_path_dir = self.create_empty_dir(
+        path=os.path.join(self._output_dir, 'writePath'))
     self._generate_skps()
     self._run_render_pictures(['-r', self._input_skp_dir,
                                '--bbh', 'grid', '256', '256',
                                '--mode', 'tile', '256', '256',
                                '--writeChecksumBasedFilenames',
-                               '--writePath', self._temp_dir,
+                               '--writePath', write_path_dir,
                                '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
             "red.skp": {
                 # Manually verified these 6 images, all 256x256 tiles,
                 # consistent with a tiled version of the 640x400 red rect
                 # with black borders.
                 "tiled-images": [{
                     "checksumAlgorithm" : "bitmap-64bitMD5",
@@ skipping 59 matching lines @@
                 }, {
                     "checksumAlgorithm" : "bitmap-64bitMD5",
                     "checksumValue" : 10673669813016809363,
                     "comparisonResult" : "no-comparison",
                     "filepath" : "green_skp/bitmap-64bitMD5_10673669813016809363.png",
                 }],
             }
         }
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
-    self._assert_directory_contents(self._temp_dir, [
-        'red_skp', 'green_skp', 'actuals.json'])
+    self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp'])
     self._assert_directory_contents(
-        os.path.join(self._temp_dir, 'red_skp'),
+        os.path.join(write_path_dir, 'red_skp'),
         ['bitmap-64bitMD5_5815827069051002745.png',
          'bitmap-64bitMD5_9323613075234140270.png',
          'bitmap-64bitMD5_16670399404877552232.png',
          'bitmap-64bitMD5_2507897274083364964.png',
          'bitmap-64bitMD5_7325267995523877959.png',
          'bitmap-64bitMD5_2181381724594493116.png'])
     self._assert_directory_contents(
-        os.path.join(self._temp_dir, 'green_skp'),
+        os.path.join(write_path_dir, 'green_skp'),
         ['bitmap-64bitMD5_12587324416545178013.png',
          'bitmap-64bitMD5_7624374914829746293.png',
          'bitmap-64bitMD5_5686489729535631913.png',
          'bitmap-64bitMD5_7980646035555096146.png',
          'bitmap-64bitMD5_17817086664365875131.png',
          'bitmap-64bitMD5_10673669813016809363.png'])
 
   def _run_render_pictures(self, args):
     binary = self.find_path_to_program('render_pictures')
     return self.run_command([binary,
@@ skipping 100 matching lines @@
     self.assertMultiLineEqual(prettyprinted_expected_dict,
                               prettyprinted_json_dict)
 
 
 def main():
   base_unittest.main(RenderPicturesTest)
 
 
 if __name__ == '__main__':
   main()