Truly ignore failures in skimage.

tools/tests/skimage_self_test.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Self-test for skimage.
 
 import filecmp
 import os
 import subprocess
@@ skipping 21 matching lines @@
             return binary_full_path
     raise BinaryNotFoundException
 
 # Quit early if two files have different content.
 def DieIfFilesMismatch(expected, actual):
     if not filecmp.cmp(expected, actual):
         print 'Error: file mismatch! expected=%s , actual=%s' % (
             expected, actual)
         exit(1)
 
+def test_invalid_file(file_dir, skimage_binary):
+    """ Test the return value of skimage when an invalid file is decoded.
+        If there is no expectation file, or the file expects a particular
+        result, skimage should return nonzero indicating failure.
+        If the file has no expectation, or ignore-failure is set to true,
+        skimage should return zero indicating success. """
+    invalid_file = os.path.join(file_dir, "skimage", "input", "bad-images",
+                                "invalid.png")
+    # No expectations file:
+    args = [skimage_binary, "--readPath", invalid_file]
+    result = subprocess.call(args)
+    if 0 == result:
+      print "'%s' should have reported failure!" % " ".join(args)
+      exit(1)
+
+    # Directory holding all expectations files
+    expectations_dir = os.path.join(file_dir, "skimage", "input", "bad-images")
+
+    # Expectations file expecting a valid decode:
+    incorrect_expectations = os.path.join(expectations_dir,
+                                          "incorrect-results.json")
+    args = [skimage_binary, "--readPath", invalid_file,
+            "--readExpectationsPath", incorrect_expectations]
+    result = subprocess.call(args)
+    if 0 == result:
+      print "'%s' should have reported failure!" % " ".join(args)
+      exit(1)
+
+    # Empty expectations:
+    empty_expectations = os.path.join(expectations_dir, "empty-results.json")
+    subprocess.check_call([skimage_binary, "--readPath", invalid_file,
+                           "--readExpectationsPath", empty_expectations])
+
+    # Ignore failure:
+    ignore_expectations = os.path.join(expectations_dir, "ignore-results.json")
+    subprocess.check_call([skimage_binary, "--readPath", invalid_file,
+                           "--readExpectationsPath", ignore_expectations])
+
+def test_incorrect_expectations(file_dir, skimage_binary):
+    """ Test that comparing to incorrect expectations fails, unless
+        ignore-failures is set to true. """
+    valid_file = os.path.join(file_dir, "skimage", "input",
+                                    "images-with-known-hashes",
+                                    "1209453360120438698.png")
+    expectations_dir = os.path.join(file_dir, "skimage", "input",
+                                    "images-with-known-hashes")
+
+    incorrect_results = os.path.join(expectations_dir,
+                                     "incorrect-results.json")
+    args = [skimage_binary, "--readPath", valid_file, "--readExpectationsPath",
+            incorrect_results]
+    result = subprocess.call(args)
+    if 0 == result:
+      print "'%s' should have reported failure!" % " ".join(args)
+      exit(1)
+
+    ignore_results = os.path.join(expectations_dir, "ignore-failures.json")
+    subprocess.check_call([skimage_binary, "--readPath", valid_file,
+                           "--readExpectationsPath", ignore_results])
+
 def main():
     # Use the directory of this file as the out directory
     file_dir = os.path.abspath(os.path.dirname(__file__))
 
     trunk_dir = os.path.normpath(os.path.join(file_dir, os.pardir, os.pardir))
 
     # Find the binary
     skimage_binary = PickBinaryPath(trunk_dir)
     print "Running " + skimage_binary
 
     # Generate an expectations file from known images.
     images_dir = os.path.join(file_dir, "skimage", "input",
                               "images-with-known-hashes")
     expectations_path = os.path.join(file_dir, "skimage", "output-actual",
                                      "create-expectations", "expectations.json")
     subprocess.check_call([skimage_binary, "--readPath", images_dir,
                            "--createExpectationsPath", expectations_path])
 
     # Make sure the expectations file was generated correctly.
     golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
                                        "create-expectations",
                                        "expectations.json")
     DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
 
     # Tell skimage to read back the expectations file it just wrote, and
     # confirm that the images in images_dir match it.
     subprocess.check_call([skimage_binary, "--readPath", images_dir,
                            "--readExpectationsPath", expectations_path])
 
-    # TODO(scroggo): Add a test that compares expectations and image files that
-    # are known to NOT match, and make sure it returns an error.
+    test_incorrect_expectations(file_dir=file_dir,
+                                skimage_binary=skimage_binary)
 
     # Generate an expectations file from an empty directory.
     empty_dir = tempfile.mkdtemp()
     expectations_path = os.path.join(file_dir, "skimage", "output-actual",
                                      "empty-dir", "expectations.json")
     subprocess.check_call([skimage_binary, "--readPath", empty_dir,
                            "--createExpectationsPath", expectations_path])
     golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
                                        "empty-dir", "expectations.json")
     DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
     os.rmdir(empty_dir)
 
     # Generate an expectations file from a nonexistent directory.
     expectations_path = os.path.join(file_dir, "skimage", "output-actual",
                                      "nonexistent-dir", "expectations.json")
     subprocess.check_call([skimage_binary, "--readPath", "/nonexistent/dir",
                            "--createExpectationsPath", expectations_path])
     golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
                                        "nonexistent-dir", "expectations.json")
     DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
 
+    test_invalid_file(file_dir=file_dir, skimage_binary=skimage_binary)
+
     # Done with all tests.
     print "Self tests succeeded!"
 
 if __name__ == "__main__":
     main()