Add gm/confirm_no_failures_in_json.py

gm/tests/run.sh

 #!/bin/bash
 
 # Self-tests for gm, based on tools/tests/run.sh
 #
 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit,
 # so make sure that they still pass when you make changes to gm!
 #
 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh
 #
 # TODO: because this is written as a shell script (instead of, say, Python)
@@ skipping 26 matching lines @@
     echo "compare_directories requires exactly 2 parameters, got $#"
     exit 1
   fi
   diff -r --exclude=.* $1 $2
   if [ $? != 0 ]; then
     echo "failed in: compare_directories $1 $2"
     ENCOUNTERED_ANY_ERRORS=1
   fi
 }
 
+# Run a command, and validate that it succeeds (returns 0).
+function assert_passes {
+  COMMAND="$1"
+  OUTPUT=$($COMMAND 2>&1)
+  if [ $? != 0 ]; then
+    echo "This command was supposed to pass, but failed: [$COMMAND]"
+    echo $OUTPUT
+    ENCOUNTERED_ANY_ERRORS=1
+  fi
+}
+
+# Run a command, and validate that it fails (returns nonzero).
+function assert_fails {
+  COMMAND="$1"
+  OUTPUT=$($COMMAND 2>&1)
+  if [ $? == 0 ]; then
+    echo "This command was supposed to fail, but passed: [$COMMAND]"
+    echo $OUTPUT
+    ENCOUNTERED_ANY_ERRORS=1
+  fi
+}
+
 # Run gm...
 # - with the arguments in $1
 # - writing stdout into $2/$OUTPUT_ACTUAL_SUBDIR/stdout
 # - writing json summary into $2/$OUTPUT_ACTUAL_SUBDIR/json-summary.txt
 # - writing return value into $2/$OUTPUT_ACTUAL_SUBDIR/return_value
 # Then compare all of those against $2/$OUTPUT_EXPECTED_SUBDIR .
 function gm_test {
   if [ $# != 2 ]; then
     echo "gm_test requires exactly 2 parameters, got $#"
     exit 1
@@ skipping 122 matching lines @@
 
 # Test what happens if a subset of the renderModes fail (e.g. pipe)
 gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure"
 
 # Confirm that IntentionallySkipped tests are recorded as such.
 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
 
 # Ignore some error types (including ExpectationsMismatch)
 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignore-expectations-mismatch"
 
+# Exercise confirm_no_failures_in_json.py
+PASSING_CASES="compared-against-identical-bytes-json compared-against-identical-pixels-json"
+FAILING_CASES="compared-against-different-pixels-json"
+for CASE in $PASSING_CASES; do
+  assert_passes "python gm/confirm_no_failures_in_json.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt"
+done
+for CASE in $FAILING_CASES; do
+  assert_fails "python gm/confirm_no_failures_in_json.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt"
+done
+
 if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then
   echo "All tests passed."
   exit 0
 else
   exit 1
 fi