gm: add self-test to confirm that ignore-failure field is respected

gm/tests/run.sh

 #!/bin/bash
 
 # Self-tests for gm, based on tools/tests/run.sh
 #
 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit,
 # so make sure that they still pass when you make changes to gm!
 #
 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh
 #
 # TODO: because this is written as a shell script (instead of, say, Python)
@@ skipping 145 matching lines @@
     --writeJsonSummaryPath $JSON_DIR/identical-pixels.json
 
   THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels
   mkdir -p $THIS_IMAGE_DIR
   $GM_BINARY --hierarchy --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR
   mv $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.png
   mv $THIS_IMAGE_DIR/565/selftest2.png  $THIS_IMAGE_DIR/565/selftest1.png
   $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \
     --writeJsonSummaryPath $JSON_DIR/different-pixels.json
 
+  # Create another JSON expectations file which is identical to
+  # different-pixels.json, but in which the *first* ignore-failure is changed
+  # from false to true.
+  OLD='"ignore-failure" : false'
+  NEW='"ignore-failure" : true'
+  sed -e "0,/$OLD/{s/$OLD/$NEW/}" $JSON_DIR/different-pixels.json \

[epoger, 2013/08/30 19:53:02]

Patchset 2 modified the input file such that exactly ONE of the tests is marked
as "ignore-failure":true.

+      >$JSON_DIR/different-pixels-ignore-some-failures.json
+
   THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy
   mkdir -p $THIS_IMAGE_DIR
   $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR
   mv $THIS_IMAGE_DIR/selftest2_8888.png $THIS_IMAGE_DIR/selftest1_8888.png
   mv $THIS_IMAGE_DIR/selftest2_565.png  $THIS_IMAGE_DIR/selftest1_565.png
   $GM_BINARY --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \
     --writeJsonSummaryPath $JSON_DIR/different-pixels-no-hierarchy.json
 
   mkdir -p $IMAGES_DIR/empty-dir
 }
@@ skipping 10 matching lines @@
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json"
 
 # Compare generated image against an input image file with identical pixels but different PNG encoding.
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json"
 
 # Compare generated image against an input image file with different pixels.
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
 
+# Compare different pixels, but with a SUBSET of the expectations marked as
+# ignore-failure.
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures"
+
 # Compare generated image against an empty "expected image" dir.
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
 
 # Compare generated image against a nonexistent "expected image" dir.
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r ../path/to/nowhere" "$GM_OUTPUTS/compared-against-nonexistent-dir"
 
 # Compare generated image against an empty "expected image" dir, but NOT in verbose mode.
 gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/nonverbose"
 
 # Add pdf to the list of configs.
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/add-config-pdf"
 
-# If run without "-r", the JSON's "actual-results" section should contain
-# actual checksums marked as "failure-ignored", but the "expected-results"
-# section should be empty.
+# Test what happens if run without -r (no expected-results.json to compare
+# against).
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath"
 
 # Test what happens if a subset of the renderModes fail (e.g. pipe)
 gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure"
 
 # Confirm that IntentionallySkipped tests are recorded as such.
 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
 
 # Ignore some error types (including ExpectationsMismatch)
 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignore-expectations-mismatch"
@@ skipping 14 matching lines @@
 for CASE in $FAILING_CASES; do
   assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt"
 done
 
 if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then
   echo "All tests passed."
   exit 0
 else
   exit 1
 fi