GM self-tests: always run all tests (don't stop at first failure)

gm/tests/run.sh

 #!/bin/bash
 
 # Self-tests for gm, based on tools/tests/run.sh
 #
 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit,
 # so make sure that they still pass when you make changes to gm!
 #
 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh
 #
 # TODO: because this is written as a shell script (instead of, say, Python)
 # it only runs on Linux and Mac.
 # See https://code.google.com/p/skia/issues/detail?id=677
 # ('make tools/tests/run.sh work cross-platform')
 # Ideally, these tests should pass on all development platforms...
 # otherwise, how can developers be expected to test them before committing a
 # change?
 
 # cd into .../trunk so all the paths will work
 cd $(dirname $0)/../..
 
 # TODO(epoger): make it look in Release and/or Debug
 GM_BINARY=out/Debug/gm
 
 OUTPUT_ACTUAL_SUBDIR=output-actual
 OUTPUT_EXPECTED_SUBDIR=output-expected
 CONFIGS="--config 8888 565"
 
+ENCOUNTERED_ANY_ERRORS=0
+
 # Compare contents of all files within directories $1 and $2,
 # EXCEPT for any dotfiles.
 # If there are any differences, a description is written to stdout and
 # we exit with a nonzero return value.
 # Otherwise, we write nothing to stdout and return.
 function compare_directories {
   if [ $# != 2 ]; then
     echo "compare_directories requires exactly 2 parameters, got $#"
     exit 1
   fi
   diff -r --exclude=.* $1 $2
   if [ $? != 0 ]; then
     echo "failed in: compare_directories $1 $2"
-    exit 1
+    ENCOUNTERED_ANY_ERRORS=1

[borenet, 2013/04/26 16:04:09]

Alternatively, you could add error messages to this variable:
ERRORS="${ERRORS}\n${NEW_ERROR_MSG}"

And test if it's empty at the end.

[epoger, 2013/04/26 16:10:55]

The question is, what would I put into $ERRORS that would be a useful summary?

I suppose I could pipe (or tee) the output of diff to $ERRORS, but I'm not sure
whether that output would make sense without the context provided by the output
generated as it goes.

I think it's best to just tell the human it failed, and trust her to scroll up
and see what happened, for now.  Maybe we can improve that later...

[borenet, 2013/04/26 16:12:43]

Fair enough.

   fi
 }
 
 # Run gm...
 # - with the arguments in $1
 # - writing stdout into $2/$OUTPUT_ACTUAL_SUBDIR/stdout
 # - writing json summary into $2/$OUTPUT_ACTUAL_SUBDIR/json-summary.txt
 # - writing return value into $2/$OUTPUT_ACTUAL_SUBDIR/return_value
 # Then compare all of those against $2/$OUTPUT_EXPECTED_SUBDIR .
 function gm_test {
   if [ $# != 2 ]; then
     echo "gm_test requires exactly 2 parameters, got $#"
     exit 1

[borenet, 2013/04/26 16:04:09]

What about here?

[epoger, 2013/04/26 16:10:55]

In this case, there was an error within the run.sh script, and I think we should
Stop The Presses.

   fi
   GM_ARGS="$1"
   ACTUAL_OUTPUT_DIR="$2/$OUTPUT_ACTUAL_SUBDIR"
   EXPECTED_OUTPUT_DIR="$2/$OUTPUT_EXPECTED_SUBDIR"
   JSON_SUMMARY_FILE="$ACTUAL_OUTPUT_DIR/json-summary.txt"
 
   rm -rf $ACTUAL_OUTPUT_DIR
   mkdir -p $ACTUAL_OUTPUT_DIR
   COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummaryPath $JSON_SUMMARY_FILE"
   echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
@@ skipping 16 matching lines @@
 # Create input dir (at path $1) with expectations (both image and json)
 # that gm will match or mismatch as appropriate.
 #
 # We used to check these files into SVN, but then we needed to rebasline them
 # when our drawing changed at all... so, as proposed in
 # http://code.google.com/p/skia/issues/detail?id=1068 , we generate them
 # new each time.
 function create_inputs_dir {
   if [ $# != 1 ]; then
     echo "create_inputs_dir requires exactly 1 parameter, got $#"
     exit 1

[borenet, 2013/04/26 16:04:09]

And here?

[epoger, 2013/04/26 16:10:55]

In this case, there was an error within the run.sh script, and I think we should
Stop The Presses.

   fi
   INPUTS_DIR="$1"
   IMAGES_DIR=$INPUTS_DIR/images
   JSON_DIR=$INPUTS_DIR/json
   mkdir -p $IMAGES_DIR $JSON_DIR
 
   mkdir -p $IMAGES_DIR/identical-bytes
   # Run GM to write out the images actually generated.
   $GM_BINARY --hierarchy --match selftest1 $CONFIGS \
     -w $IMAGES_DIR/identical-bytes
@@ skipping 62 matching lines @@
 
 # Test what happens if a subset of the renderModes fail (e.g. pipe)
 gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure"
 
 # Confirm that IntentionallySkipped tests are recorded as such.
 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
 
 # Ignore some error types (including ExpectationsMismatch)
 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignore-expectations-mismatch"
 
-echo "All tests passed."
+if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then
+  echo "All tests passed."
+  exit 0
+else
+  exit 1
+fi