OLD | NEW |
(Empty) | |
| 1 #!/bin/bash |
| 2 |
| 3 # Self-tests for gm, based on tools/tests/run.sh |
| 4 # |
| 5 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit, |
| 6 # so make sure that they still pass when you make changes to gm! |
| 7 # |
| 8 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh |
| 9 # |
| 10 # TODO: because this is written as a shell script (instead of, say, Python) |
| 11 # it only runs on Linux and Mac. |
| 12 # See https://code.google.com/p/skia/issues/detail?id=677 |
| 13 # ('make tools/tests/run.sh work cross-platform') |
| 14 # Ideally, these tests should pass on all development platforms... |
| 15 # otherwise, how can developers be expected to test them before committing a |
| 16 # change? |
| 17 |
| 18 # cd into .../trunk so all the paths will work |
| 19 cd $(dirname $0)/../.. |
| 20 |
| 21 # TODO(epoger): make it look in Release and/or Debug |
| 22 GM_BINARY=out/Debug/gm |
| 23 |
| 24 OUTPUT_ACTUAL_SUBDIR=output-actual |
| 25 OUTPUT_EXPECTED_SUBDIR=output-expected |
| 26 CONFIGS="--config 8888 565" |
| 27 |
| 28 ENCOUNTERED_ANY_ERRORS=0 |
| 29 |
| 30 # Compare contents of all files within directories $1 and $2, |
| 31 # EXCEPT for any dotfiles. |
| 32 # If there are any differences, a description is written to stdout and |
| 33 # we exit with a nonzero return value. |
| 34 # Otherwise, we write nothing to stdout and return. |
| 35 function compare_directories { |
| 36 if [ $# != 2 ]; then |
| 37 echo "compare_directories requires exactly 2 parameters, got $#" |
| 38 exit 1 |
| 39 fi |
| 40 diff -r --exclude=.* $1 $2 |
| 41 if [ $? != 0 ]; then |
| 42 echo "failed in: compare_directories $1 $2" |
| 43 ENCOUNTERED_ANY_ERRORS=1 |
| 44 fi |
| 45 } |
| 46 |
| 47 # Run a command, and validate that it succeeds (returns 0). |
| 48 function assert_passes { |
| 49 COMMAND="$1" |
| 50 echo |
| 51 echo "assert_passes $COMMAND ..." |
| 52 $COMMAND |
| 53 if [ $? != 0 ]; then |
| 54 echo "This command was supposed to pass, but failed: [$COMMAND]" |
| 55 ENCOUNTERED_ANY_ERRORS=1 |
| 56 fi |
| 57 } |
| 58 |
| 59 # Run a command, and validate that it fails (returns nonzero). |
| 60 function assert_fails { |
| 61 COMMAND="$1" |
| 62 echo |
| 63 echo "assert_fails $COMMAND ..." |
| 64 $COMMAND |
| 65 if [ $? == 0 ]; then |
| 66 echo "This command was supposed to fail, but passed: [$COMMAND]" |
| 67 ENCOUNTERED_ANY_ERRORS=1 |
| 68 fi |
| 69 } |
| 70 |
| 71 # Run gm... |
| 72 # - with the arguments in $1 |
| 73 # - writing json summary into $2/$OUTPUT_ACTUAL_SUBDIR/json-summary.txt |
| 74 # - writing return value into $2/$OUTPUT_ACTUAL_SUBDIR/return_value |
| 75 # Then compare all of those against $2/$OUTPUT_EXPECTED_SUBDIR . |
| 76 function gm_test { |
| 77 if [ $# != 2 ]; then |
| 78 echo "gm_test requires exactly 2 parameters, got $#" |
| 79 exit 1 |
| 80 fi |
| 81 GM_ARGS="$1" |
| 82 ACTUAL_OUTPUT_DIR="$2/$OUTPUT_ACTUAL_SUBDIR" |
| 83 EXPECTED_OUTPUT_DIR="$2/$OUTPUT_EXPECTED_SUBDIR" |
| 84 JSON_SUMMARY_FILE="$ACTUAL_OUTPUT_DIR/json-summary.txt" |
| 85 |
| 86 rm -rf $ACTUAL_OUTPUT_DIR |
| 87 mkdir -p $ACTUAL_OUTPUT_DIR |
| 88 |
| 89 COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummaryPath $JSON_SUMMARY_FILE --write
Path $ACTUAL_OUTPUT_DIR/writePath --mismatchPath $ACTUAL_OUTPUT_DIR/mismatchPath
--missingExpectationsPath $ACTUAL_OUTPUT_DIR/missingExpectationsPath" |
| 90 |
| 91 $COMMAND |
| 92 echo $? >$ACTUAL_OUTPUT_DIR/return_value |
| 93 |
| 94 # Replace image file contents with just the filename, for two reasons: |
| 95 # 1. Image file encoding may vary by platform |
| 96 # 2. https://code.google.com/p/chromium/issues/detail?id=169600 |
| 97 # ('gcl/upload.py fail to upload binary files to rietveld') |
| 98 for IMAGEFILE in $(find $ACTUAL_OUTPUT_DIR -name \*.png); do |
| 99 echo "[contents of $IMAGEFILE]" >$IMAGEFILE |
| 100 done |
| 101 for IMAGEFILE in $(find $ACTUAL_OUTPUT_DIR -name \*.pdf); do |
| 102 echo "[contents of $IMAGEFILE]" >$IMAGEFILE |
| 103 done |
| 104 |
| 105 # Add a file to any empty subdirectories. |
| 106 for DIR in $(find $ACTUAL_OUTPUT_DIR -mindepth 1 -type d); do |
| 107 echo "Created additional file to make sure directory isn't empty, because se
lf-test cannot handle empty directories." >$DIR/bogusfile |
| 108 done |
| 109 |
| 110 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR |
| 111 } |
| 112 |
| 113 # Swap contents of two files at paths $1 and $2. |
| 114 function swap_files { |
| 115 if [ $# != 2 ]; then |
| 116 echo "swap_files requires exactly 2 parameters, got $#" |
| 117 exit 1 |
| 118 fi |
| 119 mv "$1" "$1.tmp" |
| 120 mv "$2" "$1" |
| 121 mv "$1.tmp" "$2" |
| 122 } |
| 123 |
| 124 # Create input dir (at path $1) with expectations (both image and json) |
| 125 # that gm will match or mismatch as appropriate. |
| 126 # |
| 127 # We used to check these files into SVN, but then we needed to rebaseline them |
| 128 # when our drawing changed at all... so, as proposed in |
| 129 # http://code.google.com/p/skia/issues/detail?id=1068 , we generate them |
| 130 # new each time. |
| 131 function create_inputs_dir { |
| 132 if [ $# != 1 ]; then |
| 133 echo "create_inputs_dir requires exactly 1 parameter, got $#" |
| 134 exit 1 |
| 135 fi |
| 136 INPUTS_DIR="$1" |
| 137 IMAGES_DIR=$INPUTS_DIR/images |
| 138 JSON_DIR=$INPUTS_DIR/json |
| 139 mkdir -p $IMAGES_DIR $JSON_DIR |
| 140 |
| 141 THIS_IMAGE_DIR=$IMAGES_DIR/identical-bytes |
| 142 mkdir -p $THIS_IMAGE_DIR |
| 143 # Run GM to write out the images actually generated. |
| 144 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -w $THIS_IMAGE_DIR |
| 145 # Run GM again to read in those images and write them out as a JSON summary. |
| 146 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ |
| 147 --writeJsonSummaryPath $JSON_DIR/identical-bytes.json |
| 148 |
| 149 THIS_IMAGE_DIR=$IMAGES_DIR/identical-pixels |
| 150 mkdir -p $THIS_IMAGE_DIR |
| 151 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -w $THIS_IMAGE_DIR |
| 152 echo "more bytes that do not change the image pixels" \ |
| 153 >> $THIS_IMAGE_DIR/8888/selftest1.png |
| 154 echo "more bytes that do not change the image pixels" \ |
| 155 >> $THIS_IMAGE_DIR/565/selftest1.png |
| 156 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ |
| 157 --writeJsonSummaryPath $JSON_DIR/identical-pixels.json |
| 158 |
| 159 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels |
| 160 mkdir -p $THIS_IMAGE_DIR |
| 161 $GM_BINARY --hierarchy --match selftest $CONFIGS -w $THIS_IMAGE_DIR |
| 162 swap_files $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.p
ng |
| 163 swap_files $THIS_IMAGE_DIR/565/selftest2.png $THIS_IMAGE_DIR/565/selftest1.pn
g |
| 164 $GM_BINARY --hierarchy --match selftest $CONFIGS -r $THIS_IMAGE_DIR \ |
| 165 --writeJsonSummaryPath $JSON_DIR/different-pixels.json |
| 166 |
| 167 # Create another JSON expectations file which is identical to |
| 168 # different-pixels.json, but in which the *first* ignore-failure is changed |
| 169 # from false to true. |
| 170 OLD='"ignore-failure" : false' |
| 171 NEW='"ignore-failure" : true' |
| 172 sed -e "0,/$OLD/{s/$OLD/$NEW/}" $JSON_DIR/different-pixels.json \ |
| 173 >$JSON_DIR/different-pixels-ignore-some-failures.json |
| 174 |
| 175 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy |
| 176 mkdir -p $THIS_IMAGE_DIR |
| 177 $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR |
| 178 mv $THIS_IMAGE_DIR/selftest2_8888.png $THIS_IMAGE_DIR/selftest1_8888.png |
| 179 mv $THIS_IMAGE_DIR/selftest2_565.png $THIS_IMAGE_DIR/selftest1_565.png |
| 180 $GM_BINARY --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ |
| 181 --writeJsonSummaryPath $JSON_DIR/different-pixels-no-hierarchy.json |
| 182 |
| 183 mkdir -p $IMAGES_DIR/empty-dir |
| 184 |
| 185 echo "# Comment line" >$GM_IGNORE_FAILURES_FILE |
| 186 echo "" >>$GM_IGNORE_FAILURES_FILE |
| 187 echo "# ignore any runs of the 'selftest1' test" >>$GM_IGNORE_FAILURES_FILE |
| 188 echo "selftest1" >>$GM_IGNORE_FAILURES_FILE |
| 189 echo "" >>$GM_IGNORE_FAILURES_FILE |
| 190 echo "# make sure we don't do partial matches (should NOT ignore 'selftest2' r
uns)" >>$GM_IGNORE_FAILURES_FILE |
| 191 echo "selftest" >>$GM_IGNORE_FAILURES_FILE |
| 192 } |
| 193 |
| 194 GM_TESTDIR=gm/tests |
| 195 GM_INPUTS=$GM_TESTDIR/inputs |
| 196 GM_OUTPUTS=$GM_TESTDIR/outputs |
| 197 GM_TEMPFILES=$GM_TESTDIR/tempfiles |
| 198 GM_IGNORE_FAILURES_FILE=$GM_INPUTS/ignored-tests.txt |
| 199 |
| 200 create_inputs_dir $GM_INPUTS |
| 201 |
| 202 # Compare generated image against an input image file with identical bytes. |
| 203 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/i
dentical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images" |
| 204 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide
ntical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json" |
| 205 |
| 206 # Compare generated image against an input image file with identical pixels but
different PNG encoding. |
| 207 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/i
dentical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images" |
| 208 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide
ntical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json" |
| 209 |
| 210 # Compare generated image against an input image file with different pixels. |
| 211 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/d
ifferent-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images" |
| 212 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/dif
ferent-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json" |
| 213 |
| 214 # Exercise --ignoreFailuresFile flag. |
| 215 # This should run two GM tests: selftest1 and selftest2. |
| 216 # Failures in selftest1 should be ignored, but failures in selftest2 should not. |
| 217 gm_test "--verbose --hierarchy --match selftest --ignoreFailuresFile $GM_IGNORE_
FAILURES_FILE $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ig
noring-one-test" |
| 218 |
| 219 # Compare different pixels, but with a SUBSET of the expectations marked as |
| 220 # ignore-failure. |
| 221 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/dif
ferent-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures" |
| 222 |
| 223 # Compare generated image against an empty "expected image" dir. |
| 224 # Even the tests that have been marked as ignore-failure (selftest1) should |
| 225 # show up as no-comparison. |
| 226 gm_test "--verbose --hierarchy --match selftest --ignoreFailuresFile $GM_IGNORE_
FAILURES_FILE $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-aga
inst-empty-dir" |
| 227 |
| 228 # Compare generated image against a nonexistent "expected image" dir. |
| 229 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r ../path/to/nowhere"
"$GM_OUTPUTS/compared-against-nonexistent-dir" |
| 230 |
| 231 # Compare generated image against an empty "expected image" dir, but NOT in verb
ose mode. |
| 232 gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir"
"$GM_OUTPUTS/nonverbose" |
| 233 |
| 234 # Add pdf to the list of configs. |
| 235 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json
/identical-bytes.json" "$GM_OUTPUTS/add-config-pdf" |
| 236 |
| 237 # Test what happens if run without -r (no expected-results.json to compare |
| 238 # against). |
| 239 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readp
ath" |
| 240 |
| 241 # Test what happens if a subset of the renderModes fail (e.g. pipe) |
| 242 gm_test "--pipe --simulatePipePlaybackFailure --verbose --hierarchy --match self
test1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playb
ack-failure" |
| 243 |
| 244 # Confirm that IntentionallySkipped tests are recorded as such. |
| 245 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUT
S/intentionally-skipped-tests" |
| 246 |
| 247 # Ignore some error types (including ExpectationsMismatch) |
| 248 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierar
chy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OU
TPUTS/ignore-expectations-mismatch" |
| 249 |
| 250 # Test non-hierarchical mode. |
| 251 gm_test "--verbose --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixel
s-no-hierarchy.json" "$GM_OUTPUTS/no-hierarchy" |
| 252 |
| 253 # Try writing out actual images using checksum-based filenames, like we do when |
| 254 # uploading to Google Storage. |
| 255 gm_test "--verbose --writeChecksumBasedFilenames --match selftest1 $CONFIGS -r $
GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/checksum-based-f
ilenames" |
| 256 |
| 257 # Exercise display_json_results.py |
| 258 PASSING_CASES="compared-against-identical-bytes-json compared-against-identical-
pixels-json" |
| 259 FAILING_CASES="compared-against-different-pixels-json" |
| 260 for CASE in $PASSING_CASES; do |
| 261 assert_passes "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXP
ECTED_SUBDIR/json-summary.txt" |
| 262 done |
| 263 for CASE in $FAILING_CASES; do |
| 264 assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPE
CTED_SUBDIR/json-summary.txt" |
| 265 done |
| 266 |
| 267 # Exercise all Python unittests. |
| 268 assert_passes "python gm/test_all.py" |
| 269 |
| 270 echo |
| 271 if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then |
| 272 echo "All tests passed." |
| 273 exit 0 |
| 274 else |
| 275 echo "Some tests failed." |
| 276 exit 1 |
| 277 fi |
OLD | NEW |