Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(79)

Side by Side Diff: gm/tests/run.sh

Issue 23799004: gm: add self-test to confirm that ignore-failure field is respected (Closed) Base URL: http://skia.googlecode.com/svn/trunk/
Patch Set: Created 7 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/bin/bash 1 #!/bin/bash
2 2
3 # Self-tests for gm, based on tools/tests/run.sh 3 # Self-tests for gm, based on tools/tests/run.sh
4 # 4 #
5 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit, 5 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit,
6 # so make sure that they still pass when you make changes to gm! 6 # so make sure that they still pass when you make changes to gm!
7 # 7 #
8 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh 8 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh
9 # 9 #
10 # TODO: because this is written as a shell script (instead of, say, Python) 10 # TODO: because this is written as a shell script (instead of, say, Python)
(...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after
156 --writeJsonSummaryPath $JSON_DIR/identical-pixels.json 156 --writeJsonSummaryPath $JSON_DIR/identical-pixels.json
157 157
158 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels 158 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels
159 mkdir -p $THIS_IMAGE_DIR 159 mkdir -p $THIS_IMAGE_DIR
160 $GM_BINARY --hierarchy --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR 160 $GM_BINARY --hierarchy --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR
161 mv $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.png 161 mv $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.png
162 mv $THIS_IMAGE_DIR/565/selftest2.png $THIS_IMAGE_DIR/565/selftest1.png 162 mv $THIS_IMAGE_DIR/565/selftest2.png $THIS_IMAGE_DIR/565/selftest1.png
163 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ 163 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \
164 --writeJsonSummaryPath $JSON_DIR/different-pixels.json 164 --writeJsonSummaryPath $JSON_DIR/different-pixels.json
165 165
166 # TODO: for now, this just copies in the different-pixels.json input file
epoger 2013/08/30 19:32:03 Patchset 1 adds a new "different-pixels-ignore-som
167 # verbatim. In the next patchset, I will use "sed" to change one
168 # ignore-failure from false to true.
169 cp $JSON_DIR/different-pixels.json $JSON_DIR/different-pixels-ignore-some-fail ures.json
170
166 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy 171 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy
167 mkdir -p $THIS_IMAGE_DIR 172 mkdir -p $THIS_IMAGE_DIR
168 $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR 173 $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR
169 mv $THIS_IMAGE_DIR/selftest2_8888.png $THIS_IMAGE_DIR/selftest1_8888.png 174 mv $THIS_IMAGE_DIR/selftest2_8888.png $THIS_IMAGE_DIR/selftest1_8888.png
170 mv $THIS_IMAGE_DIR/selftest2_565.png $THIS_IMAGE_DIR/selftest1_565.png 175 mv $THIS_IMAGE_DIR/selftest2_565.png $THIS_IMAGE_DIR/selftest1_565.png
171 $GM_BINARY --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ 176 $GM_BINARY --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \
172 --writeJsonSummaryPath $JSON_DIR/different-pixels-no-hierarchy.json 177 --writeJsonSummaryPath $JSON_DIR/different-pixels-no-hierarchy.json
173 178
174 mkdir -p $IMAGES_DIR/empty-dir 179 mkdir -p $IMAGES_DIR/empty-dir
175 } 180 }
(...skipping 10 matching lines...) Expand all
186 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide ntical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json" 191 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide ntical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json"
187 192
188 # Compare generated image against an input image file with identical pixels but different PNG encoding. 193 # Compare generated image against an input image file with identical pixels but different PNG encoding.
189 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/i dentical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images" 194 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/i dentical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
190 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide ntical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json" 195 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide ntical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json"
191 196
192 # Compare generated image against an input image file with different pixels. 197 # Compare generated image against an input image file with different pixels.
193 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/d ifferent-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images" 198 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/d ifferent-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
194 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/dif ferent-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json" 199 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/dif ferent-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
195 200
201 # Compare different pixels, but with a SUBSET of the expectations marked as
202 # ignore-failure.
203 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/dif ferent-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures"
204
196 # Compare generated image against an empty "expected image" dir. 205 # Compare generated image against an empty "expected image" dir.
197 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/e mpty-dir" "$GM_OUTPUTS/compared-against-empty-dir" 206 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/e mpty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
198 207
199 # Compare generated image against a nonexistent "expected image" dir. 208 # Compare generated image against a nonexistent "expected image" dir.
200 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r ../path/to/nowhere" "$GM_OUTPUTS/compared-against-nonexistent-dir" 209 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r ../path/to/nowhere" "$GM_OUTPUTS/compared-against-nonexistent-dir"
201 210
202 # Compare generated image against an empty "expected image" dir, but NOT in verb ose mode. 211 # Compare generated image against an empty "expected image" dir, but NOT in verb ose mode.
203 gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/nonverbose" 212 gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/nonverbose"
204 213
205 # Add pdf to the list of configs. 214 # Add pdf to the list of configs.
206 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json /identical-bytes.json" "$GM_OUTPUTS/add-config-pdf" 215 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json /identical-bytes.json" "$GM_OUTPUTS/add-config-pdf"
207 216
208 # If run without "-r", the JSON's "actual-results" section should contain 217 # Test what happens if run without -r (no expected-results.json to compare
209 # actual checksums marked as "failure-ignored", but the "expected-results" 218 # against).
210 # section should be empty.
211 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readp ath" 219 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readp ath"
212 220
213 # Test what happens if a subset of the renderModes fail (e.g. pipe) 221 # Test what happens if a subset of the renderModes fail (e.g. pipe)
214 gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $ CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-fai lure" 222 gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $ CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-fai lure"
215 223
216 # Confirm that IntentionallySkipped tests are recorded as such. 224 # Confirm that IntentionallySkipped tests are recorded as such.
217 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUT S/intentionally-skipped-tests" 225 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUT S/intentionally-skipped-tests"
218 226
219 # Ignore some error types (including ExpectationsMismatch) 227 # Ignore some error types (including ExpectationsMismatch)
220 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierar chy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OU TPUTS/ignore-expectations-mismatch" 228 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierar chy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OU TPUTS/ignore-expectations-mismatch"
(...skipping 14 matching lines...) Expand all
235 for CASE in $FAILING_CASES; do 243 for CASE in $FAILING_CASES; do
236 assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPE CTED_SUBDIR/json-summary.txt" 244 assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPE CTED_SUBDIR/json-summary.txt"
237 done 245 done
238 246
239 if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then 247 if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then
240 echo "All tests passed." 248 echo "All tests passed."
241 exit 0 249 exit 0
242 else 250 else
243 exit 1 251 exit 1
244 fi 252 fi
OLDNEW
« no previous file with comments | « gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698