OLD | NEW |
| (Empty) |
1 #!/bin/bash | |
2 | |
3 # Self-tests for gm, based on tools/tests/run.sh | |
4 # | |
5 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit, | |
6 # so make sure that they still pass when you make changes to gm! | |
7 # | |
8 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh | |
9 # | |
10 # TODO: because this is written as a shell script (instead of, say, Python) | |
11 # it only runs on Linux and Mac. | |
12 # See https://code.google.com/p/skia/issues/detail?id=677 | |
13 # ('make tools/tests/run.sh work cross-platform') | |
14 # Ideally, these tests should pass on all development platforms... | |
15 # otherwise, how can developers be expected to test them before committing a | |
16 # change? | |
17 | |
18 # cd into .../trunk so all the paths will work | |
19 cd $(dirname $0)/../.. | |
20 | |
21 # TODO(epoger): make it look in Release and/or Debug | |
22 GM_BINARY=out/Debug/gm | |
23 | |
24 OUTPUT_ACTUAL_SUBDIR=output-actual | |
25 OUTPUT_EXPECTED_SUBDIR=output-expected | |
26 CONFIGS="--config 8888 565" | |
27 | |
28 ENCOUNTERED_ANY_ERRORS=0 | |
29 | |
30 # Compare contents of all files within directories $1 and $2, | |
31 # EXCEPT for any dotfiles. | |
32 # If there are any differences, a description is written to stdout and | |
33 # we exit with a nonzero return value. | |
34 # Otherwise, we write nothing to stdout and return. | |
35 function compare_directories { | |
36 if [ $# != 2 ]; then | |
37 echo "compare_directories requires exactly 2 parameters, got $#" | |
38 exit 1 | |
39 fi | |
40 diff -r --exclude=.* $1 $2 | |
41 if [ $? != 0 ]; then | |
42 echo "failed in: compare_directories $1 $2" | |
43 ENCOUNTERED_ANY_ERRORS=1 | |
44 fi | |
45 } | |
46 | |
47 # Run a command, and validate that it succeeds (returns 0). | |
48 function assert_passes { | |
49 COMMAND="$1" | |
50 echo | |
51 echo "assert_passes $COMMAND ..." | |
52 $COMMAND | |
53 if [ $? != 0 ]; then | |
54 echo "This command was supposed to pass, but failed: [$COMMAND]" | |
55 ENCOUNTERED_ANY_ERRORS=1 | |
56 fi | |
57 } | |
58 | |
59 # Run a command, and validate that it fails (returns nonzero). | |
60 function assert_fails { | |
61 COMMAND="$1" | |
62 echo | |
63 echo "assert_fails $COMMAND ..." | |
64 $COMMAND | |
65 if [ $? == 0 ]; then | |
66 echo "This command was supposed to fail, but passed: [$COMMAND]" | |
67 ENCOUNTERED_ANY_ERRORS=1 | |
68 fi | |
69 } | |
70 | |
71 # Run gm... | |
72 # - with the arguments in $1 | |
73 # - writing json summary into $2/$OUTPUT_ACTUAL_SUBDIR/json-summary.txt | |
74 # - writing return value into $2/$OUTPUT_ACTUAL_SUBDIR/return_value | |
75 # Then compare all of those against $2/$OUTPUT_EXPECTED_SUBDIR . | |
76 function gm_test { | |
77 if [ $# != 2 ]; then | |
78 echo "gm_test requires exactly 2 parameters, got $#" | |
79 exit 1 | |
80 fi | |
81 GM_ARGS="$1" | |
82 ACTUAL_OUTPUT_DIR="$2/$OUTPUT_ACTUAL_SUBDIR" | |
83 EXPECTED_OUTPUT_DIR="$2/$OUTPUT_EXPECTED_SUBDIR" | |
84 JSON_SUMMARY_FILE="$ACTUAL_OUTPUT_DIR/json-summary.txt" | |
85 | |
86 rm -rf $ACTUAL_OUTPUT_DIR | |
87 mkdir -p $ACTUAL_OUTPUT_DIR | |
88 | |
89 COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummaryPath $JSON_SUMMARY_FILE --write
Path $ACTUAL_OUTPUT_DIR/writePath --mismatchPath $ACTUAL_OUTPUT_DIR/mismatchPath
--missingExpectationsPath $ACTUAL_OUTPUT_DIR/missingExpectationsPath" | |
90 | |
91 $COMMAND | |
92 echo $? >$ACTUAL_OUTPUT_DIR/return_value | |
93 | |
94 # Replace image file contents with just the filename, for two reasons: | |
95 # 1. Image file encoding may vary by platform | |
96 # 2. https://code.google.com/p/chromium/issues/detail?id=169600 | |
97 # ('gcl/upload.py fail to upload binary files to rietveld') | |
98 for IMAGEFILE in $(find $ACTUAL_OUTPUT_DIR -name \*.png); do | |
99 echo "[contents of $IMAGEFILE]" >$IMAGEFILE | |
100 done | |
101 for IMAGEFILE in $(find $ACTUAL_OUTPUT_DIR -name \*.pdf); do | |
102 echo "[contents of $IMAGEFILE]" >$IMAGEFILE | |
103 done | |
104 | |
105 # Add a file to any empty subdirectories. | |
106 for DIR in $(find $ACTUAL_OUTPUT_DIR -mindepth 1 -type d); do | |
107 echo "Created additional file to make sure directory isn't empty, because se
lf-test cannot handle empty directories." >$DIR/bogusfile | |
108 done | |
109 | |
110 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR | |
111 } | |
112 | |
113 # Swap contents of two files at paths $1 and $2. | |
114 function swap_files { | |
115 if [ $# != 2 ]; then | |
116 echo "swap_files requires exactly 2 parameters, got $#" | |
117 exit 1 | |
118 fi | |
119 mv "$1" "$1.tmp" | |
120 mv "$2" "$1" | |
121 mv "$1.tmp" "$2" | |
122 } | |
123 | |
124 # Create input dir (at path $1) with expectations (both image and json) | |
125 # that gm will match or mismatch as appropriate. | |
126 # | |
127 # We used to check these files into SVN, but then we needed to rebaseline them | |
128 # when our drawing changed at all... so, as proposed in | |
129 # http://code.google.com/p/skia/issues/detail?id=1068 , we generate them | |
130 # new each time. | |
131 function create_inputs_dir { | |
132 if [ $# != 1 ]; then | |
133 echo "create_inputs_dir requires exactly 1 parameter, got $#" | |
134 exit 1 | |
135 fi | |
136 INPUTS_DIR="$1" | |
137 IMAGES_DIR=$INPUTS_DIR/images | |
138 JSON_DIR=$INPUTS_DIR/json | |
139 mkdir -p $IMAGES_DIR $JSON_DIR | |
140 | |
141 THIS_IMAGE_DIR=$IMAGES_DIR/identical-bytes | |
142 mkdir -p $THIS_IMAGE_DIR | |
143 # Run GM to write out the images actually generated. | |
144 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -w $THIS_IMAGE_DIR | |
145 # Run GM again to read in those images and write them out as a JSON summary. | |
146 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ | |
147 --writeJsonSummaryPath $JSON_DIR/identical-bytes.json | |
148 | |
149 THIS_IMAGE_DIR=$IMAGES_DIR/identical-pixels | |
150 mkdir -p $THIS_IMAGE_DIR | |
151 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -w $THIS_IMAGE_DIR | |
152 echo "more bytes that do not change the image pixels" \ | |
153 >> $THIS_IMAGE_DIR/8888/selftest1.png | |
154 echo "more bytes that do not change the image pixels" \ | |
155 >> $THIS_IMAGE_DIR/565/selftest1.png | |
156 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ | |
157 --writeJsonSummaryPath $JSON_DIR/identical-pixels.json | |
158 | |
159 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels | |
160 mkdir -p $THIS_IMAGE_DIR | |
161 $GM_BINARY --hierarchy --match selftest $CONFIGS -w $THIS_IMAGE_DIR | |
162 swap_files $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.p
ng | |
163 swap_files $THIS_IMAGE_DIR/565/selftest2.png $THIS_IMAGE_DIR/565/selftest1.pn
g | |
164 $GM_BINARY --hierarchy --match selftest $CONFIGS -r $THIS_IMAGE_DIR \ | |
165 --writeJsonSummaryPath $JSON_DIR/different-pixels.json | |
166 | |
167 # Create another JSON expectations file which is identical to | |
168 # different-pixels.json, but in which the *first* ignore-failure is changed | |
169 # from false to true. | |
170 OLD='"ignore-failure" : false' | |
171 NEW='"ignore-failure" : true' | |
172 sed -e "0,/$OLD/{s/$OLD/$NEW/}" $JSON_DIR/different-pixels.json \ | |
173 >$JSON_DIR/different-pixels-ignore-some-failures.json | |
174 | |
175 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy | |
176 mkdir -p $THIS_IMAGE_DIR | |
177 $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR | |
178 mv $THIS_IMAGE_DIR/selftest2_8888.png $THIS_IMAGE_DIR/selftest1_8888.png | |
179 mv $THIS_IMAGE_DIR/selftest2_565.png $THIS_IMAGE_DIR/selftest1_565.png | |
180 $GM_BINARY --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ | |
181 --writeJsonSummaryPath $JSON_DIR/different-pixels-no-hierarchy.json | |
182 | |
183 mkdir -p $IMAGES_DIR/empty-dir | |
184 | |
185 echo "# Comment line" >$GM_IGNORE_FAILURES_FILE | |
186 echo "" >>$GM_IGNORE_FAILURES_FILE | |
187 echo "# ignore any runs of the 'selftest1' test" >>$GM_IGNORE_FAILURES_FILE | |
188 echo "selftest1" >>$GM_IGNORE_FAILURES_FILE | |
189 echo "" >>$GM_IGNORE_FAILURES_FILE | |
190 echo "# make sure we don't do partial matches (should NOT ignore 'selftest2' r
uns)" >>$GM_IGNORE_FAILURES_FILE | |
191 echo "selftest" >>$GM_IGNORE_FAILURES_FILE | |
192 } | |
193 | |
194 GM_TESTDIR=gm/tests | |
195 GM_INPUTS=$GM_TESTDIR/inputs | |
196 GM_OUTPUTS=$GM_TESTDIR/outputs | |
197 GM_TEMPFILES=$GM_TESTDIR/tempfiles | |
198 GM_IGNORE_FAILURES_FILE=$GM_INPUTS/ignored-tests.txt | |
199 | |
200 create_inputs_dir $GM_INPUTS | |
201 | |
202 # Compare generated image against an input image file with identical bytes. | |
203 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/i
dentical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images" | |
204 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide
ntical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json" | |
205 | |
206 # Compare generated image against an input image file with identical pixels but
different PNG encoding. | |
207 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/i
dentical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images" | |
208 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide
ntical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json" | |
209 | |
210 # Compare generated image against an input image file with different pixels. | |
211 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/d
ifferent-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images" | |
212 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/dif
ferent-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json" | |
213 | |
214 # Exercise --ignoreFailuresFile flag. | |
215 # This should run two GM tests: selftest1 and selftest2. | |
216 # Failures in selftest1 should be ignored, but failures in selftest2 should not. | |
217 gm_test "--verbose --hierarchy --match selftest --ignoreFailuresFile $GM_IGNORE_
FAILURES_FILE $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ig
noring-one-test" | |
218 | |
219 # Compare different pixels, but with a SUBSET of the expectations marked as | |
220 # ignore-failure. | |
221 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/dif
ferent-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures" | |
222 | |
223 # Compare generated image against an empty "expected image" dir. | |
224 # Even the tests that have been marked as ignore-failure (selftest1) should | |
225 # show up as no-comparison. | |
226 gm_test "--verbose --hierarchy --match selftest --ignoreFailuresFile $GM_IGNORE_
FAILURES_FILE $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-aga
inst-empty-dir" | |
227 | |
228 # Compare generated image against a nonexistent "expected image" dir. | |
229 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r ../path/to/nowhere"
"$GM_OUTPUTS/compared-against-nonexistent-dir" | |
230 | |
231 # Compare generated image against an empty "expected image" dir, but NOT in verb
ose mode. | |
232 gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir"
"$GM_OUTPUTS/nonverbose" | |
233 | |
234 # Add pdf to the list of configs. | |
235 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json
/identical-bytes.json" "$GM_OUTPUTS/add-config-pdf" | |
236 | |
237 # Test what happens if run without -r (no expected-results.json to compare | |
238 # against). | |
239 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readp
ath" | |
240 | |
241 # Test what happens if a subset of the renderModes fail (e.g. pipe) | |
242 gm_test "--pipe --simulatePipePlaybackFailure --verbose --hierarchy --match self
test1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playb
ack-failure" | |
243 | |
244 # Confirm that IntentionallySkipped tests are recorded as such. | |
245 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUT
S/intentionally-skipped-tests" | |
246 | |
247 # Ignore some error types (including ExpectationsMismatch) | |
248 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierar
chy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OU
TPUTS/ignore-expectations-mismatch" | |
249 | |
250 # Test non-hierarchical mode. | |
251 gm_test "--verbose --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixel
s-no-hierarchy.json" "$GM_OUTPUTS/no-hierarchy" | |
252 | |
253 # Try writing out actual images using checksum-based filenames, like we do when | |
254 # uploading to Google Storage. | |
255 gm_test "--verbose --writeChecksumBasedFilenames --match selftest1 $CONFIGS -r $
GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/checksum-based-f
ilenames" | |
256 | |
257 # Exercise display_json_results.py | |
258 PASSING_CASES="compared-against-identical-bytes-json compared-against-identical-
pixels-json" | |
259 FAILING_CASES="compared-against-different-pixels-json" | |
260 for CASE in $PASSING_CASES; do | |
261 assert_passes "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXP
ECTED_SUBDIR/json-summary.txt" | |
262 done | |
263 for CASE in $FAILING_CASES; do | |
264 assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPE
CTED_SUBDIR/json-summary.txt" | |
265 done | |
266 | |
267 # Exercise all Python unittests. | |
268 assert_passes "python gm/test_all.py" | |
269 | |
270 echo | |
271 if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then | |
272 echo "All tests passed." | |
273 exit 0 | |
274 else | |
275 echo "Some tests failed." | |
276 exit 1 | |
277 fi | |
OLD | NEW |