| OLD | NEW |
| 1 #!/bin/bash | 1 #!/bin/bash |
| 2 | 2 |
| 3 # Tests for our tools. | 3 # Tests for our tools. |
| 4 # | 4 # |
| 5 # TODO: currently, this only passes on Linux (which is the platform that | 5 # TODO: currently, this only passes on Linux (which is the platform that |
| 6 # the housekeeper bot runs on, e.g. | 6 # the housekeeper bot runs on, e.g. |
| 7 # http://70.32.156.51:10117/builders/Skia_PerCommit_House_Keeping/builds/1415/st
eps/RunToolSelfTests/logs/stdio ) | 7 # http://70.32.156.51:10117/builders/Skia_PerCommit_House_Keeping/builds/1415/st
eps/RunToolSelfTests/logs/stdio ) |
| 8 # See https://code.google.com/p/skia/issues/detail?id=677 | 8 # See https://code.google.com/p/skia/issues/detail?id=677 |
| 9 # ('make tools/tests/run.sh work cross-platform') | 9 # ('make tools/tests/run.sh work cross-platform') |
| 10 # Ideally, these tests should pass on all development platforms... | 10 # Ideally, these tests should pass on all development platforms... |
| (...skipping 108 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 119 $COMMAND 2>$ACTUAL_OUTPUT_DIR/stderr | 119 $COMMAND 2>$ACTUAL_OUTPUT_DIR/stderr |
| 120 echo $? >$ACTUAL_OUTPUT_DIR/return_value | 120 echo $? >$ACTUAL_OUTPUT_DIR/return_value |
| 121 END_TIMESTAMP=$(date +%s) | 121 END_TIMESTAMP=$(date +%s) |
| 122 | 122 |
| 123 SECONDS_RUN=$(expr $END_TIMESTAMP - $START_TIMESTAMP) | 123 SECONDS_RUN=$(expr $END_TIMESTAMP - $START_TIMESTAMP) |
| 124 echo "check_bench_regressions.py took $SECONDS_RUN seconds to complete" | 124 echo "check_bench_regressions.py took $SECONDS_RUN seconds to complete" |
| 125 | 125 |
| 126 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR | 126 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR |
| 127 } | 127 } |
| 128 | 128 |
| 129 # Test rebaseline.py's JSON-format expectations rebaselining capability. | |
| 130 # | |
| 131 # Copy expected-results.json files from $1 into a dir where they can be modified
. | |
| 132 # Run rebaseline.py with arguments in $2, recording its output. | |
| 133 # Then compare the output (and modified expected-results.json files) to the | |
| 134 # content of $2/output-expected. | |
| 135 function rebaseline_test { | |
| 136 if [ $# != 3 ]; then | |
| 137 echo "rebaseline_test requires exactly 3 parameters, got $#" | |
| 138 exit 1 | |
| 139 fi | |
| 140 COPY_EXPECTATIONS_FROM_DIR="$1" | |
| 141 ARGS="$2" | |
| 142 ACTUAL_OUTPUT_DIR="$3/output-actual" | |
| 143 EXPECTED_OUTPUT_DIR="$3/output-expected" | |
| 144 | |
| 145 rm -rf $ACTUAL_OUTPUT_DIR | |
| 146 mkdir -p $ACTUAL_OUTPUT_DIR | |
| 147 EXPECTATIONS_TO_MODIFY_DIR="$ACTUAL_OUTPUT_DIR/gm-expectations" | |
| 148 BUILDERS=$(ls $COPY_EXPECTATIONS_FROM_DIR) | |
| 149 for BUILDER in $BUILDERS; do | |
| 150 mkdir -p $EXPECTATIONS_TO_MODIFY_DIR/$BUILDER | |
| 151 cp $COPY_EXPECTATIONS_FROM_DIR/$BUILDER/expected-results.json \ | |
| 152 $EXPECTATIONS_TO_MODIFY_DIR/$BUILDER | |
| 153 done | |
| 154 COMMAND="python tools/rebaseline.py --expectations-root $EXPECTATIONS_TO_MODIF
Y_DIR $ARGS" | |
| 155 echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line | |
| 156 $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout | |
| 157 echo $? >$ACTUAL_OUTPUT_DIR/return_value | |
| 158 | |
| 159 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR | |
| 160 } | |
| 161 | |
| 162 # Run jsondiff.py with arguments in $1, recording its output. | 129 # Run jsondiff.py with arguments in $1, recording its output. |
| 163 # Then compare that output to the content of $2/output-expected. | 130 # Then compare that output to the content of $2/output-expected. |
| 164 function jsondiff_test { | 131 function jsondiff_test { |
| 165 if [ $# != 2 ]; then | 132 if [ $# != 2 ]; then |
| 166 echo "jsondiff_test requires exactly 2 parameters, got $#" | 133 echo "jsondiff_test requires exactly 2 parameters, got $#" |
| 167 exit 1 | 134 exit 1 |
| 168 fi | 135 fi |
| 169 ARGS="$1" | 136 ARGS="$1" |
| 170 ACTUAL_OUTPUT_DIR="$2/output-actual" | 137 ACTUAL_OUTPUT_DIR="$2/output-actual" |
| 171 EXPECTED_OUTPUT_DIR="$2/output-expected" | 138 EXPECTED_OUTPUT_DIR="$2/output-expected" |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 210 # | 177 # |
| 211 | 178 |
| 212 # Parse a collection of bench data | 179 # Parse a collection of bench data |
| 213 PLATFORM=Perf-Android-Nexus7-Tegra3-Arm7-Release | 180 PLATFORM=Perf-Android-Nexus7-Tegra3-Arm7-Release |
| 214 REVISION=69c9e1a7261a3c8361e2b2c109d6340862149e34 | 181 REVISION=69c9e1a7261a3c8361e2b2c109d6340862149e34 |
| 215 download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_NO_INDIVIDU
AL_TILES" | 182 download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_NO_INDIVIDU
AL_TILES" |
| 216 download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_YES_INDIVID
UAL_TILES" | 183 download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_YES_INDIVID
UAL_TILES" |
| 217 benchalert_test $PLATFORM $REVISION | 184 benchalert_test $PLATFORM $REVISION |
| 218 | 185 |
| 219 # | 186 # |
| 220 # Test rebaseline.py ... | |
| 221 # | |
| 222 | |
| 223 REBASELINE_INPUT=tools/tests/rebaseline/input | |
| 224 REBASELINE_OUTPUT=tools/tests/rebaseline/output | |
| 225 rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/
json1 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Win7-ShuttleA-H
D2000-x86-Release" "$REBASELINE_OUTPUT/using-json1-expectations" | |
| 226 rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/
json1 --bugs 1234 5678 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Tes
t-Win7-ShuttleA-HD2000-x86-Release --notes notes_content --unreviewed" "$REBASEL
INE_OUTPUT/marked-unreviewed" | |
| 227 rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/
json1 --add-new --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Mac10
.6-MacMini4.1-GeForce320M-x86-Release Test-Win7-ShuttleA-HD2000-x86-Release" "$R
EBASELINE_OUTPUT/add-new" | |
| 228 | |
| 229 # | |
| 230 # Test jsondiff.py ... | 187 # Test jsondiff.py ... |
| 231 # | 188 # |
| 232 | 189 |
| 233 JSONDIFF_INPUT=tools/tests/jsondiff/input | 190 JSONDIFF_INPUT=tools/tests/jsondiff/input |
| 234 JSONDIFF_OUTPUT=tools/tests/jsondiff/output | 191 JSONDIFF_OUTPUT=tools/tests/jsondiff/output |
| 235 jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUT
PUT/old-vs-new" | 192 jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUT
PUT/old-vs-new" |
| 236 | 193 |
| 237 | 194 |
| 238 # | 195 # |
| 239 # Launch all the self-tests which have been written in Python. | 196 # Launch all the self-tests which have been written in Python. |
| 240 # | 197 # |
| 241 # TODO: Over time, we should move all of our tests into Python, and delete | 198 # TODO: Over time, we should move all of our tests into Python, and delete |
| 242 # the bash tests above. | 199 # the bash tests above. |
| 243 # See https://code.google.com/p/skia/issues/detail?id=677 | 200 # See https://code.google.com/p/skia/issues/detail?id=677 |
| 244 # ('make tools/tests/run.sh work cross-platform') | 201 # ('make tools/tests/run.sh work cross-platform') |
| 245 # | 202 # |
| 246 | 203 |
| 247 COMMAND="python tools/tests/run_all.py" | 204 COMMAND="python tools/tests/run_all.py" |
| 248 echo "$COMMAND" | 205 echo "$COMMAND" |
| 249 $COMMAND | 206 $COMMAND |
| 250 ret=$? | 207 ret=$? |
| 251 if [ $ret -ne 0 ]; then | 208 if [ $ret -ne 0 ]; then |
| 252 echo "failure in Python self-tests; see stack trace above" | 209 echo "failure in Python self-tests; see stack trace above" |
| 253 exit 1 | 210 exit 1 |
| 254 fi | 211 fi |
| 255 | 212 |
| 256 | 213 |
| 257 echo "All tests passed." | 214 echo "All tests passed." |
| OLD | NEW |