OLD | NEW |
| (Empty) |
1 #!/bin/bash | |
2 # | |
3 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
4 # for details. All rights reserved. Use of this source code is governed by a | |
5 # BSD-style license that can be found in the LICENSE file. | |
6 | |
7 # Compares a series of compiler reivsions against a target application for | |
8 # statistic collection. Outputs a gnuplot consumable table. | |
9 | |
10 APP="" | |
11 REV="" | |
12 ONE_DELTA="" | |
13 BASE_PATH=$(pwd) | |
14 SCRIPT_PATH=$(dirname $0) | |
15 SCRIPT_PATH=$(cd $SCRIPT_PATH; pwd) | |
16 RUNS=10 | |
17 COUNT=50 | |
18 LOW_REV="" | |
19 | |
20 function printHelp() { | |
21 exitValue=${1:1} | |
22 echo "Compare performance of multiple compiler revisions against a given targe
t application." | |
23 echo "Creates a cache of pre-built compiler revisions in compiler/revs/ for la
ter comparison." | |
24 echo "The target output of this script is a gnuplot consumable list of stats (
for now), located" | |
25 echo "in tmp_performance_comparisons/compiler_plots.dat." | |
26 echo "" | |
27 echo " Usage:" | |
28 echo " -a=, --app= The dart app file to test (required)." | |
29 echo " -d=, --one-delta= The filename, relative to app, to touch in order t
o trigger a one-delta compile." | |
30 echo " -r=, --revision= The compiler revision to start comparing against (
default to current repository revision)." | |
31 echo " -c=, --count= The number of compiler revisions test against. De
fault 50." | |
32 echo " -l=, --low-rev= Alternative to --count; set the lowest revision to
run to" | |
33 echo " -n=, --runs= How many times each compiler is run against the ta
rget application." | |
34 echo " -h, --help What you see is what you get." | |
35 exit $exitValue | |
36 } | |
37 | |
38 function failTest() { | |
39 if [ ! $1 -eq 0 ]; then | |
40 echo $2 | |
41 exit $1 | |
42 fi | |
43 } | |
44 | |
45 RESPONSE[0]="Performance: Better" | |
46 RESPONSE[1]="Performance: No Change" | |
47 RESPONSE[2]="Performance: Worse" | |
48 | |
49 function calcStat() { | |
50 # Assume we're always making improvments, S2 will be presented as a larger val
ue | |
51 LINEMATCH_DEV="s/${1}: .*stdev: \([0-9]*\).*/\1/p" | |
52 LINEMATCH_VALUE="s/${1}: average: \([0-9]*\).*/\1/p" | |
53 S1_DEV=$(sed -n -e "$LINEMATCH_DEV" $STAT1) | |
54 if [ "" != "$S1_DEV" ]; then | |
55 S1_VALUE=$(sed -n -e "$LINEMATCH_VALUE" $STAT1) | |
56 echo -ne "\t"$S1_VALUE"\t"$S1_DEV >> $PLOTS | |
57 else | |
58 echo -ne "\t-1\t-1" >> $PLOTS | |
59 fi | |
60 return 0 | |
61 } | |
62 | |
63 function calcStats() { | |
64 echo -n $1 >> $PLOTS | |
65 calcStat full-compile | |
66 calcStat zero-delta-compile | |
67 calcStat one-delta-compile | |
68 echo "" >> $PLOTS | |
69 } | |
70 | |
71 if [ $# -eq 0 ]; then | |
72 printHelp; | |
73 fi | |
74 | |
75 for i in $* | |
76 do | |
77 case $i in | |
78 --one-delta=*|-d=*) | |
79 ONE_DELTA=${i#*=} | |
80 COMPARE_OPTIONS+="--one-delta=$ONE_DELTA ";; | |
81 --app=*|-a=*) | |
82 APP=${i#*=};; | |
83 --revision=*|-r=*) | |
84 REV=${i#*=};; | |
85 --count=*|-c=*) | |
86 COUNT=${i#*=} | |
87 LOW_REV="";; | |
88 --runs=*|-n=*) | |
89 RUNS=${i#*=};; | |
90 --low-rev=*|-l=*) | |
91 LOW_REV=${i#*=} | |
92 COUNT=0;; | |
93 --help|-h) | |
94 printHelp 0;; | |
95 *) | |
96 echo "Parameter $i not recognized" | |
97 printHelp 1;; | |
98 esac | |
99 done | |
100 | |
101 COMPARE_OPTIONS+="-r=$RUNS " | |
102 | |
103 if ((RUNS > 0)); then | |
104 if [ "" = "$APP" ] || [ ! -r $APP ]; then | |
105 echo "Required --app" " got: $APP" | |
106 printHelp 1 | |
107 fi | |
108 APP=$( cd "$( dirname "$APP" )" && pwd )/$( basename "$APP") | |
109 COMPARE_OPTIONS+="--app=$APP " | |
110 else | |
111 echo "Building up compiler cache" | |
112 APP="Compiler cache" | |
113 fi | |
114 | |
115 ROOT_OF_REPO=$BASE_PATH | |
116 TEST_DIR=$BASE_PATH | |
117 while true; do | |
118 ls -d .gclient > /dev/null 2>&1 | |
119 if [ $? -eq 0 ]; then | |
120 echo "Root found: $ROOT_OF_REPO" | |
121 break; | |
122 fi | |
123 if [ "$TEST_DIR" = "/" ]; then | |
124 failTest 1 "Hit the root directory; no .git/ found?!" | |
125 fi | |
126 ROOT_OF_REPO=$TEST_DIR | |
127 cd .. | |
128 TEST_DIR=$(pwd) | |
129 done | |
130 | |
131 # Make a temporary directory in the current path and checkout the revision | |
132 TMP_DIR=$ROOT_OF_REPO/compiler/tmp_performance_comparisons | |
133 mkdir -p $TMP_DIR | |
134 | |
135 LOG_FILE=$TMP_DIR/compiler_compare.log | |
136 PLOTS=$TMP_DIR/compiler_plots.dat | |
137 STAT1=$TMP_DIR/compiler_metrics.txt | |
138 COMPARE_OPTIONS+="--output=$TMP_DIR " | |
139 | |
140 # zero out files | |
141 echo "" > $LOG_FILE | |
142 | |
143 # switch to tmp for remainder of building | |
144 cd $TMP_DIR | |
145 gclient config https://dart.googlecode.com/svn/branches/bleeding_edge/deps/compi
ler.deps >> $LOG_FILE 2>&1 | |
146 failTest $? "Error calling gclient config" | |
147 | |
148 if [ "" == "$REV" ]; then | |
149 echo "No revision specified; checking out head for test" | |
150 REV=`svn info https://dart.googlecode.com/svn/branches/bleeding_edge/deps/comp
iler.deps | sed -n -e 's/Revision: \([0-9]*\)/\1/p'` | |
151 echo "Head revision = $REV" | |
152 fi | |
153 | |
154 function failStats() { | |
155 echo -e "$1\t-1\t0\t-1\t0\t-1\t0" >> $PLOTS | |
156 return 0; | |
157 } | |
158 | |
159 function compileRevision() { | |
160 REVISION=$1 | |
161 PREBUILT_DIR=$ROOT_OF_REPO/analyzer/revs/$REVISION/prebuilt | |
162 PREBUILT_BIN=$PREBUILT_DIR/analyzer/bin/dart_analyzer | |
163 if [ ! -x $PREBUILT_BIN ]; then | |
164 echo "No prebuilt, building and caching" | |
165 echo "Checking out clean version of $REVISION; will take some time. Look at
$LOG_FILE for progress" | |
166 date | |
167 cd $TMP_DIR | |
168 gclient sync -t --revision=$REVISION >> $LOG_FILE 2>&1 | |
169 failTest $? "Error calling gclient sync" | |
170 echo "Run hooks" | |
171 gclient runhooks >> $LOG_FILE 2>&1 | |
172 | |
173 echo "Compiling clean version of dart_analyzer" | |
174 date | |
175 cd compiler | |
176 ../tools/build.py --mode release >> $LOG_FILE 2>&1 | |
177 if [ ! $? -eq 0 ]; then | |
178 echo "error compiling" | |
179 failStats $REVISION | |
180 return 1; | |
181 fi | |
182 | |
183 # Give the metrics system a backwards compatible way of getting to the | |
184 # artifacts that it needs. | |
185 cd .. | |
186 mkdir -p $ROOT_OF_REPO/compiler/revs/$REVISION/prebuilt | |
187 cd $ROOT_OF_REPO/compiler/revs/$REVISION/prebuilt | |
188 COMPILER_OUTDIR=$TMP_DIR/compiler/out/Release_ia32 | |
189 cp -r $COMPILER_OUTDIR/compiler ./compiler | |
190 else | |
191 echo "Cached prebuilt of $REVISION!" | |
192 fi | |
193 | |
194 # Short circuit if we're just filling in the build cache | |
195 if [ $RUNS -eq 0 ]; then | |
196 echo "run in compile only mode, no stats generating" | |
197 return 0; | |
198 fi | |
199 | |
200 # Do the second test | |
201 echo "Running test with dart_analyzer $REVISION!" | |
202 date | |
203 echo $SCRIPT_PATH/analyzer_metrics.sh --stats-prefix=$REVISION --analyzer=$PRE
BUILT_DIR/analyzer/bin/dart_analyzer $COMPARE_OPTIONS >> $LOG_FILE 2>&1 | |
204 $SCRIPT_PATH/analyzer_metrics.sh --stats-prefix=$REVISION --analyzer=$PREBUILT
_DIR/analyzer/bin/dart_analyzer $COMPARE_OPTIONS > $STAT1 | |
205 if [ ! $? -eq 0 ]; then | |
206 echo "error sampling" | |
207 failStats $REVISION | |
208 return 2; | |
209 fi | |
210 | |
211 # Output the reivision to the PLOTS file; newline added after stats | |
212 calcStats $REVISION | |
213 } | |
214 | |
215 echo -e "#Rev\tFull-ms\tdev\tZeroD\tdev\tOneD\tdev" > $PLOTS | |
216 if [ "$LOW_REV" ]; then | |
217 COUNT=$(( REV - LOW_REV + 1 )) | |
218 else | |
219 LOW_REV=$(( REV - COUNT + 1 )) | |
220 fi | |
221 for (( i = REV ; i >= LOW_REV ; i-- )) | |
222 do | |
223 echo "["$( basename "$APP")": "$((REV - i + 1))"/"$COUNT", rev:$i]" | |
224 compileRevision $i | |
225 done | |
226 | |
OLD | NEW |