OLD | NEW |
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library test.timing; | 5 library test.timing; |
6 | 6 |
7 import 'dart:async'; | 7 import 'dart:async'; |
8 import 'dart:io'; | 8 import 'dart:io'; |
9 import 'dart:math'; | 9 import 'dart:math'; |
10 | 10 |
11 import 'package:path/path.dart'; | 11 import 'package:path/path.dart'; |
12 | 12 |
13 import '../integration/integration_test_methods.dart'; | 13 import '../integration/integration_test_methods.dart'; |
14 import '../integration/integration_tests.dart'; | 14 import '../integration/integration_tests.dart'; |
15 | 15 |
16 /** | 16 /** |
| 17 * Instances of the class [TimingResult] represent the timing information |
| 18 * gathered while executing a given timing test. |
| 19 */ |
| 20 class TimingResult { |
| 21 /** |
| 22 * The number of nanoseconds in a millisecond. |
| 23 */ |
| 24 static int NANOSECONDS_PER_MILLISECOND = 1000000; |
| 25 |
| 26 /** |
| 27 * The amount of time spent executing each test, in nanoseconds. |
| 28 */ |
| 29 List<int> times; |
| 30 |
| 31 /** |
| 32 * Initialize a newly created timing result. |
| 33 */ |
| 34 TimingResult(this.times); |
| 35 |
| 36 /** |
| 37 * The average amount of time spent executing a single iteration, in |
| 38 * milliseconds. |
| 39 */ |
| 40 int get averageTime { |
| 41 return totalTime ~/ times.length; |
| 42 } |
| 43 |
| 44 /** |
| 45 * The maximum amount of time spent executing a single iteration, in |
| 46 * milliseconds. |
| 47 */ |
| 48 int get maxTime { |
| 49 int maxTime = 0; |
| 50 int count = times.length; |
| 51 for (int i = 0; i < count; i++) { |
| 52 maxTime = max(maxTime, times[i]); |
| 53 } |
| 54 return maxTime ~/ NANOSECONDS_PER_MILLISECOND; |
| 55 } |
| 56 |
| 57 /** |
| 58 * The minimum amount of time spent executing a single iteration, in |
| 59 * milliseconds. |
| 60 */ |
| 61 int get minTime { |
| 62 |
| 63 int minTime = times[0]; |
| 64 int count = times.length; |
| 65 for (int i = 1; i < count; i++) { |
| 66 minTime = min(minTime, times[i]); |
| 67 } |
| 68 return minTime ~/ NANOSECONDS_PER_MILLISECOND; |
| 69 } |
| 70 |
| 71 /** |
| 72 * The standard deviation of the times. |
| 73 */ |
| 74 double get standardDeviation { |
| 75 return computeStandardDeviation(toMilliseconds(times)); |
| 76 } |
| 77 |
| 78 /** |
| 79 * The total amount of time spent executing the test, in milliseconds. |
| 80 */ |
| 81 int get totalTime { |
| 82 int totalTime = 0; |
| 83 int count = times.length; |
| 84 for (int i = 0; i < count; i++) { |
| 85 totalTime += times[i]; |
| 86 } |
| 87 return totalTime ~/ NANOSECONDS_PER_MILLISECOND; |
| 88 } |
| 89 |
| 90 /** |
| 91 * Compute the standard deviation of the given set of [values]. |
| 92 */ |
| 93 double computeStandardDeviation(List<int> values) { |
| 94 int count = values.length; |
| 95 double sumOfValues = 0.0; |
| 96 for (int i = 0; i < count; i++) { |
| 97 sumOfValues += values[i]; |
| 98 } |
| 99 double average = sumOfValues / count; |
| 100 double sumOfDiffSquared = 0.0; |
| 101 for (int i = 0; i < count; i++) { |
| 102 double diff = values[i] - average; |
| 103 sumOfDiffSquared += diff * diff; |
| 104 } |
| 105 return sqrt((sumOfDiffSquared / (count - 1))); |
| 106 } |
| 107 |
| 108 /** |
| 109 * Convert the given [times], expressed in nanoseconds, to times expressed in |
| 110 * milliseconds. |
| 111 */ |
| 112 List<int> toMilliseconds(List<int> times) { |
| 113 int count = times.length; |
| 114 List<int> convertedValues = new List<int>(); |
| 115 for (int i = 0; i < count; i++) { |
| 116 convertedValues.add(times[i] ~/ NANOSECONDS_PER_MILLISECOND); |
| 117 } |
| 118 return convertedValues; |
| 119 } |
| 120 } |
| 121 |
| 122 /** |
17 * The abstract class [TimingTest] defines the behavior of objects that measure | 123 * The abstract class [TimingTest] defines the behavior of objects that measure |
18 * the time required to perform some sequence of server operations. | 124 * the time required to perform some sequence of server operations. |
19 */ | 125 */ |
20 abstract class TimingTest extends IntegrationTestMixin { | 126 abstract class TimingTest extends IntegrationTestMixin { |
21 /** | 127 /** |
22 * The connection to the analysis server. | |
23 */ | |
24 Server server; | |
25 | |
26 /** | |
27 * The temporary directory in which source files can be stored. | |
28 */ | |
29 Directory sourceDirectory; | |
30 | |
31 /** | |
32 * A flag indicating whether the teardown process should skip sending a | |
33 * "server.shutdown" request because the server is known to have already | |
34 * shutdown. | |
35 */ | |
36 bool skipShutdown = false; | |
37 | |
38 /** | |
39 * The number of times the test will be performed in order to warm up the VM. | 128 * The number of times the test will be performed in order to warm up the VM. |
40 */ | 129 */ |
41 static final int DEFAULT_WARMUP_COUNT = 10; | 130 static final int DEFAULT_WARMUP_COUNT = 10; |
42 | 131 |
43 /** | 132 /** |
44 * The number of times the test will be performed in order to compute a time. | 133 * The number of times the test will be performed in order to compute a time. |
45 */ | 134 */ |
46 static final int DEFAULT_TIMING_COUNT = 10; | 135 static final int DEFAULT_TIMING_COUNT = 10; |
47 | 136 |
48 /** | 137 /** |
49 * The file suffix used to identify Dart files. | 138 * The file suffix used to identify Dart files. |
50 */ | 139 */ |
51 static final String DART_SUFFIX = '.dart'; | 140 static final String DART_SUFFIX = '.dart'; |
52 | 141 |
53 /** | 142 /** |
54 * The file suffix used to identify HTML files. | 143 * The file suffix used to identify HTML files. |
55 */ | 144 */ |
56 static final String HTML_SUFFIX = '.html'; | 145 static final String HTML_SUFFIX = '.html'; |
57 | 146 |
58 /** | 147 /** |
59 * The amount of time to give the server to respond to a shutdown request | 148 * The amount of time to give the server to respond to a shutdown request |
60 * before forcibly terminating it. | 149 * before forcibly terminating it. |
61 */ | 150 */ |
62 static const Duration SHUTDOWN_TIMEOUT = const Duration(seconds: 5); | 151 static const Duration SHUTDOWN_TIMEOUT = const Duration(seconds: 5); |
63 | 152 |
64 /** | 153 /** |
| 154 * The connection to the analysis server. |
| 155 */ |
| 156 Server server; |
| 157 |
| 158 /** |
| 159 * The temporary directory in which source files can be stored. |
| 160 */ |
| 161 Directory sourceDirectory; |
| 162 |
| 163 /** |
| 164 * A flag indicating whether the teardown process should skip sending a |
| 165 * "server.shutdown" request because the server is known to have already |
| 166 * shutdown. |
| 167 */ |
| 168 bool skipShutdown = false; |
| 169 |
| 170 /** |
65 * Initialize a newly created test. | 171 * Initialize a newly created test. |
66 */ | 172 */ |
67 TimingTest(); | 173 TimingTest(); |
68 | 174 |
69 /** | 175 /** |
70 * Return the number of iterations that should be performed in order to warm | |
71 * up the VM. | |
72 */ | |
73 int get warmupCount => DEFAULT_WARMUP_COUNT; | |
74 | |
75 /** | |
76 * Return the number of iterations that should be performed in order to | 176 * Return the number of iterations that should be performed in order to |
77 * compute a time. | 177 * compute a time. |
78 */ | 178 */ |
79 int get timingCount => DEFAULT_TIMING_COUNT; | 179 int get timingCount => DEFAULT_TIMING_COUNT; |
80 | 180 |
81 /** | 181 /** |
| 182 * Return the number of iterations that should be performed in order to warm |
| 183 * up the VM. |
| 184 */ |
| 185 int get warmupCount => DEFAULT_WARMUP_COUNT; |
| 186 |
| 187 /** |
82 * Perform any operations that need to be performed once before any iterations
. | 188 * Perform any operations that need to be performed once before any iterations
. |
83 */ | 189 */ |
84 Future oneTimeSetUp() { | 190 Future oneTimeSetUp() { |
85 initializeInttestMixin(); | 191 initializeInttestMixin(); |
86 server = new Server(); | 192 server = new Server(); |
87 sourceDirectory = Directory.systemTemp.createTempSync('analysisServer'); | 193 sourceDirectory = Directory.systemTemp.createTempSync('analysisServer'); |
88 Completer serverConnected = new Completer(); | 194 Completer serverConnected = new Completer(); |
89 onServerConnected.listen((_) { | 195 onServerConnected.listen((_) { |
90 serverConnected.complete(); | 196 serverConnected.complete(); |
91 }); | 197 }); |
92 skipShutdown = true; | 198 skipShutdown = true; |
93 return server.start(profileServer: true).then((params) { | 199 return server.start(profileServer: true).then((params) { |
94 server.listenToOutput(dispatchNotification); | 200 server.listenToOutput(dispatchNotification); |
95 server.exitCode.then((_) { | 201 server.exitCode.then((_) { |
96 skipShutdown = true; | 202 skipShutdown = true; |
97 }); | 203 }); |
98 return serverConnected.future; | 204 return serverConnected.future; |
99 }); | 205 }); |
100 } | 206 } |
101 | 207 |
102 /** | 208 /** |
103 * Perform any operations that need to be performed before each iteration. | |
104 */ | |
105 Future setUp(); | |
106 | |
107 /** | |
108 * Perform any operations that part of a single iteration. It is the execution | |
109 * of this method that will be measured. | |
110 */ | |
111 Future perform(); | |
112 | |
113 /** | |
114 * Perform any operations that need to be performed after each iteration. | |
115 */ | |
116 Future tearDown(); | |
117 | |
118 /** | |
119 * Perform any operations that need to be performed once after all iterations. | 209 * Perform any operations that need to be performed once after all iterations. |
120 */ | 210 */ |
121 Future oneTimeTearDown() { | 211 Future oneTimeTearDown() { |
122 return _shutdownIfNeeded().then((_) { | 212 return _shutdownIfNeeded().then((_) { |
123 sourceDirectory.deleteSync(recursive: true); | 213 sourceDirectory.deleteSync(recursive: true); |
124 }); | 214 }); |
125 } | 215 } |
126 | 216 |
127 /** | 217 /** |
| 218 * Perform any operations that part of a single iteration. It is the execution |
| 219 * of this method that will be measured. |
| 220 */ |
| 221 Future perform(); |
| 222 |
| 223 /** |
128 * Return a future that will complete with a timing result representing the | 224 * Return a future that will complete with a timing result representing the |
129 * number of milliseconds required to perform the operation the specified | 225 * number of milliseconds required to perform the operation the specified |
130 * number of times. | 226 * number of times. |
131 */ | 227 */ |
132 Future<TimingResult> run() { | 228 Future<TimingResult> run() { |
133 List<int> times = new List<int>(); | 229 List<int> times = new List<int>(); |
134 return oneTimeSetUp().then((_) { | 230 return oneTimeSetUp().then((_) { |
135 return _repeat(warmupCount, null).then((_) { | 231 return _repeat(warmupCount, null).then((_) { |
136 return _repeat(timingCount, times).then((_) { | 232 return _repeat(timingCount, times).then((_) { |
137 return oneTimeTearDown().then((_) { | 233 return oneTimeTearDown().then((_) { |
138 return new Future.value(new TimingResult(times)); | 234 return new Future.value(new TimingResult(times)); |
139 }); | 235 }); |
140 }); | 236 }); |
141 }); | 237 }); |
142 }); | 238 }); |
143 } | 239 } |
144 | 240 |
145 /** | 241 /** |
| 242 * Perform any operations that need to be performed before each iteration. |
| 243 */ |
| 244 Future setUp(); |
| 245 |
| 246 /** |
146 * Convert the given [relativePath] to an absolute path, by interpreting it | 247 * Convert the given [relativePath] to an absolute path, by interpreting it |
147 * relative to [sourceDirectory]. On Windows any forward slashes in | 248 * relative to [sourceDirectory]. On Windows any forward slashes in |
148 * [relativePath] are converted to backslashes. | 249 * [relativePath] are converted to backslashes. |
149 */ | 250 */ |
150 String sourcePath(String relativePath) { | 251 String sourcePath(String relativePath) { |
151 return join(sourceDirectory.path, relativePath.replaceAll('/', separator)); | 252 return join(sourceDirectory.path, relativePath.replaceAll('/', separator)); |
152 } | 253 } |
153 | 254 |
154 /** | 255 /** |
| 256 * Perform any operations that need to be performed after each iteration. |
| 257 */ |
| 258 Future tearDown(); |
| 259 |
| 260 /** |
155 * Write a source file with the given absolute [pathname] and [contents]. | 261 * Write a source file with the given absolute [pathname] and [contents]. |
156 * | 262 * |
157 * If the file didn't previously exist, it is created. If it did, it is | 263 * If the file didn't previously exist, it is created. If it did, it is |
158 * overwritten. | 264 * overwritten. |
159 * | 265 * |
160 * Parent directories are created as necessary. | 266 * Parent directories are created as necessary. |
161 */ | 267 */ |
162 void writeFile(String pathname, String contents) { | 268 void writeFile(String pathname, String contents) { |
163 new Directory(dirname(pathname)).createSync(recursive: true); | 269 new Directory(dirname(pathname)).createSync(recursive: true); |
164 new File(pathname).writeAsStringSync(contents); | 270 new File(pathname).writeAsStringSync(contents); |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
204 return new Future.value(); | 310 return new Future.value(); |
205 } | 311 } |
206 // Give the server a short time to comply with the shutdown request; if it | 312 // Give the server a short time to comply with the shutdown request; if it |
207 // doesn't exit, then forcibly terminate it. | 313 // doesn't exit, then forcibly terminate it. |
208 sendServerShutdown(); | 314 sendServerShutdown(); |
209 return server.exitCode.timeout(SHUTDOWN_TIMEOUT, onTimeout: () { | 315 return server.exitCode.timeout(SHUTDOWN_TIMEOUT, onTimeout: () { |
210 return server.kill(); | 316 return server.kill(); |
211 }); | 317 }); |
212 } | 318 } |
213 } | 319 } |
214 | |
215 /** | |
216 * Instances of the class [TimingResult] represent the timing information | |
217 * gathered while executing a given timing test. | |
218 */ | |
219 class TimingResult { | |
220 /** | |
221 * The amount of time spent executing each test, in nanoseconds. | |
222 */ | |
223 List<int> times; | |
224 | |
225 /** | |
226 * The number of nanoseconds in a millisecond. | |
227 */ | |
228 static int NANOSECONDS_PER_MILLISECOND = 1000000; | |
229 | |
230 /** | |
231 * Initialize a newly created timing result. | |
232 */ | |
233 TimingResult(this.times); | |
234 | |
235 /** | |
236 * The average amount of time spent executing a single iteration, in | |
237 * milliseconds. | |
238 */ | |
239 int get averageTime { | |
240 return totalTime ~/ times.length; | |
241 } | |
242 | |
243 /** | |
244 * The maximum amount of time spent executing a single iteration, in | |
245 * milliseconds. | |
246 */ | |
247 int get maxTime { | |
248 int maxTime = 0; | |
249 int count = times.length; | |
250 for (int i = 0; i < count; i++) { | |
251 maxTime = max(maxTime, times[i]); | |
252 } | |
253 return maxTime ~/ NANOSECONDS_PER_MILLISECOND; | |
254 } | |
255 | |
256 /** | |
257 * The minimum amount of time spent executing a single iteration, in | |
258 * milliseconds. | |
259 */ | |
260 int get minTime { | |
261 | |
262 int minTime = times[0]; | |
263 int count = times.length; | |
264 for (int i = 1; i < count; i++) { | |
265 minTime = min(minTime, times[i]); | |
266 } | |
267 return minTime ~/ NANOSECONDS_PER_MILLISECOND; | |
268 } | |
269 | |
270 /** | |
271 * The standard deviation of the times. | |
272 */ | |
273 double get standardDeviation { | |
274 return computeStandardDeviation(toMilliseconds(times)); | |
275 } | |
276 | |
277 /** | |
278 * The total amount of time spent executing the test, in milliseconds. | |
279 */ | |
280 int get totalTime { | |
281 int totalTime = 0; | |
282 int count = times.length; | |
283 for (int i = 0; i < count; i++) { | |
284 totalTime += times[i]; | |
285 } | |
286 return totalTime ~/ NANOSECONDS_PER_MILLISECOND; | |
287 } | |
288 | |
289 /** | |
290 * Compute the standard deviation of the given set of [values]. | |
291 */ | |
292 double computeStandardDeviation(List<int> values) { | |
293 int count = values.length; | |
294 double sumOfValues = 0.0; | |
295 for (int i = 0; i < count; i++) { | |
296 sumOfValues += values[i]; | |
297 } | |
298 double average = sumOfValues / count; | |
299 double sumOfDiffSquared = 0.0; | |
300 for (int i = 0; i < count; i++) { | |
301 double diff = values[i] - average; | |
302 sumOfDiffSquared += diff * diff; | |
303 } | |
304 return sqrt((sumOfDiffSquared / (count - 1))); | |
305 } | |
306 | |
307 /** | |
308 * Convert the given [times], expressed in nanoseconds, to times expressed in | |
309 * milliseconds. | |
310 */ | |
311 List<int> toMilliseconds(List<int> times) { | |
312 int count = times.length; | |
313 List<int> convertedValues = new List<int>(); | |
314 for (int i = 0; i < count; i++) { | |
315 convertedValues.add(times[i] ~/ NANOSECONDS_PER_MILLISECOND); | |
316 } | |
317 return convertedValues; | |
318 } | |
319 } | |
OLD | NEW |