| OLD | NEW |
| 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library test.timing; | 5 library test.timing; |
| 6 | 6 |
| 7 import 'dart:async'; | 7 import 'dart:async'; |
| 8 import 'dart:io'; | 8 import 'dart:io'; |
| 9 import 'dart:math'; | 9 import 'dart:math'; |
| 10 | 10 |
| 11 import 'package:path/path.dart'; | 11 import 'package:path/path.dart'; |
| 12 | 12 |
| 13 import '../integration/integration_test_methods.dart'; | 13 import '../integration/integration_test_methods.dart'; |
| 14 import '../integration/integration_tests.dart'; | 14 import '../integration/integration_tests.dart'; |
| 15 | 15 |
| 16 /** | 16 /** |
| 17 * Instances of the class [TimingResult] represent the timing information |
| 18 * gathered while executing a given timing test. |
| 19 */ |
| 20 class TimingResult { |
| 21 /** |
| 22 * The number of nanoseconds in a millisecond. |
| 23 */ |
| 24 static int NANOSECONDS_PER_MILLISECOND = 1000000; |
| 25 |
| 26 /** |
| 27 * The amount of time spent executing each test, in nanoseconds. |
| 28 */ |
| 29 List<int> times; |
| 30 |
| 31 /** |
| 32 * Initialize a newly created timing result. |
| 33 */ |
| 34 TimingResult(this.times); |
| 35 |
| 36 /** |
| 37 * The average amount of time spent executing a single iteration, in |
| 38 * milliseconds. |
| 39 */ |
| 40 int get averageTime { |
| 41 return totalTime ~/ times.length; |
| 42 } |
| 43 |
| 44 /** |
| 45 * The maximum amount of time spent executing a single iteration, in |
| 46 * milliseconds. |
| 47 */ |
| 48 int get maxTime { |
| 49 int maxTime = 0; |
| 50 int count = times.length; |
| 51 for (int i = 0; i < count; i++) { |
| 52 maxTime = max(maxTime, times[i]); |
| 53 } |
| 54 return maxTime ~/ NANOSECONDS_PER_MILLISECOND; |
| 55 } |
| 56 |
| 57 /** |
| 58 * The minimum amount of time spent executing a single iteration, in |
| 59 * milliseconds. |
| 60 */ |
| 61 int get minTime { |
| 62 |
| 63 int minTime = times[0]; |
| 64 int count = times.length; |
| 65 for (int i = 1; i < count; i++) { |
| 66 minTime = min(minTime, times[i]); |
| 67 } |
| 68 return minTime ~/ NANOSECONDS_PER_MILLISECOND; |
| 69 } |
| 70 |
| 71 /** |
| 72 * The standard deviation of the times. |
| 73 */ |
| 74 double get standardDeviation { |
| 75 return computeStandardDeviation(toMilliseconds(times)); |
| 76 } |
| 77 |
| 78 /** |
| 79 * The total amount of time spent executing the test, in milliseconds. |
| 80 */ |
| 81 int get totalTime { |
| 82 int totalTime = 0; |
| 83 int count = times.length; |
| 84 for (int i = 0; i < count; i++) { |
| 85 totalTime += times[i]; |
| 86 } |
| 87 return totalTime ~/ NANOSECONDS_PER_MILLISECOND; |
| 88 } |
| 89 |
| 90 /** |
| 91 * Compute the standard deviation of the given set of [values]. |
| 92 */ |
| 93 double computeStandardDeviation(List<int> values) { |
| 94 int count = values.length; |
| 95 double sumOfValues = 0.0; |
| 96 for (int i = 0; i < count; i++) { |
| 97 sumOfValues += values[i]; |
| 98 } |
| 99 double average = sumOfValues / count; |
| 100 double sumOfDiffSquared = 0.0; |
| 101 for (int i = 0; i < count; i++) { |
| 102 double diff = values[i] - average; |
| 103 sumOfDiffSquared += diff * diff; |
| 104 } |
| 105 return sqrt((sumOfDiffSquared / (count - 1))); |
| 106 } |
| 107 |
| 108 /** |
| 109 * Convert the given [times], expressed in nanoseconds, to times expressed in |
| 110 * milliseconds. |
| 111 */ |
| 112 List<int> toMilliseconds(List<int> times) { |
| 113 int count = times.length; |
| 114 List<int> convertedValues = new List<int>(); |
| 115 for (int i = 0; i < count; i++) { |
| 116 convertedValues.add(times[i] ~/ NANOSECONDS_PER_MILLISECOND); |
| 117 } |
| 118 return convertedValues; |
| 119 } |
| 120 } |
| 121 |
| 122 /** |
| 17 * The abstract class [TimingTest] defines the behavior of objects that measure | 123 * The abstract class [TimingTest] defines the behavior of objects that measure |
| 18 * the time required to perform some sequence of server operations. | 124 * the time required to perform some sequence of server operations. |
| 19 */ | 125 */ |
| 20 abstract class TimingTest extends IntegrationTestMixin { | 126 abstract class TimingTest extends IntegrationTestMixin { |
| 21 /** | 127 /** |
| 22 * The connection to the analysis server. | |
| 23 */ | |
| 24 Server server; | |
| 25 | |
| 26 /** | |
| 27 * The temporary directory in which source files can be stored. | |
| 28 */ | |
| 29 Directory sourceDirectory; | |
| 30 | |
| 31 /** | |
| 32 * A flag indicating whether the teardown process should skip sending a | |
| 33 * "server.shutdown" request because the server is known to have already | |
| 34 * shutdown. | |
| 35 */ | |
| 36 bool skipShutdown = false; | |
| 37 | |
| 38 /** | |
| 39 * The number of times the test will be performed in order to warm up the VM. | 128 * The number of times the test will be performed in order to warm up the VM. |
| 40 */ | 129 */ |
| 41 static final int DEFAULT_WARMUP_COUNT = 10; | 130 static final int DEFAULT_WARMUP_COUNT = 10; |
| 42 | 131 |
| 43 /** | 132 /** |
| 44 * The number of times the test will be performed in order to compute a time. | 133 * The number of times the test will be performed in order to compute a time. |
| 45 */ | 134 */ |
| 46 static final int DEFAULT_TIMING_COUNT = 10; | 135 static final int DEFAULT_TIMING_COUNT = 10; |
| 47 | 136 |
| 48 /** | 137 /** |
| 49 * The file suffix used to identify Dart files. | 138 * The file suffix used to identify Dart files. |
| 50 */ | 139 */ |
| 51 static final String DART_SUFFIX = '.dart'; | 140 static final String DART_SUFFIX = '.dart'; |
| 52 | 141 |
| 53 /** | 142 /** |
| 54 * The file suffix used to identify HTML files. | 143 * The file suffix used to identify HTML files. |
| 55 */ | 144 */ |
| 56 static final String HTML_SUFFIX = '.html'; | 145 static final String HTML_SUFFIX = '.html'; |
| 57 | 146 |
| 58 /** | 147 /** |
| 59 * The amount of time to give the server to respond to a shutdown request | 148 * The amount of time to give the server to respond to a shutdown request |
| 60 * before forcibly terminating it. | 149 * before forcibly terminating it. |
| 61 */ | 150 */ |
| 62 static const Duration SHUTDOWN_TIMEOUT = const Duration(seconds: 5); | 151 static const Duration SHUTDOWN_TIMEOUT = const Duration(seconds: 5); |
| 63 | 152 |
| 64 /** | 153 /** |
| 154 * The connection to the analysis server. |
| 155 */ |
| 156 Server server; |
| 157 |
| 158 /** |
| 159 * The temporary directory in which source files can be stored. |
| 160 */ |
| 161 Directory sourceDirectory; |
| 162 |
| 163 /** |
| 164 * A flag indicating whether the teardown process should skip sending a |
| 165 * "server.shutdown" request because the server is known to have already |
| 166 * shutdown. |
| 167 */ |
| 168 bool skipShutdown = false; |
| 169 |
| 170 /** |
| 65 * Initialize a newly created test. | 171 * Initialize a newly created test. |
| 66 */ | 172 */ |
| 67 TimingTest(); | 173 TimingTest(); |
| 68 | 174 |
| 69 /** | 175 /** |
| 70 * Return the number of iterations that should be performed in order to warm | |
| 71 * up the VM. | |
| 72 */ | |
| 73 int get warmupCount => DEFAULT_WARMUP_COUNT; | |
| 74 | |
| 75 /** | |
| 76 * Return the number of iterations that should be performed in order to | 176 * Return the number of iterations that should be performed in order to |
| 77 * compute a time. | 177 * compute a time. |
| 78 */ | 178 */ |
| 79 int get timingCount => DEFAULT_TIMING_COUNT; | 179 int get timingCount => DEFAULT_TIMING_COUNT; |
| 80 | 180 |
| 81 /** | 181 /** |
| 182 * Return the number of iterations that should be performed in order to warm |
| 183 * up the VM. |
| 184 */ |
| 185 int get warmupCount => DEFAULT_WARMUP_COUNT; |
| 186 |
| 187 /** |
| 82 * Perform any operations that need to be performed once before any iterations
. | 188 * Perform any operations that need to be performed once before any iterations
. |
| 83 */ | 189 */ |
| 84 Future oneTimeSetUp() { | 190 Future oneTimeSetUp() { |
| 85 initializeInttestMixin(); | 191 initializeInttestMixin(); |
| 86 server = new Server(); | 192 server = new Server(); |
| 87 sourceDirectory = Directory.systemTemp.createTempSync('analysisServer'); | 193 sourceDirectory = Directory.systemTemp.createTempSync('analysisServer'); |
| 88 Completer serverConnected = new Completer(); | 194 Completer serverConnected = new Completer(); |
| 89 onServerConnected.listen((_) { | 195 onServerConnected.listen((_) { |
| 90 serverConnected.complete(); | 196 serverConnected.complete(); |
| 91 }); | 197 }); |
| 92 skipShutdown = true; | 198 skipShutdown = true; |
| 93 return server.start(profileServer: true).then((params) { | 199 return server.start(profileServer: true).then((params) { |
| 94 server.listenToOutput(dispatchNotification); | 200 server.listenToOutput(dispatchNotification); |
| 95 server.exitCode.then((_) { | 201 server.exitCode.then((_) { |
| 96 skipShutdown = true; | 202 skipShutdown = true; |
| 97 }); | 203 }); |
| 98 return serverConnected.future; | 204 return serverConnected.future; |
| 99 }); | 205 }); |
| 100 } | 206 } |
| 101 | 207 |
| 102 /** | 208 /** |
| 103 * Perform any operations that need to be performed before each iteration. | |
| 104 */ | |
| 105 Future setUp(); | |
| 106 | |
| 107 /** | |
| 108 * Perform any operations that part of a single iteration. It is the execution | |
| 109 * of this method that will be measured. | |
| 110 */ | |
| 111 Future perform(); | |
| 112 | |
| 113 /** | |
| 114 * Perform any operations that need to be performed after each iteration. | |
| 115 */ | |
| 116 Future tearDown(); | |
| 117 | |
| 118 /** | |
| 119 * Perform any operations that need to be performed once after all iterations. | 209 * Perform any operations that need to be performed once after all iterations. |
| 120 */ | 210 */ |
| 121 Future oneTimeTearDown() { | 211 Future oneTimeTearDown() { |
| 122 return _shutdownIfNeeded().then((_) { | 212 return _shutdownIfNeeded().then((_) { |
| 123 sourceDirectory.deleteSync(recursive: true); | 213 sourceDirectory.deleteSync(recursive: true); |
| 124 }); | 214 }); |
| 125 } | 215 } |
| 126 | 216 |
| 127 /** | 217 /** |
| 218 * Perform any operations that part of a single iteration. It is the execution |
| 219 * of this method that will be measured. |
| 220 */ |
| 221 Future perform(); |
| 222 |
| 223 /** |
| 128 * Return a future that will complete with a timing result representing the | 224 * Return a future that will complete with a timing result representing the |
| 129 * number of milliseconds required to perform the operation the specified | 225 * number of milliseconds required to perform the operation the specified |
| 130 * number of times. | 226 * number of times. |
| 131 */ | 227 */ |
| 132 Future<TimingResult> run() { | 228 Future<TimingResult> run() { |
| 133 List<int> times = new List<int>(); | 229 List<int> times = new List<int>(); |
| 134 return oneTimeSetUp().then((_) { | 230 return oneTimeSetUp().then((_) { |
| 135 return _repeat(warmupCount, null).then((_) { | 231 return _repeat(warmupCount, null).then((_) { |
| 136 return _repeat(timingCount, times).then((_) { | 232 return _repeat(timingCount, times).then((_) { |
| 137 return oneTimeTearDown().then((_) { | 233 return oneTimeTearDown().then((_) { |
| 138 return new Future.value(new TimingResult(times)); | 234 return new Future.value(new TimingResult(times)); |
| 139 }); | 235 }); |
| 140 }); | 236 }); |
| 141 }); | 237 }); |
| 142 }); | 238 }); |
| 143 } | 239 } |
| 144 | 240 |
| 145 /** | 241 /** |
| 242 * Perform any operations that need to be performed before each iteration. |
| 243 */ |
| 244 Future setUp(); |
| 245 |
| 246 /** |
| 146 * Convert the given [relativePath] to an absolute path, by interpreting it | 247 * Convert the given [relativePath] to an absolute path, by interpreting it |
| 147 * relative to [sourceDirectory]. On Windows any forward slashes in | 248 * relative to [sourceDirectory]. On Windows any forward slashes in |
| 148 * [relativePath] are converted to backslashes. | 249 * [relativePath] are converted to backslashes. |
| 149 */ | 250 */ |
| 150 String sourcePath(String relativePath) { | 251 String sourcePath(String relativePath) { |
| 151 return join(sourceDirectory.path, relativePath.replaceAll('/', separator)); | 252 return join(sourceDirectory.path, relativePath.replaceAll('/', separator)); |
| 152 } | 253 } |
| 153 | 254 |
| 154 /** | 255 /** |
| 256 * Perform any operations that need to be performed after each iteration. |
| 257 */ |
| 258 Future tearDown(); |
| 259 |
| 260 /** |
| 155 * Write a source file with the given absolute [pathname] and [contents]. | 261 * Write a source file with the given absolute [pathname] and [contents]. |
| 156 * | 262 * |
| 157 * If the file didn't previously exist, it is created. If it did, it is | 263 * If the file didn't previously exist, it is created. If it did, it is |
| 158 * overwritten. | 264 * overwritten. |
| 159 * | 265 * |
| 160 * Parent directories are created as necessary. | 266 * Parent directories are created as necessary. |
| 161 */ | 267 */ |
| 162 void writeFile(String pathname, String contents) { | 268 void writeFile(String pathname, String contents) { |
| 163 new Directory(dirname(pathname)).createSync(recursive: true); | 269 new Directory(dirname(pathname)).createSync(recursive: true); |
| 164 new File(pathname).writeAsStringSync(contents); | 270 new File(pathname).writeAsStringSync(contents); |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 204 return new Future.value(); | 310 return new Future.value(); |
| 205 } | 311 } |
| 206 // Give the server a short time to comply with the shutdown request; if it | 312 // Give the server a short time to comply with the shutdown request; if it |
| 207 // doesn't exit, then forcibly terminate it. | 313 // doesn't exit, then forcibly terminate it. |
| 208 sendServerShutdown(); | 314 sendServerShutdown(); |
| 209 return server.exitCode.timeout(SHUTDOWN_TIMEOUT, onTimeout: () { | 315 return server.exitCode.timeout(SHUTDOWN_TIMEOUT, onTimeout: () { |
| 210 return server.kill(); | 316 return server.kill(); |
| 211 }); | 317 }); |
| 212 } | 318 } |
| 213 } | 319 } |
| 214 | |
| 215 /** | |
| 216 * Instances of the class [TimingResult] represent the timing information | |
| 217 * gathered while executing a given timing test. | |
| 218 */ | |
| 219 class TimingResult { | |
| 220 /** | |
| 221 * The amount of time spent executing each test, in nanoseconds. | |
| 222 */ | |
| 223 List<int> times; | |
| 224 | |
| 225 /** | |
| 226 * The number of nanoseconds in a millisecond. | |
| 227 */ | |
| 228 static int NANOSECONDS_PER_MILLISECOND = 1000000; | |
| 229 | |
| 230 /** | |
| 231 * Initialize a newly created timing result. | |
| 232 */ | |
| 233 TimingResult(this.times); | |
| 234 | |
| 235 /** | |
| 236 * The average amount of time spent executing a single iteration, in | |
| 237 * milliseconds. | |
| 238 */ | |
| 239 int get averageTime { | |
| 240 return totalTime ~/ times.length; | |
| 241 } | |
| 242 | |
| 243 /** | |
| 244 * The maximum amount of time spent executing a single iteration, in | |
| 245 * milliseconds. | |
| 246 */ | |
| 247 int get maxTime { | |
| 248 int maxTime = 0; | |
| 249 int count = times.length; | |
| 250 for (int i = 0; i < count; i++) { | |
| 251 maxTime = max(maxTime, times[i]); | |
| 252 } | |
| 253 return maxTime ~/ NANOSECONDS_PER_MILLISECOND; | |
| 254 } | |
| 255 | |
| 256 /** | |
| 257 * The minimum amount of time spent executing a single iteration, in | |
| 258 * milliseconds. | |
| 259 */ | |
| 260 int get minTime { | |
| 261 | |
| 262 int minTime = times[0]; | |
| 263 int count = times.length; | |
| 264 for (int i = 1; i < count; i++) { | |
| 265 minTime = min(minTime, times[i]); | |
| 266 } | |
| 267 return minTime ~/ NANOSECONDS_PER_MILLISECOND; | |
| 268 } | |
| 269 | |
| 270 /** | |
| 271 * The standard deviation of the times. | |
| 272 */ | |
| 273 double get standardDeviation { | |
| 274 return computeStandardDeviation(toMilliseconds(times)); | |
| 275 } | |
| 276 | |
| 277 /** | |
| 278 * The total amount of time spent executing the test, in milliseconds. | |
| 279 */ | |
| 280 int get totalTime { | |
| 281 int totalTime = 0; | |
| 282 int count = times.length; | |
| 283 for (int i = 0; i < count; i++) { | |
| 284 totalTime += times[i]; | |
| 285 } | |
| 286 return totalTime ~/ NANOSECONDS_PER_MILLISECOND; | |
| 287 } | |
| 288 | |
| 289 /** | |
| 290 * Compute the standard deviation of the given set of [values]. | |
| 291 */ | |
| 292 double computeStandardDeviation(List<int> values) { | |
| 293 int count = values.length; | |
| 294 double sumOfValues = 0.0; | |
| 295 for (int i = 0; i < count; i++) { | |
| 296 sumOfValues += values[i]; | |
| 297 } | |
| 298 double average = sumOfValues / count; | |
| 299 double sumOfDiffSquared = 0.0; | |
| 300 for (int i = 0; i < count; i++) { | |
| 301 double diff = values[i] - average; | |
| 302 sumOfDiffSquared += diff * diff; | |
| 303 } | |
| 304 return sqrt((sumOfDiffSquared / (count - 1))); | |
| 305 } | |
| 306 | |
| 307 /** | |
| 308 * Convert the given [times], expressed in nanoseconds, to times expressed in | |
| 309 * milliseconds. | |
| 310 */ | |
| 311 List<int> toMilliseconds(List<int> times) { | |
| 312 int count = times.length; | |
| 313 List<int> convertedValues = new List<int>(); | |
| 314 for (int i = 0; i < count; i++) { | |
| 315 convertedValues.add(times[i] ~/ NANOSECONDS_PER_MILLISECOND); | |
| 316 } | |
| 317 return convertedValues; | |
| 318 } | |
| 319 } | |
| OLD | NEW |