| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/python | |
| 2 | |
| 3 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| 4 # for details. All rights reserved. Use of this source code is governed by a | |
| 5 # BSD-style license that can be found in the LICENSE file. | |
| 6 | |
| 7 import datetime | |
| 8 import math | |
| 9 import optparse | |
| 10 import os | |
| 11 from os.path import dirname, abspath | |
| 12 import pickle | |
| 13 import platform | |
| 14 import random | |
| 15 import re | |
| 16 import shutil | |
| 17 import stat | |
| 18 import subprocess | |
| 19 import sys | |
| 20 import time | |
| 21 | |
| 22 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__))))) | |
| 23 TOP_LEVEL_DIR = abspath(os.path.join(dirname(abspath(__file__)), '..', '..', | |
| 24 '..')) | |
| 25 DART_REPO_LOC = abspath(os.path.join(dirname(abspath(__file__)), '..', '..', | |
| 26 '..', '..', '..', | |
| 27 'dart_checkout_for_perf_testing', | |
| 28 'dart')) | |
| 29 # How far back in time we want to test. | |
| 30 EARLIEST_REVISION = 33076 | |
| 31 sys.path.append(TOOLS_PATH) | |
| 32 sys.path.append(os.path.join(TOP_LEVEL_DIR, 'internal', 'tests')) | |
| 33 import post_results | |
| 34 import utils | |
| 35 | |
| 36 """This script runs to track performance and size progress of | |
| 37 different svn revisions. It tests to see if there a newer version of the code on | |
| 38 the server, and will sync and run the performance tests if so.""" | |
| 39 class TestRunner(object): | |
| 40 | |
| 41 def __init__(self): | |
| 42 self.verbose = False | |
| 43 self.has_shell = False | |
| 44 if platform.system() == 'Windows': | |
| 45 # On Windows, shell must be true to get the correct environment variables. | |
| 46 self.has_shell = True | |
| 47 self.current_revision_num = None | |
| 48 | |
| 49 def RunCmd(self, cmd_list, outfile=None, append=False, std_in=''): | |
| 50 """Run the specified command and print out any output to stdout. | |
| 51 | |
| 52 Args: | |
| 53 cmd_list: a list of strings that make up the command to run | |
| 54 outfile: a string indicating the name of the file that we should write | |
| 55 stdout to | |
| 56 append: True if we want to append to the file instead of overwriting it | |
| 57 std_in: a string that should be written to the process executing to | |
| 58 interact with it (if needed)""" | |
| 59 if self.verbose: | |
| 60 print ' '.join(cmd_list) | |
| 61 out = subprocess.PIPE | |
| 62 if outfile: | |
| 63 mode = 'w' | |
| 64 if append: | |
| 65 mode = 'a+' | |
| 66 out = open(outfile, mode) | |
| 67 if append: | |
| 68 # Annoying Windows "feature" -- append doesn't actually append unless | |
| 69 # you explicitly go to the end of the file. | |
| 70 # http://mail.python.org/pipermail/python-list/2009-October/1221859.html | |
| 71 out.seek(0, os.SEEK_END) | |
| 72 p = subprocess.Popen(cmd_list, stdout = out, stderr=subprocess.PIPE, | |
| 73 stdin=subprocess.PIPE, shell=self.has_shell) | |
| 74 output, stderr = p.communicate(std_in) | |
| 75 if output: | |
| 76 print output | |
| 77 if stderr: | |
| 78 print stderr | |
| 79 return output, stderr | |
| 80 | |
| 81 def RunBrowserPerfRunnerCmd(self, browser, url_path, file_path_to_test_code, | |
| 82 trace_file, code_root=''): | |
| 83 command_list = [os.path.join(DART_REPO_LOC, utils.GetBuildRoot( | |
| 84 utils.GuessOS(), 'release', 'ia32'), 'dart-sdk', 'bin', 'dart'), | |
| 85 '--package-root=%s' % os.path.join(file_path_to_test_code, 'packages'), | |
| 86 os.path.join(file_path_to_test_code, 'packages', | |
| 87 'browser_controller', 'browser_perf_testing.dart'), '--browser', | |
| 88 browser, '--test_path=%s' % url_path] | |
| 89 if code_root != '': | |
| 90 command_list += ['--code_root=%s' % code_root] | |
| 91 | |
| 92 if browser == 'dartium': | |
| 93 dartium_path = os.path.join(DART_REPO_LOC, 'client', 'tests', 'dartium') | |
| 94 if platform.system() == 'Windows': | |
| 95 dartium_path = os.path.join(dartium_path, 'chrome.exe'); | |
| 96 elif platform.system() == 'Darwin': | |
| 97 dartium_path = os.path.join(dartium_path, 'Chromium.app', 'Contents', | |
| 98 'MacOS', 'Chromium') | |
| 99 else: | |
| 100 dartium_path = os.path.join(dartium_path, 'chrome') | |
| 101 command_list += ['--executable=%s' % dartium_path] | |
| 102 | |
| 103 self.RunCmd(command_list, trace_file, append=True) | |
| 104 | |
| 105 def TimeCmd(self, cmd): | |
| 106 """Determine the amount of (real) time it takes to execute a given | |
| 107 command.""" | |
| 108 start = time.time() | |
| 109 self.RunCmd(cmd) | |
| 110 return time.time() - start | |
| 111 | |
| 112 def ClearOutUnversionedFiles(self): | |
| 113 """Remove all files that are unversioned by svn.""" | |
| 114 if os.path.exists(DART_REPO_LOC): | |
| 115 os.chdir(DART_REPO_LOC) | |
| 116 results, _ = self.RunCmd(['svn', 'st']) | |
| 117 for line in results.split('\n'): | |
| 118 if line.startswith('?'): | |
| 119 to_remove = line.split()[1] | |
| 120 if os.path.isdir(to_remove): | |
| 121 shutil.rmtree(to_remove, onerror=TestRunner._OnRmError) | |
| 122 else: | |
| 123 os.remove(to_remove) | |
| 124 elif any(line.startswith(status) for status in ['A', 'M', 'C', 'D']): | |
| 125 self.RunCmd(['svn', 'revert', line.split()[1]]) | |
| 126 | |
| 127 def GetArchive(self, archive_name): | |
| 128 """Wrapper around the pulling down a specific archive from Google Storage. | |
| 129 Adds a specific revision argument as needed. | |
| 130 Returns: A tuple of a boolean (True if we successfully downloaded the | |
| 131 binary), and the stdout and stderr from running this command.""" | |
| 132 num_fails = 0 | |
| 133 while True: | |
| 134 cmd = ['python', os.path.join(DART_REPO_LOC, 'tools', 'get_archive.py'), | |
| 135 archive_name] | |
| 136 if int(self.current_revision_num) != -1: | |
| 137 cmd += ['-r', str(self.current_revision_num)] | |
| 138 stdout, stderr = self.RunCmd(cmd) | |
| 139 if 'Please try again later' in stdout and num_fails < 20: | |
| 140 time.sleep(100) | |
| 141 num_fails += 1 | |
| 142 else: | |
| 143 break | |
| 144 return (num_fails < 20, stdout, stderr) | |
| 145 | |
| 146 def _Sync(self, revision_num=None): | |
| 147 """Update the repository to the latest or specified revision.""" | |
| 148 os.chdir(dirname(DART_REPO_LOC)) | |
| 149 self.ClearOutUnversionedFiles() | |
| 150 if not revision_num: | |
| 151 self.RunCmd(['gclient', 'sync']) | |
| 152 else: | |
| 153 self.RunCmd(['gclient', 'sync', '-r', str(revision_num), '-t']) | |
| 154 | |
| 155 shutil.copytree(os.path.join(TOP_LEVEL_DIR, 'internal'), | |
| 156 os.path.join(DART_REPO_LOC, 'internal')) | |
| 157 shutil.rmtree(os.path.join(DART_REPO_LOC, 'third_party', 'gsutil'), | |
| 158 onerror=TestRunner._OnRmError) | |
| 159 shutil.copytree(os.path.join(TOP_LEVEL_DIR, 'third_party', 'gsutil'), | |
| 160 os.path.join(DART_REPO_LOC, 'third_party', 'gsutil')) | |
| 161 shutil.copy(os.path.join(TOP_LEVEL_DIR, 'tools', 'get_archive.py'), | |
| 162 os.path.join(DART_REPO_LOC, 'tools', 'get_archive.py')) | |
| 163 shutil.copy( | |
| 164 os.path.join(TOP_LEVEL_DIR, 'tools', 'testing', 'run_selenium.py'), | |
| 165 os.path.join(DART_REPO_LOC, 'tools', 'testing', 'run_selenium.py')) | |
| 166 | |
| 167 @staticmethod | |
| 168 def _OnRmError(func, path, exc_info): | |
| 169 """On Windows, the output directory is marked as "Read Only," which causes | |
| 170 an error to be thrown when we use shutil.rmtree. This helper function | |
| 171 changes the permissions so we can still delete the directory.""" | |
| 172 if os.path.exists(path): | |
| 173 os.chmod(path, stat.S_IWRITE) | |
| 174 os.unlink(path) | |
| 175 | |
| 176 def SyncAndBuild(self, suites, revision_num=None): | |
| 177 """Make sure we have the latest version of of the repo, and build it. We | |
| 178 begin and end standing in DART_REPO_LOC. | |
| 179 | |
| 180 Args: | |
| 181 suites: The set of suites that we wish to build. | |
| 182 | |
| 183 Returns: | |
| 184 err_code = 1 if there was a problem building.""" | |
| 185 self._Sync(revision_num) | |
| 186 if not revision_num: | |
| 187 revision_num = SearchForRevision() | |
| 188 | |
| 189 self.current_revision_num = revision_num | |
| 190 success, stdout, stderr = self.GetArchive('sdk') | |
| 191 if (not os.path.exists(os.path.join( | |
| 192 DART_REPO_LOC, 'tools', 'get_archive.py')) or not success | |
| 193 or 'InvalidUriError' in stderr or "Couldn't download" in stdout or | |
| 194 'Unable to download' in stdout): | |
| 195 # Couldn't find the SDK on Google Storage. Build it locally. | |
| 196 | |
| 197 # TODO(efortuna): Currently always building ia32 architecture because we | |
| 198 # don't have test statistics for what's passing on x64. Eliminate arch | |
| 199 # specification when we have tests running on x64, too. | |
| 200 shutil.rmtree(os.path.join(os.getcwd(), | |
| 201 utils.GetBuildRoot(utils.GuessOS())), | |
| 202 onerror=TestRunner._OnRmError) | |
| 203 lines = self.RunCmd(['python', os.path.join('tools', 'build.py'), '-m', | |
| 204 'release', '--arch=ia32', 'create_sdk']) | |
| 205 | |
| 206 for line in lines: | |
| 207 if 'BUILD FAILED' in line: | |
| 208 # Someone checked in a broken build! Stop trying to make it work | |
| 209 # and wait to try again. | |
| 210 print 'Broken Build' | |
| 211 return 1 | |
| 212 return 0 | |
| 213 | |
| 214 def EnsureOutputDirectory(self, dir_name): | |
| 215 """Test that the listed directory name exists, and if not, create one for | |
| 216 our output to be placed. | |
| 217 | |
| 218 Args: | |
| 219 dir_name: the directory we will create if it does not exist.""" | |
| 220 dir_path = os.path.join(TOP_LEVEL_DIR, 'tools', | |
| 221 'testing', 'perf_testing', dir_name) | |
| 222 if not os.path.exists(dir_path): | |
| 223 os.makedirs(dir_path) | |
| 224 print 'Creating output directory ', dir_path | |
| 225 | |
| 226 def HasInterestingCode(self, revision_num=None): | |
| 227 """Tests if there are any versions of files that might change performance | |
| 228 results on the server. | |
| 229 | |
| 230 Returns: | |
| 231 (False, None): There is no interesting code to run. | |
| 232 (True, revisionNumber): There is interesting code to run at revision | |
| 233 revisionNumber. | |
| 234 (True, None): There is interesting code to run by syncing to the | |
| 235 tip-of-tree.""" | |
| 236 if not os.path.exists(DART_REPO_LOC): | |
| 237 self._Sync() | |
| 238 os.chdir(DART_REPO_LOC) | |
| 239 no_effect = ['dart/client', 'dart/compiler', 'dart/editor', | |
| 240 'dart/lib/html/doc', 'dart/pkg', 'dart/tests', 'dart/samples', | |
| 241 'dart/lib/dartdoc', 'dart/lib/i18n', 'dart/lib/unittest', | |
| 242 'dart/tools/dartc', 'dart/tools/get_archive.py', | |
| 243 'dart/tools/test.py', 'dart/tools/testing', | |
| 244 'dart/tools/utils', 'dart/third_party', 'dart/utils'] | |
| 245 definitely_yes = ['dart/samples/third_party/dromaeo', | |
| 246 'dart/lib/html/dart2js', 'dart/lib/html/dartium', | |
| 247 'dart/lib/scripts', 'dart/lib/src', | |
| 248 'dart/third_party/WebCore'] | |
| 249 def GetFileList(revision): | |
| 250 """Determine the set of files that were changed for a particular | |
| 251 revision.""" | |
| 252 # TODO(efortuna): This assumes you're using svn. Have a git fallback as | |
| 253 # well. Pass 'p' in if we have a new certificate for the svn server, we | |
| 254 # want to (p)ermanently accept it. | |
| 255 results, _ = self.RunCmd([ | |
| 256 'svn', 'log', 'http://dart.googlecode.com/svn/branches/bleeding_edge', | |
| 257 '-v', '-r', str(revision)], std_in='p\r\n') | |
| 258 results = results.split('\n') | |
| 259 if len(results) <= 3: | |
| 260 return [] | |
| 261 else: | |
| 262 # Trim off the details about revision number and commit message. We're | |
| 263 # only interested in the files that are changed. | |
| 264 results = results[3:] | |
| 265 changed_files = [] | |
| 266 for result in results: | |
| 267 if len(result) <= 1: | |
| 268 break | |
| 269 tokens = result.split() | |
| 270 if len(tokens) > 1: | |
| 271 changed_files += [tokens[1].replace('/branches/bleeding_edge/', '')] | |
| 272 return changed_files | |
| 273 | |
| 274 def HasPerfAffectingResults(files_list): | |
| 275 """Determine if this set of changed files might effect performance | |
| 276 tests.""" | |
| 277 def IsSafeFile(f): | |
| 278 if not any(f.startswith(prefix) for prefix in definitely_yes): | |
| 279 return any(f.startswith(prefix) for prefix in no_effect) | |
| 280 return False | |
| 281 return not all(IsSafeFile(f) for f in files_list) | |
| 282 | |
| 283 if revision_num: | |
| 284 return (HasPerfAffectingResults(GetFileList( | |
| 285 revision_num)), revision_num) | |
| 286 else: | |
| 287 latest_interesting_server_rev = None | |
| 288 while not latest_interesting_server_rev: | |
| 289 results, _ = self.RunCmd(['svn', 'st', '-u'], std_in='p\r\n') | |
| 290 if len(results.split('\n')) >= 2: | |
| 291 latest_interesting_server_rev = int( | |
| 292 results.split('\n')[-2].split()[-1]) | |
| 293 if self.backfill: | |
| 294 done_cls = list(UpdateSetOfDoneCls()) | |
| 295 done_cls.sort() | |
| 296 if done_cls: | |
| 297 last_done_cl = int(done_cls[-1]) | |
| 298 else: | |
| 299 last_done_cl = EARLIEST_REVISION | |
| 300 while latest_interesting_server_rev >= last_done_cl: | |
| 301 file_list = GetFileList(latest_interesting_server_rev) | |
| 302 if HasPerfAffectingResults(file_list): | |
| 303 return (True, latest_interesting_server_rev) | |
| 304 else: | |
| 305 UpdateSetOfDoneCls(latest_interesting_server_rev) | |
| 306 latest_interesting_server_rev -= 1 | |
| 307 else: | |
| 308 last_done_cl = int(SearchForRevision(DART_REPO_LOC)) + 1 | |
| 309 while last_done_cl <= latest_interesting_server_rev: | |
| 310 file_list = GetFileList(last_done_cl) | |
| 311 if HasPerfAffectingResults(file_list): | |
| 312 return (True, last_done_cl) | |
| 313 else: | |
| 314 UpdateSetOfDoneCls(last_done_cl) | |
| 315 last_done_cl += 1 | |
| 316 return (False, None) | |
| 317 | |
| 318 def GetOsDirectory(self): | |
| 319 """Specifies the name of the directory for the testing build of dart, which | |
| 320 has yet a different naming convention from utils.getBuildRoot(...).""" | |
| 321 if platform.system() == 'Windows': | |
| 322 return 'windows' | |
| 323 elif platform.system() == 'Darwin': | |
| 324 return 'macos' | |
| 325 else: | |
| 326 return 'linux' | |
| 327 | |
| 328 def ParseArgs(self): | |
| 329 parser = optparse.OptionParser() | |
| 330 parser.add_option('--suites', '-s', dest='suites', help='Run the specified ' | |
| 331 'comma-separated test suites from set: %s' % \ | |
| 332 ','.join(TestBuilder.AvailableSuiteNames()), | |
| 333 action='store', default=None) | |
| 334 parser.add_option('--forever', '-f', dest='continuous', help='Run this scri' | |
| 335 'pt forever, always checking for the next svn checkin', | |
| 336 action='store_true', default=False) | |
| 337 parser.add_option('--nobuild', '-n', dest='no_build', action='store_true', | |
| 338 help='Do not sync with the repository and do not ' | |
| 339 'rebuild.', default=False) | |
| 340 parser.add_option('--noupload', '-u', dest='no_upload', action='store_true', | |
| 341 help='Do not post the results of the run.', default=False) | |
| 342 parser.add_option('--notest', '-t', dest='no_test', action='store_true', | |
| 343 help='Do not run the tests.', default=False) | |
| 344 parser.add_option('--verbose', '-v', dest='verbose', | |
| 345 help='Print extra debug output', action='store_true', | |
| 346 default=False) | |
| 347 parser.add_option('--backfill', '-b', dest='backfill', | |
| 348 help='Backfill earlier CLs with additional results when ' | |
| 349 'there is idle time.', action='store_true', | |
| 350 default=False) | |
| 351 | |
| 352 args, ignored = parser.parse_args() | |
| 353 | |
| 354 if not args.suites: | |
| 355 suites = TestBuilder.AvailableSuiteNames() | |
| 356 else: | |
| 357 suites = [] | |
| 358 suitelist = args.suites.split(',') | |
| 359 for name in suitelist: | |
| 360 if name in TestBuilder.AvailableSuiteNames(): | |
| 361 suites.append(name) | |
| 362 else: | |
| 363 print ('Error: Invalid suite %s not in ' % name) + \ | |
| 364 '%s' % ','.join(TestBuilder.AvailableSuiteNames()) | |
| 365 sys.exit(1) | |
| 366 self.suite_names = suites | |
| 367 self.no_build = args.no_build | |
| 368 self.no_upload = args.no_upload | |
| 369 self.no_test = args.no_test | |
| 370 self.verbose = args.verbose | |
| 371 self.backfill = args.backfill | |
| 372 return args.continuous | |
| 373 | |
| 374 def RunTestSequence(self, revision_num=None, num_reruns=1): | |
| 375 """Run the set of commands to (possibly) build, run, and post the results | |
| 376 of our tests. Returns 0 on a successful run, 1 if we fail to post results or | |
| 377 the run failed, -1 if the build is broken. | |
| 378 """ | |
| 379 suites = [] | |
| 380 success = True | |
| 381 if not self.no_build and self.SyncAndBuild(suites, revision_num) == 1: | |
| 382 return -1 # The build is broken. | |
| 383 | |
| 384 if not self.current_revision_num: | |
| 385 self.current_revision_num = SearchForRevision(DART_REPO_LOC) | |
| 386 | |
| 387 for name in self.suite_names: | |
| 388 for run in range(num_reruns): | |
| 389 suites += [TestBuilder.MakeTest(name, self)] | |
| 390 | |
| 391 for test in suites: | |
| 392 success = success and test.Run() | |
| 393 if success: | |
| 394 return 0 | |
| 395 else: | |
| 396 return 1 | |
| 397 | |
| 398 | |
| 399 class Test(object): | |
| 400 """The base class to provide shared code for different tests we will run and | |
| 401 post. At a high level, each test has three visitors (the tester and the | |
| 402 file_processor) that perform operations on the test object.""" | |
| 403 | |
| 404 def __init__(self, result_folder_name, platform_list, variants, | |
| 405 values_list, test_runner, tester, file_processor, | |
| 406 extra_metrics=['Geo-Mean']): | |
| 407 """Args: | |
| 408 result_folder_name: The name of the folder where a tracefile of | |
| 409 performance results will be stored. | |
| 410 platform_list: A list containing the platform(s) that our data has been | |
| 411 run on. (command line, firefox, chrome, etc) | |
| 412 variants: A list specifying whether we hold data about Frog | |
| 413 generated code, plain JS code, or a combination of both, or | |
| 414 Dart depending on the test. | |
| 415 values_list: A list containing the type of data we will be graphing | |
| 416 (benchmarks, percentage passing, etc). | |
| 417 test_runner: Reference to the parent test runner object that notifies a | |
| 418 test when to run. | |
| 419 tester: The visitor that actually performs the test running mechanics. | |
| 420 file_processor: The visitor that processes files in the format | |
| 421 appropriate for this test. | |
| 422 extra_metrics: A list of any additional measurements we wish to keep | |
| 423 track of (such as the geometric mean of a set, the sum, etc).""" | |
| 424 self.result_folder_name = result_folder_name | |
| 425 # cur_time is used as a timestamp of when this performance test was run. | |
| 426 self.cur_time = str(time.mktime(datetime.datetime.now().timetuple())) | |
| 427 self.values_list = values_list | |
| 428 self.platform_list = platform_list | |
| 429 self.test_runner = test_runner | |
| 430 self.tester = tester | |
| 431 self.file_processor = file_processor | |
| 432 self.revision_dict = dict() | |
| 433 self.values_dict = dict() | |
| 434 self.extra_metrics = extra_metrics | |
| 435 # Initialize our values store. | |
| 436 for platform in platform_list: | |
| 437 self.revision_dict[platform] = dict() | |
| 438 self.values_dict[platform] = dict() | |
| 439 for f in variants: | |
| 440 self.revision_dict[platform][f] = dict() | |
| 441 self.values_dict[platform][f] = dict() | |
| 442 for val in values_list: | |
| 443 self.revision_dict[platform][f][val] = [] | |
| 444 self.values_dict[platform][f][val] = [] | |
| 445 for extra_metric in extra_metrics: | |
| 446 self.revision_dict[platform][f][extra_metric] = [] | |
| 447 self.values_dict[platform][f][extra_metric] = [] | |
| 448 | |
| 449 def IsValidCombination(self, platform, variant): | |
| 450 """Check whether data should be captured for this platform/variant | |
| 451 combination. | |
| 452 """ | |
| 453 if variant == 'dart_html' and platform != 'dartium': | |
| 454 return False | |
| 455 if platform == 'dartium' and (variant == 'js' or variant == 'dart2js_html'): | |
| 456 # Testing JavaScript performance on Dartium is a waste of time. Should be | |
| 457 # same as Chrome. | |
| 458 return False | |
| 459 if (platform == 'safari' and variant == 'dart2js' and | |
| 460 int(self.test_runner.current_revision_num) < 10193): | |
| 461 # In revision 10193 we fixed a bug that allows Safari 6 to run dart2js | |
| 462 # code. Since we can't change the Safari version on the machine, we're | |
| 463 # just not running | |
| 464 # for this case. | |
| 465 return False | |
| 466 return True | |
| 467 | |
| 468 def Run(self): | |
| 469 """Run the benchmarks/tests from the command line and plot the | |
| 470 results. | |
| 471 """ | |
| 472 for visitor in [self.tester, self.file_processor]: | |
| 473 visitor.Prepare() | |
| 474 | |
| 475 os.chdir(TOP_LEVEL_DIR) | |
| 476 self.test_runner.EnsureOutputDirectory(self.result_folder_name) | |
| 477 self.test_runner.EnsureOutputDirectory(os.path.join( | |
| 478 'old', self.result_folder_name)) | |
| 479 os.chdir(DART_REPO_LOC) | |
| 480 if not self.test_runner.no_test: | |
| 481 self.tester.RunTests() | |
| 482 | |
| 483 os.chdir(os.path.join(TOP_LEVEL_DIR, 'tools', 'testing', 'perf_testing')) | |
| 484 | |
| 485 files = os.listdir(self.result_folder_name) | |
| 486 post_success = True | |
| 487 for afile in files: | |
| 488 if not afile.startswith('.'): | |
| 489 should_move_file = self.file_processor.ProcessFile(afile, True) | |
| 490 if should_move_file: | |
| 491 shutil.move(os.path.join(self.result_folder_name, afile), | |
| 492 os.path.join('old', self.result_folder_name, afile)) | |
| 493 else: | |
| 494 post_success = False | |
| 495 | |
| 496 return post_success | |
| 497 | |
| 498 | |
| 499 class Tester(object): | |
| 500 """The base level visitor class that runs tests. It contains convenience | |
| 501 methods that many Tester objects use. Any class that would like to be a | |
| 502 TesterVisitor must implement the RunTests() method.""" | |
| 503 | |
| 504 def __init__(self, test): | |
| 505 self.test = test | |
| 506 | |
| 507 def Prepare(self): | |
| 508 """Perform any initial setup required before the test is run.""" | |
| 509 pass | |
| 510 | |
| 511 def AddSvnRevisionToTrace(self, outfile, browser = None): | |
| 512 """Add the svn version number to the provided tracefile.""" | |
| 513 def get_dartium_revision(): | |
| 514 version_file_name = os.path.join(DART_REPO_LOC, 'client', 'tests', | |
| 515 'dartium', 'LAST_VERSION') | |
| 516 try: | |
| 517 version_file = open(version_file_name, 'r') | |
| 518 version = version_file.read().split('.')[-3].split('-')[-1] | |
| 519 version_file.close() | |
| 520 return version | |
| 521 except IOError as e: | |
| 522 dartium_dir = os.path.join(DART_REPO_LOC, 'client', 'tests', 'dartium') | |
| 523 if (os.path.exists(os.path.join(dartium_dir, 'Chromium.app', 'Contents', | |
| 524 'MacOS', 'Chromium') or os.path.exists(os.path.join(dartium_dir, | |
| 525 'chrome.exe'))) or | |
| 526 os.path.exists(os.path.join(dartium_dir, 'chrome'))): | |
| 527 print "Error: VERSION file wasn't found." | |
| 528 return SearchForRevision() | |
| 529 else: | |
| 530 raise | |
| 531 | |
| 532 if browser and browser == 'dartium': | |
| 533 revision = get_dartium_revision() | |
| 534 self.test.test_runner.RunCmd(['echo', 'Revision: ' + revision], outfile) | |
| 535 else: | |
| 536 revision = SearchForRevision() | |
| 537 self.test.test_runner.RunCmd(['echo', 'Revision: ' + revision], outfile) | |
| 538 | |
| 539 | |
| 540 class Processor(object): | |
| 541 """The base level vistor class that processes tests. It contains convenience | |
| 542 methods that many File Processor objects use. Any class that would like to be | |
| 543 a ProcessorVisitor must implement the ProcessFile() method.""" | |
| 544 | |
| 545 SCORE = 'Score' | |
| 546 COMPILE_TIME = 'CompileTime' | |
| 547 CODE_SIZE = 'CodeSize' | |
| 548 | |
| 549 def __init__(self, test): | |
| 550 self.test = test | |
| 551 | |
| 552 def Prepare(self): | |
| 553 """Perform any initial setup required before the test is run.""" | |
| 554 pass | |
| 555 | |
| 556 def OpenTraceFile(self, afile, not_yet_uploaded): | |
| 557 """Find the correct location for the trace file, and open it. | |
| 558 Args: | |
| 559 afile: The tracefile name. | |
| 560 not_yet_uploaded: True if this file is to be found in a directory that | |
| 561 contains un-uploaded data. | |
| 562 Returns: A file object corresponding to the given file name.""" | |
| 563 file_path = os.path.join(self.test.result_folder_name, afile) | |
| 564 if not not_yet_uploaded: | |
| 565 file_path = os.path.join('old', file_path) | |
| 566 return open(file_path) | |
| 567 | |
| 568 def ReportResults(self, benchmark_name, score, platform, variant, | |
| 569 revision_number, metric): | |
| 570 """Store the results of the benchmark run. | |
| 571 Args: | |
| 572 benchmark_name: The name of the individual benchmark. | |
| 573 score: The numerical value of this benchmark. | |
| 574 platform: The platform the test was run on (firefox, command line, etc). | |
| 575 variant: Specifies whether the data was about generated Frog, js, a | |
| 576 combination of both, or Dart depending on the test. | |
| 577 revision_number: The revision of the code (and sometimes the revision of | |
| 578 dartium). | |
| 579 | |
| 580 Returns: True if the post was successful file.""" | |
| 581 return post_results.report_results(benchmark_name, score, platform, variant, | |
| 582 revision_number, metric) | |
| 583 | |
| 584 def CalculateGeometricMean(self, platform, variant, svn_revision): | |
| 585 """Calculate the aggregate geometric mean for JS and dart2js benchmark sets, | |
| 586 given two benchmark dictionaries.""" | |
| 587 geo_mean = 0 | |
| 588 if self.test.IsValidCombination(platform, variant): | |
| 589 for benchmark in self.test.values_list: | |
| 590 if not self.test.values_dict[platform][variant][benchmark]: | |
| 591 print 'Error determining mean for %s %s %s' % (platform, variant, | |
| 592 benchmark) | |
| 593 continue | |
| 594 geo_mean += math.log( | |
| 595 self.test.values_dict[platform][variant][benchmark][-1]) | |
| 596 | |
| 597 self.test.values_dict[platform][variant]['Geo-Mean'] += \ | |
| 598 [math.pow(math.e, geo_mean / len(self.test.values_list))] | |
| 599 self.test.revision_dict[platform][variant]['Geo-Mean'] += [svn_revision] | |
| 600 | |
| 601 def GetScoreType(self, benchmark_name): | |
| 602 """Determine the type of score for posting -- default is 'Score' (aka | |
| 603 Runtime), other options are CompileTime and CodeSize.""" | |
| 604 return self.SCORE | |
| 605 | |
| 606 | |
| 607 class RuntimePerformanceTest(Test): | |
| 608 """Super class for all runtime performance testing.""" | |
| 609 | |
| 610 def __init__(self, result_folder_name, platform_list, platform_type, | |
| 611 versions, benchmarks, test_runner, tester, file_processor): | |
| 612 """Args: | |
| 613 result_folder_name: The name of the folder where a tracefile of | |
| 614 performance results will be stored. | |
| 615 platform_list: A list containing the platform(s) that our data has been | |
| 616 run on. (command line, firefox, chrome, etc) | |
| 617 variants: A list specifying whether we hold data about Frog | |
| 618 generated code, plain JS code, or a combination of both, or | |
| 619 Dart depending on the test. | |
| 620 values_list: A list containing the type of data we will be graphing | |
| 621 (benchmarks, percentage passing, etc). | |
| 622 test_runner: Reference to the parent test runner object that notifies a | |
| 623 test when to run. | |
| 624 tester: The visitor that actually performs the test running mechanics. | |
| 625 file_processor: The visitor that processes files in the format | |
| 626 appropriate for this test. | |
| 627 extra_metrics: A list of any additional measurements we wish to keep | |
| 628 track of (such as the geometric mean of a set, the sum, etc).""" | |
| 629 super(RuntimePerformanceTest, self).__init__(result_folder_name, | |
| 630 platform_list, versions, benchmarks, test_runner, tester, | |
| 631 file_processor) | |
| 632 self.platform_list = platform_list | |
| 633 self.platform_type = platform_type | |
| 634 self.versions = versions | |
| 635 self.benchmarks = benchmarks | |
| 636 | |
| 637 | |
| 638 class BrowserTester(Tester): | |
| 639 @staticmethod | |
| 640 def GetBrowsers(add_dartium=True): | |
| 641 browsers = ['ff', 'chrome'] | |
| 642 if add_dartium: | |
| 643 browsers += ['dartium'] | |
| 644 has_shell = False | |
| 645 if platform.system() == 'Darwin': | |
| 646 browsers += ['safari'] | |
| 647 if platform.system() == 'Windows': | |
| 648 browsers += ['ie'] | |
| 649 has_shell = True | |
| 650 return browsers | |
| 651 | |
| 652 | |
| 653 class DromaeoTester(Tester): | |
| 654 DROMAEO_BENCHMARKS = { | |
| 655 'attr': ('attributes', [ | |
| 656 'getAttribute', | |
| 657 'element.property', | |
| 658 'setAttribute', | |
| 659 'element.property = value']), | |
| 660 'modify': ('modify', [ | |
| 661 'createElement', | |
| 662 'createTextNode', | |
| 663 'innerHTML', | |
| 664 'cloneNode', | |
| 665 'appendChild', | |
| 666 'insertBefore']), | |
| 667 'query': ('query', [ | |
| 668 'getElementById', | |
| 669 'getElementById (not in document)', | |
| 670 'getElementsByTagName(div)', | |
| 671 'getElementsByTagName(p)', | |
| 672 'getElementsByTagName(a)', | |
| 673 'getElementsByTagName(*)', | |
| 674 'getElementsByTagName (not in document)', | |
| 675 'getElementsByName', | |
| 676 'getElementsByName (not in document)']), | |
| 677 'traverse': ('traverse', [ | |
| 678 'firstChild', | |
| 679 'lastChild', | |
| 680 'nextSibling', | |
| 681 'previousSibling', | |
| 682 'childNodes']) | |
| 683 } | |
| 684 | |
| 685 # Use filenames that don't have unusual characters for benchmark names. | |
| 686 @staticmethod | |
| 687 def LegalizeFilename(str): | |
| 688 remap = { | |
| 689 ' ': '_', | |
| 690 '(': '_', | |
| 691 ')': '_', | |
| 692 '*': 'ALL', | |
| 693 '=': 'ASSIGN', | |
| 694 } | |
| 695 for (old, new) in remap.iteritems(): | |
| 696 str = str.replace(old, new) | |
| 697 return str | |
| 698 | |
| 699 # TODO(vsm): This is a hack to skip breaking tests. Triage this | |
| 700 # failure properly. The modify suite fails on 32-bit chrome, which | |
| 701 # is the default on mac and win. | |
| 702 @staticmethod | |
| 703 def GetValidDromaeoTags(): | |
| 704 tags = [tag for (tag, _) in DromaeoTester.DROMAEO_BENCHMARKS.values()] | |
| 705 if platform.system() == 'Darwin' or platform.system() == 'Windows': | |
| 706 tags.remove('modify') | |
| 707 return tags | |
| 708 | |
| 709 @staticmethod | |
| 710 def GetDromaeoBenchmarks(): | |
| 711 valid = DromaeoTester.GetValidDromaeoTags() | |
| 712 benchmarks = reduce(lambda l1,l2: l1+l2, | |
| 713 [tests for (tag, tests) in | |
| 714 DromaeoTester.DROMAEO_BENCHMARKS.values() | |
| 715 if tag in valid]) | |
| 716 return map(DromaeoTester.LegalizeFilename, benchmarks) | |
| 717 | |
| 718 @staticmethod | |
| 719 def GetDromaeoVersions(): | |
| 720 return ['js', 'dart2js_html', 'dart_html'] | |
| 721 | |
| 722 | |
| 723 class DromaeoTest(RuntimePerformanceTest): | |
| 724 """Runs Dromaeo tests, in the browser.""" | |
| 725 def __init__(self, test_runner): | |
| 726 super(DromaeoTest, self).__init__( | |
| 727 self.Name(), | |
| 728 BrowserTester.GetBrowsers(True), | |
| 729 'browser', | |
| 730 DromaeoTester.GetDromaeoVersions(), | |
| 731 DromaeoTester.GetDromaeoBenchmarks(), test_runner, | |
| 732 self.DromaeoPerfTester(self), | |
| 733 self.DromaeoFileProcessor(self)) | |
| 734 | |
| 735 @staticmethod | |
| 736 def Name(): | |
| 737 return 'dromaeo' | |
| 738 | |
| 739 class DromaeoPerfTester(DromaeoTester): | |
| 740 def RunTests(self): | |
| 741 """Run dromaeo in the browser.""" | |
| 742 success, _, _ = self.test.test_runner.GetArchive('dartium') | |
| 743 if not success: | |
| 744 # Unable to download dartium. Try later. | |
| 745 return | |
| 746 | |
| 747 # Build tests. | |
| 748 current_path = os.getcwd() | |
| 749 os.chdir(os.path.join(DART_REPO_LOC, 'samples', 'third_party', | |
| 750 'dromaeo')) | |
| 751 # Note: This uses debug on purpose, so that we can also run performance | |
| 752 # tests on pure Dart applications in Dartium. Pub --debug simply also | |
| 753 # moves the .dart files to the build directory. To ensure effective | |
| 754 # comparison, though, ensure that minify: true is set in your transformer | |
| 755 # compilation step in your pubspec. | |
| 756 stdout, _ = self.test.test_runner.RunCmd([os.path.join(DART_REPO_LOC, | |
| 757 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'), | |
| 758 'dart-sdk', 'bin', 'pub'), 'build', '--mode=debug']) | |
| 759 os.chdir(current_path) | |
| 760 if 'failed' in stdout: | |
| 761 return | |
| 762 | |
| 763 versions = DromaeoTester.GetDromaeoVersions() | |
| 764 | |
| 765 for browser in BrowserTester.GetBrowsers(): | |
| 766 for version_name in versions: | |
| 767 if not self.test.IsValidCombination(browser, version_name): | |
| 768 continue | |
| 769 version = DromaeoTest.DromaeoPerfTester.GetDromaeoUrlQuery( | |
| 770 browser, version_name) | |
| 771 self.test.trace_file = os.path.join(TOP_LEVEL_DIR, | |
| 772 'tools', 'testing', 'perf_testing', self.test.result_folder_name, | |
| 773 'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name)) | |
| 774 self.AddSvnRevisionToTrace(self.test.trace_file, browser) | |
| 775 url_path = '/'.join(['/code_root', 'build', 'web', 'index%s.html?%s'%( | |
| 776 '-dart' if version_name == 'dart_html' else '-js', | |
| 777 version)]) | |
| 778 | |
| 779 self.test.test_runner.RunBrowserPerfRunnerCmd(browser, url_path, | |
| 780 os.path.join(DART_REPO_LOC, 'samples', 'third_party', 'dromaeo'), | |
| 781 self.test.trace_file) | |
| 782 | |
| 783 @staticmethod | |
| 784 def GetDromaeoUrlQuery(browser, version): | |
| 785 version = version.replace('_','AND') | |
| 786 tags = DromaeoTester.GetValidDromaeoTags() | |
| 787 return 'OR'.join([ '%sAND%s' % (version, tag) for tag in tags]) | |
| 788 | |
| 789 | |
| 790 class DromaeoFileProcessor(Processor): | |
| 791 def ProcessFile(self, afile, should_post_file): | |
| 792 """Comb through the html to find the performance results. | |
| 793 Returns: True if we successfully posted our data to storage.""" | |
| 794 parts = afile.split('-') | |
| 795 browser = parts[2] | |
| 796 version = parts[3] | |
| 797 | |
| 798 bench_dict = self.test.values_dict[browser][version] | |
| 799 | |
| 800 f = self.OpenTraceFile(afile, should_post_file) | |
| 801 lines = f.readlines() | |
| 802 i = 0 | |
| 803 revision_num = 0 | |
| 804 revision_pattern = r'Revision: (\d+)' | |
| 805 suite_pattern = r'<div class="result-item done">(.+?)</ol></div>' | |
| 806 result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)' | |
| 807 | |
| 808 upload_success = True | |
| 809 for line in lines: | |
| 810 rev = re.match(revision_pattern, line.strip().replace('"', '')) | |
| 811 if rev: | |
| 812 revision_num = int(rev.group(1)) | |
| 813 continue | |
| 814 | |
| 815 suite_results = re.findall(suite_pattern, line) | |
| 816 if suite_results: | |
| 817 for suite_result in suite_results: | |
| 818 results = re.findall(r'<li>(.*?)</li>', suite_result) | |
| 819 if results: | |
| 820 for result in results: | |
| 821 r = re.match(result_pattern, result) | |
| 822 name = DromaeoTester.LegalizeFilename(r.group(1).strip(':')) | |
| 823 score = float(r.group(2)) | |
| 824 bench_dict[name] += [float(score)] | |
| 825 self.test.revision_dict[browser][version][name] += \ | |
| 826 [revision_num] | |
| 827 if not self.test.test_runner.no_upload and should_post_file: | |
| 828 upload_success = upload_success and self.ReportResults( | |
| 829 name, score, browser, version, revision_num, | |
| 830 self.GetScoreType(name)) | |
| 831 else: | |
| 832 upload_success = False | |
| 833 | |
| 834 f.close() | |
| 835 self.CalculateGeometricMean(browser, version, revision_num) | |
| 836 return upload_success | |
| 837 | |
| 838 class TodoMVCTester(BrowserTester): | |
| 839 @staticmethod | |
| 840 def GetVersions(): | |
| 841 return ['js', 'dart2js_html', 'dart_html'] | |
| 842 | |
| 843 @staticmethod | |
| 844 def GetBenchmarks(): | |
| 845 return ['TodoMVCstartup'] | |
| 846 | |
| 847 class TodoMVCStartupTest(RuntimePerformanceTest): | |
| 848 """Start up TodoMVC and see how long it takes to start.""" | |
| 849 def __init__(self, test_runner): | |
| 850 super(TodoMVCStartupTest, self).__init__( | |
| 851 self.Name(), | |
| 852 BrowserTester.GetBrowsers(True), | |
| 853 'browser', | |
| 854 TodoMVCTester.GetVersions(), | |
| 855 TodoMVCTester.GetBenchmarks(), test_runner, | |
| 856 self.TodoMVCStartupTester(self), | |
| 857 self.TodoMVCFileProcessor(self)) | |
| 858 | |
| 859 @staticmethod | |
| 860 def Name(): | |
| 861 return 'todoMvcStartup' | |
| 862 | |
| 863 class TodoMVCStartupTester(BrowserTester): | |
| 864 def RunTests(self): | |
| 865 """Run dromaeo in the browser.""" | |
| 866 success, _, _ = self.test.test_runner.GetArchive('dartium') | |
| 867 if not success: | |
| 868 # Unable to download dartium. Try later. | |
| 869 return | |
| 870 | |
| 871 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') | |
| 872 current_path = os.getcwd() | |
| 873 | |
| 874 os.chdir(os.path.join(DART_REPO_LOC, 'samples', 'third_party', | |
| 875 'todomvc_performance')) | |
| 876 self.test.test_runner.RunCmd([os.path.join(DART_REPO_LOC, | |
| 877 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'), | |
| 878 'dart-sdk', 'bin', 'pub'), 'build', '--mode=debug']) | |
| 879 os.chdir('js_todomvc'); | |
| 880 self.test.test_runner.RunCmd([os.path.join(DART_REPO_LOC, | |
| 881 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'), | |
| 882 'dart-sdk', 'bin', 'pub'), 'get']) | |
| 883 | |
| 884 versions = TodoMVCTester.GetVersions() | |
| 885 | |
| 886 for browser in BrowserTester.GetBrowsers(): | |
| 887 for version_name in versions: | |
| 888 if not self.test.IsValidCombination(browser, version_name): | |
| 889 continue | |
| 890 self.test.trace_file = os.path.join(TOP_LEVEL_DIR, | |
| 891 'tools', 'testing', 'perf_testing', self.test.result_folder_name, | |
| 892 'todoMvcStartup-%s-%s-%s' % (self.test.cur_time, browser, | |
| 893 version_name)) | |
| 894 self.AddSvnRevisionToTrace(self.test.trace_file, browser) | |
| 895 | |
| 896 if version_name == 'js': | |
| 897 code_root = os.path.join(DART_REPO_LOC, 'samples', 'third_party', | |
| 898 'todomvc_performance', 'js_todomvc') | |
| 899 self.test.test_runner.RunBrowserPerfRunnerCmd(browser, | |
| 900 '/code_root/index.html', code_root, self.test.trace_file, | |
| 901 code_root) | |
| 902 else: | |
| 903 self.test.test_runner.RunBrowserPerfRunnerCmd(browser, | |
| 904 '/code_root/build/web/startup-performance.html', os.path.join( | |
| 905 DART_REPO_LOC, 'samples', 'third_party', 'todomvc_performance'), | |
| 906 self.test.trace_file) | |
| 907 | |
| 908 class TodoMVCFileProcessor(Processor): | |
| 909 def ProcessFile(self, afile, should_post_file): | |
| 910 """Comb through the html to find the performance results. | |
| 911 Returns: True if we successfully posted our data to storage.""" | |
| 912 parts = afile.split('-') | |
| 913 browser = parts[2] | |
| 914 version = parts[3] | |
| 915 | |
| 916 bench_dict = self.test.values_dict[browser][version] | |
| 917 | |
| 918 f = self.OpenTraceFile(afile, should_post_file) | |
| 919 lines = f.readlines() | |
| 920 i = 0 | |
| 921 revision_num = 0 | |
| 922 revision_pattern = r'Revision: (\d+)' | |
| 923 result_pattern = r'The startup time is (\d+)' | |
| 924 | |
| 925 upload_success = True | |
| 926 for line in lines: | |
| 927 rev = re.match(revision_pattern, line.strip().replace('"', '')) | |
| 928 if rev: | |
| 929 revision_num = int(rev.group(1)) | |
| 930 continue | |
| 931 | |
| 932 results = re.search(result_pattern, line) | |
| 933 if results: | |
| 934 score = float(results.group(1)) | |
| 935 name = TodoMVCTester.GetBenchmarks()[0] | |
| 936 bench_dict[name] += [float(score)] | |
| 937 self.test.revision_dict[browser][version][name] += \ | |
| 938 [revision_num] | |
| 939 if not self.test.test_runner.no_upload and should_post_file: | |
| 940 upload_success = upload_success and self.ReportResults( | |
| 941 name, score, browser, version, revision_num, | |
| 942 self.GetScoreType(name)) | |
| 943 | |
| 944 f.close() | |
| 945 self.CalculateGeometricMean(browser, version, revision_num) | |
| 946 return upload_success | |
| 947 | |
| 948 | |
| 949 class TestBuilder(object): | |
| 950 """Construct the desired test object.""" | |
| 951 available_suites = dict((suite.Name(), suite) for suite in [ | |
| 952 DromaeoTest, TodoMVCStartupTest]) | |
| 953 | |
| 954 @staticmethod | |
| 955 def MakeTest(test_name, test_runner): | |
| 956 return TestBuilder.available_suites[test_name](test_runner) | |
| 957 | |
| 958 @staticmethod | |
| 959 def AvailableSuiteNames(): | |
| 960 return TestBuilder.available_suites.keys() | |
| 961 | |
| 962 | |
| 963 def SearchForRevision(directory = None): | |
| 964 """Find the current revision number in the desired directory. If directory is | |
| 965 None, find the revision number in the current directory.""" | |
| 966 def FindRevision(svn_info_command): | |
| 967 p = subprocess.Popen(svn_info_command, stdout = subprocess.PIPE, | |
| 968 stderr = subprocess.STDOUT, | |
| 969 shell = (platform.system() == 'Windows')) | |
| 970 output, _ = p.communicate() | |
| 971 for line in output.split('\n'): | |
| 972 if 'Revision' in line: | |
| 973 return int(line.split()[1]) | |
| 974 return -1 | |
| 975 | |
| 976 cwd = os.getcwd() | |
| 977 if not directory: | |
| 978 directory = cwd | |
| 979 os.chdir(directory) | |
| 980 revision_num = int(FindRevision(['svn', 'info'])) | |
| 981 if revision_num == -1: | |
| 982 revision_num = int(FindRevision(['git', 'svn', 'info'])) | |
| 983 os.chdir(cwd) | |
| 984 return str(revision_num) | |
| 985 | |
| 986 | |
| 987 def UpdateSetOfDoneCls(revision_num=None): | |
| 988 """Update the set of CLs that do not need additional performance runs. | |
| 989 Args: | |
| 990 revision_num: an additional number to be added to the 'done set' | |
| 991 """ | |
| 992 filename = os.path.join(TOP_LEVEL_DIR, 'cached_results.txt') | |
| 993 if not os.path.exists(filename): | |
| 994 f = open(filename, 'w') | |
| 995 results = set() | |
| 996 pickle.dump(results, f) | |
| 997 f.close() | |
| 998 f = open(filename, 'r+') | |
| 999 result_set = pickle.load(f) | |
| 1000 if revision_num: | |
| 1001 f.seek(0) | |
| 1002 result_set.add(revision_num) | |
| 1003 pickle.dump(result_set, f) | |
| 1004 f.close() | |
| 1005 return result_set | |
| 1006 | |
| 1007 | |
| 1008 def FillInBackHistory(results_set, runner): | |
| 1009 """Fill in back history performance data. This is done one of two ways, with | |
| 1010 equal probability of trying each way (falling back on the sequential version | |
| 1011 as our data becomes more densely populated).""" | |
| 1012 revision_num = int(SearchForRevision(DART_REPO_LOC)) | |
| 1013 has_run_extra = False | |
| 1014 | |
| 1015 def TryToRunAdditional(revision_number): | |
| 1016 """Determine the number of results we have stored for a particular revision | |
| 1017 number, and if it is less than 10, run some extra tests. | |
| 1018 Args: | |
| 1019 - revision_number: the revision whose performance we want to potentially | |
| 1020 test. | |
| 1021 Returns: True if we successfully ran some additional tests.""" | |
| 1022 if not runner.HasInterestingCode(revision_number)[0]: | |
| 1023 results_set = UpdateSetOfDoneCls(revision_number) | |
| 1024 return False | |
| 1025 a_test = TestBuilder.MakeTest(runner.suite_names[0], runner) | |
| 1026 benchmark_name = a_test.values_list[0] | |
| 1027 platform_name = a_test.platform_list[0] | |
| 1028 variant = a_test.values_dict[platform_name].keys()[0] | |
| 1029 num_results = post_results.get_num_results(benchmark_name, | |
| 1030 platform_name, variant, revision_number, | |
| 1031 a_test.file_processor.GetScoreType(benchmark_name)) | |
| 1032 if num_results < 10: | |
| 1033 # Run at most two more times. | |
| 1034 if num_results > 8: | |
| 1035 reruns = 10 - num_results | |
| 1036 else: | |
| 1037 reruns = 2 | |
| 1038 run = runner.RunTestSequence(revision_num=str(revision_number), | |
| 1039 num_reruns=reruns) | |
| 1040 if num_results >= 10 or run == 0 and num_results + reruns >= 10: | |
| 1041 results_set = UpdateSetOfDoneCls(revision_number) | |
| 1042 elif run != 0: | |
| 1043 return False | |
| 1044 return True | |
| 1045 | |
| 1046 # Try to get up to 10 runs of each CL, starting with the most recent | |
| 1047 # CL that does not yet have 10 runs. But only perform a set of extra | |
| 1048 # runs at most 2 at a time before checking to see if new code has been | |
| 1049 # checked in. | |
| 1050 while revision_num > EARLIEST_REVISION and not has_run_extra: | |
| 1051 if revision_num not in results_set: | |
| 1052 has_run_extra = TryToRunAdditional(revision_num) | |
| 1053 revision_num -= 1 | |
| 1054 if not has_run_extra: | |
| 1055 # No more extra back-runs to do (for now). Wait for new code. | |
| 1056 time.sleep(200) | |
| 1057 return results_set | |
| 1058 | |
| 1059 | |
| 1060 def main(): | |
| 1061 runner = TestRunner() | |
| 1062 continuous = runner.ParseArgs() | |
| 1063 | |
| 1064 if not os.path.exists(DART_REPO_LOC): | |
| 1065 os.mkdir(dirname(DART_REPO_LOC)) | |
| 1066 os.chdir(dirname(DART_REPO_LOC)) | |
| 1067 p = subprocess.Popen('gclient config https://dart.googlecode.com/svn/' + | |
| 1068 'branches/bleeding_edge/deps/all.deps', | |
| 1069 stdout=subprocess.PIPE, stderr=subprocess.PIPE, | |
| 1070 shell=True) | |
| 1071 p.communicate() | |
| 1072 if continuous: | |
| 1073 while True: | |
| 1074 results_set = UpdateSetOfDoneCls() | |
| 1075 (is_interesting, interesting_rev_num) = runner.HasInterestingCode() | |
| 1076 if is_interesting: | |
| 1077 runner.RunTestSequence(interesting_rev_num) | |
| 1078 else: | |
| 1079 if runner.backfill: | |
| 1080 results_set = FillInBackHistory(results_set, runner) | |
| 1081 else: | |
| 1082 time.sleep(200) | |
| 1083 else: | |
| 1084 runner.RunTestSequence() | |
| 1085 | |
| 1086 if __name__ == '__main__': | |
| 1087 main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1576153002:
Remove the Dromaeo and TodoMVC samples. (Closed)
Base URL: git@github.com:dart-lang/sdk.git@master