| OLD | NEW |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import argparse | 5 import argparse |
| 6 import json |
| 7 import logging |
| 6 import os | 8 import os |
| 7 import logging | |
| 8 import platform | 9 import platform |
| 9 import re | 10 import re |
| 10 import subprocess | 11 import subprocess |
| 12 import sys |
| 11 import urllib2 | 13 import urllib2 |
| 12 import json | 14 |
| 13 | 15 |
| 14 from core import path_util | 16 from core import path_util |
| 15 | 17 |
| 16 from telemetry import benchmark | 18 from telemetry import benchmark |
| 17 from telemetry import decorators | 19 from telemetry import decorators |
| 18 from telemetry.core import discover | 20 from telemetry.core import discover |
| 19 from telemetry.util import command_line | 21 from telemetry.util import command_line |
| 20 from telemetry.util import matching | 22 from telemetry.util import matching |
| 21 | 23 |
| 22 | 24 |
| 23 CHROMIUM_CONFIG_FILENAME = 'tools/run-perf-test.cfg' | |
| 24 BLINK_CONFIG_FILENAME = 'Tools/run-perf-test.cfg' | |
| 25 SUCCESS, NO_CHANGES, ERROR = range(3) | |
| 26 # Unsupported Perf bisect bots. | 25 # Unsupported Perf bisect bots. |
| 27 EXCLUDED_BOTS = { | 26 EXCLUDED_BOTS = { |
| 28 'win_xp_perf_bisect', # Goma issues: crbug.com/330900 | 27 'win_xp_perf_bisect', # Goma issues: crbug.com/330900 |
| 29 'win_perf_bisect_builder', | 28 'win_perf_bisect_builder', |
| 30 'win64_nv_tester', | 29 'win64_nv_tester', |
| 31 'winx64_bisect_builder', | 30 'winx64_bisect_builder', |
| 32 'linux_perf_bisect_builder', | 31 'linux_perf_bisect_builder', |
| 33 'mac_perf_bisect_builder', | 32 'mac_perf_bisect_builder', |
| 34 'android_perf_bisect_builder', | 33 'android_perf_bisect_builder', |
| 35 'android_arm64_perf_bisect_builder', | 34 'android_arm64_perf_bisect_builder', |
| (...skipping 14 matching lines...) Expand all Loading... |
| 50 | 49 |
| 51 INCLUDE_BOTS = [ | 50 INCLUDE_BOTS = [ |
| 52 'all', | 51 'all', |
| 53 'all-win', | 52 'all-win', |
| 54 'all-mac', | 53 'all-mac', |
| 55 'all-linux', | 54 'all-linux', |
| 56 'all-android' | 55 'all-android' |
| 57 ] | 56 ] |
| 58 | 57 |
| 59 # Default try bot to use incase builbot is unreachable. | 58 # Default try bot to use incase builbot is unreachable. |
| 60 DEFAULT_TRYBOTS = [ | 59 DEFAULT_TRYBOTS = [ |
| 61 'linux_perf_bisect', | 60 'linux_perf_bisect', |
| 62 'mac_10_11_perf_bisect', | 61 'mac_10_11_perf_bisect', |
| 63 'winx64_10_perf_bisect', | 62 'winx64_10_perf_bisect', |
| 64 'android_s5_perf_bisect', | 63 'android_s5_perf_bisect', |
| 65 ] | 64 ] |
| 66 | 65 |
| 67 assert not set(DEFAULT_TRYBOTS) & set(EXCLUDED_BOTS), ( 'A trybot cannot ' | 66 CHROMIUM_SRC_PATH = path_util.GetChromiumSrcDir() |
| 68 'present in both Default as well as Excluded bots lists.') | 67 REPO_INFO_MAP = { |
| 68 'src': { |
| 69 'src': 'src', |
| 70 'url': 'https://chromium.googlesource.com/chromium/src.git', |
| 71 }, |
| 72 'v8': { |
| 73 'src': 'src/v8', |
| 74 'url': 'https://chromium.googlesource.com/v8/v8.git', |
| 75 }, |
| 76 'skia': { |
| 77 'src': 'src/third_party/skia', |
| 78 'url': 'https://chromium.googlesource.com/skia.git', |
| 79 }, |
| 80 'angle': { |
| 81 'src': 'src/third_party/angle', |
| 82 'url': 'https://chromium.googlesource.com/angle/angle.git', |
| 83 } |
| 84 } |
| 85 |
| 86 assert not set(DEFAULT_TRYBOTS) & set(EXCLUDED_BOTS), ( |
| 87 'A trybot cannot present in both Default as well as Excluded bots lists.') |
| 88 |
| 69 | 89 |
| 70 class TrybotError(Exception): | 90 class TrybotError(Exception): |
| 71 | 91 |
| 72 def __str__(self): | 92 def __str__(self): |
| 73 return '%s\nError running tryjob.' % self.args[0] | 93 return '(ERROR) Perf Try Job: %s' % self.args[0] |
| 74 | 94 |
| 75 | 95 |
| 76 def _GetTrybotList(builders): | 96 def _GetTrybotList(builders): |
| 77 builders = ['%s' % bot.replace('_perf_bisect', '').replace('_', '-') | 97 builders = ['%s' % bot.replace('_perf_bisect', '').replace('_', '-') |
| 78 for bot in builders] | 98 for bot in builders] |
| 79 builders.extend(INCLUDE_BOTS) | 99 builders.extend(INCLUDE_BOTS) |
| 80 return sorted(builders) | 100 return sorted(builders) |
| 81 | 101 |
| 82 | 102 |
| 83 def _GetBotPlatformFromTrybotName(trybot_name): | 103 def _GetBotPlatformFromTrybotName(trybot_name): |
| 84 os_names = ['linux', 'android', 'mac', 'win'] | 104 os_names = ['linux', 'android', 'mac', 'win'] |
| 85 try: | 105 try: |
| 86 return next(b for b in os_names if b in trybot_name) | 106 return next(b for b in os_names if b in trybot_name) |
| 87 except StopIteration: | 107 except StopIteration: |
| 88 raise TrybotError('Trybot "%s" unsupported for tryjobs.' % trybot_name) | 108 raise TrybotError('Trybot "%s" unsupported for tryjobs.' % trybot_name) |
| 89 | 109 |
| 90 | 110 |
| 91 def _GetBuilderNames(trybot_name, builders): | 111 def _GetBuilderNames(trybot_name, builders): |
| 92 """ Return platform and its available bot name as dictionary.""" | 112 """Return platform and its available bot name as dictionary.""" |
| 93 os_names = ['linux', 'android', 'mac', 'win'] | 113 os_names = ['linux', 'android', 'mac', 'win'] |
| 94 if 'all' not in trybot_name: | 114 if 'all' not in trybot_name: |
| 95 bot = ['%s_perf_bisect' % trybot_name.replace('-', '_')] | 115 bot = ['%s_perf_bisect' % trybot_name.replace('-', '_')] |
| 96 bot_platform = _GetBotPlatformFromTrybotName(trybot_name) | 116 bot_platform = _GetBotPlatformFromTrybotName(trybot_name) |
| 97 if 'x64' in trybot_name: | 117 if 'x64' in trybot_name: |
| 98 bot_platform += '-x64' | 118 bot_platform += '-x64' |
| 99 return {bot_platform: bot} | 119 return {bot_platform: bot} |
| 100 | 120 |
| 101 platform_and_bots = {} | 121 platform_and_bots = {} |
| 102 for os_name in os_names: | 122 for os_name in os_names: |
| (...skipping 15 matching lines...) Expand all Loading... |
| 118 if 'all-mac' in trybot_name: | 138 if 'all-mac' in trybot_name: |
| 119 return {'mac': platform_and_bots['mac']} | 139 return {'mac': platform_and_bots['mac']} |
| 120 if 'all-android' in trybot_name: | 140 if 'all-android' in trybot_name: |
| 121 return {'android': platform_and_bots['android']} | 141 return {'android': platform_and_bots['android']} |
| 122 if 'all-linux' in trybot_name: | 142 if 'all-linux' in trybot_name: |
| 123 return {'linux': platform_and_bots['linux']} | 143 return {'linux': platform_and_bots['linux']} |
| 124 | 144 |
| 125 return platform_and_bots | 145 return platform_and_bots |
| 126 | 146 |
| 127 | 147 |
| 148 _GIT_CMD = 'git' |
| 149 |
| 150 |
| 151 if platform.system() == 'Windows': |
| 152 # On windows, the git command is installed as 'git.bat' |
| 153 _GIT_CMD = 'git.bat' |
| 154 |
| 155 |
| 128 def _RunProcess(cmd): | 156 def _RunProcess(cmd): |
| 129 logging.debug('Running process: "%s"', ' '.join(cmd)) | 157 logging.debug('Running process: "%s"', ' '.join(cmd)) |
| 130 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | 158 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| 131 out, err = proc.communicate() | 159 out, err = proc.communicate() |
| 132 returncode = proc.poll() | 160 returncode = proc.poll() |
| 133 return (returncode, out, err) | 161 return (returncode, out, err) |
| 134 | 162 |
| 135 | 163 |
| 136 _GIT_CMD = 'git' | 164 def RunGit(cmd, msg_on_error='', ignore_return_code=False): |
| 137 if platform.system() == 'Windows': | 165 try: |
| 138 # On windows, the git command is installed as 'git.bat' | 166 output = subprocess.check_output([_GIT_CMD] + cmd, stderr=subprocess.STDOUT) |
| 139 _GIT_CMD = 'git.bat' | 167 return output.decode(sys.stdout.encoding).strip() |
| 168 except subprocess.CalledProcessError as e: |
| 169 if ignore_return_code: |
| 170 return None |
| 171 raise TrybotError('%s. \n%s' % ( |
| 172 msg_on_error, e.output.decode(sys.stdout.encoding))) |
| 140 | 173 |
| 141 | 174 |
| 142 class Trybot(command_line.ArgParseCommand): | 175 class Trybot(command_line.ArgParseCommand): |
| 143 """ Run telemetry perf benchmark on trybot """ | 176 """Run telemetry perf benchmark on trybot.""" |
| 144 | 177 |
| 145 usage = 'botname benchmark_name [<benchmark run options>]' | 178 usage = 'botname benchmark_name [<benchmark run options>]' |
| 146 _builders = None | 179 _builders = None |
| 147 | 180 |
| 148 def __init__(self): | 181 def __init__(self): |
| 149 self._builder_names = None | 182 self._builder_names = None |
| 150 | 183 |
| 151 @classmethod | 184 @classmethod |
| 152 def _GetBuilderList(cls): | 185 def _GetBuilderList(cls): |
| 153 if not cls._builders: | 186 if not cls._builders: |
| 154 try: | 187 try: |
| 155 f = urllib2.urlopen( | 188 f = urllib2.urlopen( |
| 156 ('https://build.chromium.org/p/tryserver.chromium.perf/json/' | 189 ('https://build.chromium.org/p/tryserver.chromium.perf/json/' |
| 157 'builders'), | 190 'builders'), |
| 158 timeout=5) | 191 timeout=5) |
| 159 # In case of any kind of exception, allow tryjobs to use default trybots. | 192 # In case of any kind of exception, allow tryjobs to use default trybots. |
| 160 # Possible exception are ssl.SSLError, urllib2.URLError, | 193 # Possible exception are ssl.SSLError, urllib2.URLError, |
| 161 # socket.timeout, socket.error. | 194 # socket.timeout, socket.error. |
| 162 except Exception: | 195 except Exception: # pylint: disable=broad-except |
| 163 # Incase of any exception return default trybots. | 196 # Incase of any exception return default trybots. |
| 164 print ('WARNING: Unable to reach builbot to retrieve trybot ' | 197 print ('WARNING: Unable to reach builbot to retrieve trybot ' |
| 165 'information, tryjob will use default trybots.') | 198 'information, tryjob will use default trybots.') |
| 166 cls._builders = DEFAULT_TRYBOTS | 199 cls._builders = DEFAULT_TRYBOTS |
| 167 else: | 200 else: |
| 168 builders = json.loads(f.read()).keys() | 201 builders = json.loads(f.read()).keys() |
| 169 # Exclude unsupported bots like win xp and some dummy bots. | 202 # Exclude unsupported bots like win xp and some dummy bots. |
| 170 cls._builders = [bot for bot in builders if bot not in EXCLUDED_BOTS] | 203 cls._builders = [bot for bot in builders if bot not in EXCLUDED_BOTS] |
| 171 | 204 |
| 172 return cls._builders | 205 return cls._builders |
| 173 | 206 |
| 174 def _InitializeBuilderNames(self, trybot): | 207 def _InitializeBuilderNames(self, trybot): |
| 175 self._builder_names = _GetBuilderNames(trybot, self._GetBuilderList()) | 208 self._builder_names = _GetBuilderNames(trybot, self._GetBuilderList()) |
| 176 | 209 |
| 177 @classmethod | 210 @classmethod |
| 178 def CreateParser(cls): | 211 def CreateParser(cls): |
| 179 parser = argparse.ArgumentParser( | 212 parser = argparse.ArgumentParser( |
| 180 ('Run telemetry benchmarks on trybot. You can add all the benchmark ' | 213 ('Run telemetry benchmarks on trybot. You can add all the benchmark ' |
| 181 'options available except the --browser option'), | 214 'options available except the --browser option'), |
| 182 formatter_class=argparse.RawTextHelpFormatter) | 215 formatter_class=argparse.RawTextHelpFormatter) |
| 183 return parser | 216 return parser |
| 184 | 217 |
| 185 @classmethod | 218 @classmethod |
| 186 def ProcessCommandLineArgs(cls, parser, options, extra_args, environment): | 219 def ProcessCommandLineArgs(cls, parser, options, extra_args, environment): |
| 187 del environment # unused | 220 del environment # unused |
| 188 for arg in extra_args: | 221 for arg in extra_args: |
| 189 if arg == '--browser' or arg.startswith('--browser='): | 222 if arg == '--browser' or arg.startswith('--browser='): |
| 190 parser.error('--browser=... is not allowed when running trybot.') | 223 parser.error('--browser=... is not allowed when running trybot.') |
| 191 all_benchmarks = discover.DiscoverClasses( | 224 all_benchmarks = discover.DiscoverClasses( |
| 192 start_dir=path_util.GetPerfBenchmarksDir(), | 225 start_dir=path_util.GetPerfBenchmarksDir(), |
| 193 top_level_dir=path_util.GetPerfDir(), | 226 top_level_dir=path_util.GetPerfDir(), |
| 194 base_class=benchmark.Benchmark).values() | 227 base_class=benchmark.Benchmark).values() |
| 195 all_benchmark_names = [b.Name() for b in all_benchmarks] | 228 all_benchmark_names = [b.Name() for b in all_benchmarks] |
| 196 all_benchmarks_by_names = {b.Name(): b for b in all_benchmarks} | 229 all_benchmarks_by_names = {b.Name(): b for b in all_benchmarks} |
| 197 benchmark_class = all_benchmarks_by_names.get(options.benchmark_name, None) | 230 benchmark_class = all_benchmarks_by_names.get(options.benchmark_name, None) |
| 198 if not benchmark_class: | 231 if not benchmark_class: |
| 199 possible_benchmark_names = matching.GetMostLikelyMatchedObject( | 232 possible_benchmark_names = matching.GetMostLikelyMatchedObject( |
| 200 all_benchmark_names, options.benchmark_name) | 233 all_benchmark_names, options.benchmark_name) |
| 201 parser.error( | 234 parser.error( |
| 202 'No benchmark named "%s". Do you mean any of those benchmarks ' | 235 'No benchmark named "%s". Do you mean any of those benchmarks ' |
| 203 'below?\n%s' % | 236 'below?\n%s' % ( |
| 204 (options.benchmark_name, '\n'.join(possible_benchmark_names))) | 237 options.benchmark_name, '\n'.join(possible_benchmark_names))) |
| 205 is_benchmark_disabled, reason = cls.IsBenchmarkDisabledOnTrybotPlatform( | 238 is_benchmark_disabled, reason = cls.IsBenchmarkDisabledOnTrybotPlatform( |
| 206 benchmark_class, options.trybot) | 239 benchmark_class, options.trybot) |
| 207 also_run_disabled_option = '--also-run-disabled-tests' | 240 also_run_disabled_option = '--also-run-disabled-tests' |
| 208 if is_benchmark_disabled and also_run_disabled_option not in extra_args: | 241 if is_benchmark_disabled and also_run_disabled_option not in extra_args: |
| 209 parser.error('%s To run the benchmark on trybot anyway, add ' | 242 parser.error('%s To run the benchmark on trybot anyway, add ' |
| 210 '%s option.' % (reason, also_run_disabled_option)) | 243 '%s option.' % (reason, also_run_disabled_option)) |
| 211 | 244 |
| 212 @classmethod | 245 @classmethod |
| 213 def IsBenchmarkDisabledOnTrybotPlatform(cls, benchmark_class, trybot_name): | 246 def IsBenchmarkDisabledOnTrybotPlatform(cls, benchmark_class, trybot_name): |
| 214 """ Return whether benchmark will be disabled on trybot platform. | 247 """Return whether benchmark will be disabled on trybot platform. |
| 215 | 248 |
| 216 Note that we cannot tell with certainty whether the benchmark will be | 249 Note that we cannot tell with certainty whether the benchmark will be |
| 217 disabled on the trybot platform since the disable logic in ShouldDisable() | 250 disabled on the trybot platform since the disable logic in ShouldDisable() |
| 218 can be very dynamic and can only be verified on the trybot server platform. | 251 can be very dynamic and can only be verified on the trybot server platform. |
| 219 | 252 |
| 220 We are biased on the side of enabling the benchmark, and attempt to | 253 We are biased on the side of enabling the benchmark, and attempt to |
| 221 early discover whether the benchmark will be disabled as our best. | 254 early discover whether the benchmark will be disabled as our best. |
| 222 | 255 |
| 223 It should never be the case that the benchmark will be enabled on the test | 256 It should never be the case that the benchmark will be enabled on the test |
| 224 platform but this method returns True. | 257 platform but this method returns True. |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 264 parser.add_argument( | 297 parser.add_argument( |
| 265 'trybot', choices=available_bots, | 298 'trybot', choices=available_bots, |
| 266 help=('specify which bots to run telemetry benchmarks on. ' | 299 help=('specify which bots to run telemetry benchmarks on. ' |
| 267 ' Allowed values are:\n' + '\n'.join(available_bots)), | 300 ' Allowed values are:\n' + '\n'.join(available_bots)), |
| 268 metavar='<trybot name>') | 301 metavar='<trybot name>') |
| 269 parser.add_argument( | 302 parser.add_argument( |
| 270 'benchmark_name', type=str, | 303 'benchmark_name', type=str, |
| 271 help=('specify which benchmark to run. To see all available benchmarks,' | 304 help=('specify which benchmark to run. To see all available benchmarks,' |
| 272 ' run `run_benchmark list`'), | 305 ' run `run_benchmark list`'), |
| 273 metavar='<benchmark name>') | 306 metavar='<benchmark name>') |
| 307 parser.add_argument( |
| 308 '--repo_path', type=str, default=CHROMIUM_SRC_PATH, |
| 309 help='specify which repo path to use for perf try job\n', |
| 310 metavar='<repo name>') |
| 311 parser.add_argument( |
| 312 '--deps_revision', type=str, default=None, |
| 313 help='specify DEPS revision to be used by Chromium.\n', |
| 314 metavar='<repo name>') |
| 315 |
| 274 | 316 |
| 275 def Run(self, options, extra_args=None): | 317 def Run(self, options, extra_args=None): |
| 276 """Sends a tryjob to a perf trybot. | 318 """Sends a tryjob to a perf trybot. |
| 277 | 319 |
| 278 This creates a branch, telemetry-tryjob, switches to that branch, edits | 320 This creates a branch, telemetry-tryjob, switches to that branch, edits |
| 279 the bisect config, commits it, uploads the CL to rietveld, and runs a | 321 the bisect config, commits it, uploads the CL to rietveld, and runs a |
| 280 tryjob on the given bot. | 322 tryjob on the given bot. |
| 281 """ | 323 """ |
| 282 if extra_args is None: | 324 if extra_args is None: |
| 283 extra_args = [] | 325 extra_args = [] |
| 284 self._InitializeBuilderNames(options.trybot) | 326 self._InitializeBuilderNames(options.trybot) |
| 285 | 327 |
| 286 arguments = [options.benchmark_name] + extra_args | 328 original_workdir = os.getcwd() |
| 287 | 329 repo_path = os.path.abspath(options.repo_path) |
| 288 # First check if there are chromium changes to upload. | 330 try: |
| 289 status = self._AttemptTryjob(CHROMIUM_CONFIG_FILENAME, arguments) | 331 # Check the existence of repo path. |
| 290 if status not in [SUCCESS, ERROR]: | 332 if not os.path.exists(repo_path): |
| 291 # If we got here, there are no chromium changes to upload. Try blink. | 333 raise TrybotError('Repository path "%s" does not exists, please check ' |
| 292 os.chdir('third_party/WebKit/') | 334 'the value of <repo_path> argument.' % repo_path) |
| 293 status = self._AttemptTryjob(BLINK_CONFIG_FILENAME, arguments) | 335 # Change to the repo directory. |
| 294 os.chdir('../..') | 336 os.chdir(repo_path) |
| 295 if status not in [SUCCESS, ERROR]: | 337 self._AttemptTryjob(repo_path, options, extra_args) |
| 296 logging.error('No local changes found in chromium or blink trees. ' | 338 except TrybotError, error: |
| 297 'browser=%s argument sends local changes to the ' | 339 print error |
| 298 'perf trybot(s): %s.', options.trybot, | 340 return 1 |
| 299 self._builder_names.values()) | 341 finally: |
| 300 return 1 | 342 # Restore to original working directory. |
| 343 os.chdir(original_workdir) |
| 301 return 0 | 344 return 0 |
| 302 | 345 |
| 303 def _UpdateConfigAndRunTryjob(self, bot_platform, cfg_file_path, arguments): | |
| 304 """Updates perf config file, uploads changes and excutes perf try job. | |
| 305 | |
| 306 Args: | |
| 307 bot_platform: Name of the platform to be generated. | |
| 308 cfg_file_path: Perf config file path. | |
| 309 | |
| 310 Returns: | |
| 311 (result, msg) where result is one of: | |
| 312 SUCCESS if a tryjob was sent | |
| 313 NO_CHANGES if there was nothing to try, | |
| 314 ERROR if a tryjob was attempted but an error encountered | |
| 315 and msg is an error message if an error was encountered, or rietveld | |
| 316 url if success, otherwise throws TrybotError exception. | |
| 317 """ | |
| 318 config = self._GetPerfConfig(bot_platform, arguments) | |
| 319 config_to_write = 'config = %s' % json.dumps( | |
| 320 config, sort_keys=True, indent=2, separators=(',', ': ')) | |
| 321 | |
| 322 try: | |
| 323 with open(cfg_file_path, 'r') as config_file: | |
| 324 if config_to_write == config_file.read(): | |
| 325 return NO_CHANGES, '' | |
| 326 except IOError: | |
| 327 msg = 'Cannot find %s. Please run from src dir.' % cfg_file_path | |
| 328 return (ERROR, msg) | |
| 329 | |
| 330 with open(cfg_file_path, 'w') as config_file: | |
| 331 config_file.write(config_to_write) | |
| 332 # Commit the config changes locally. | |
| 333 returncode, out, err = _RunProcess( | |
| 334 [_GIT_CMD, 'commit', '-a', '-m', 'bisect config: %s' % bot_platform]) | |
| 335 if returncode: | |
| 336 raise TrybotError('Could not commit bisect config change for %s,' | |
| 337 ' error %s' % (bot_platform, err)) | |
| 338 # Upload the CL to rietveld and run a try job. | |
| 339 returncode, out, err = _RunProcess([ | |
| 340 _GIT_CMD, 'cl', 'upload', '-f', '--bypass-hooks', '-m', | |
| 341 'CL for perf tryjob on %s' % bot_platform | |
| 342 ]) | |
| 343 if returncode: | |
| 344 raise TrybotError('Could not upload to rietveld for %s, error %s' % | |
| 345 (bot_platform, err)) | |
| 346 | |
| 347 match = re.search(r'https://codereview.chromium.org/[\d]+', out) | |
| 348 if not match: | |
| 349 raise TrybotError('Could not upload CL to rietveld for %s! Output %s' % | |
| 350 (bot_platform, out)) | |
| 351 rietveld_url = match.group(0) | |
| 352 # Generate git try command for available bots. | |
| 353 git_try_command = [_GIT_CMD, 'cl', 'try', '-m', 'tryserver.chromium.perf'] | |
| 354 for bot in self._builder_names[bot_platform]: | |
| 355 git_try_command.extend(['-b', bot]) | |
| 356 returncode, out, err = _RunProcess(git_try_command) | |
| 357 if returncode: | |
| 358 raise TrybotError('Could not try CL for %s, error %s' % | |
| 359 (bot_platform, err)) | |
| 360 | |
| 361 return (SUCCESS, rietveld_url) | |
| 362 | |
| 363 def _GetPerfConfig(self, bot_platform, arguments): | 346 def _GetPerfConfig(self, bot_platform, arguments): |
| 364 """Generates the perf config for try job. | 347 """Generates the perf config for try job. |
| 365 | 348 |
| 366 Args: | 349 Args: |
| 367 bot_platform: Name of the platform to be generated. | 350 bot_platform: Name of the platform to be generated. |
| 351 arguments: Command line arguments. |
| 368 | 352 |
| 369 Returns: | 353 Returns: |
| 370 A dictionary with perf config parameters. | 354 A dictionary with perf config parameters. |
| 371 """ | 355 """ |
| 372 # To make sure that we don't mutate the original args | 356 # To make sure that we don't mutate the original args |
| 373 arguments = arguments[:] | 357 arguments = arguments[:] |
| 374 | 358 |
| 375 # Always set verbose logging for later debugging | 359 # Always set verbose logging for later debugging |
| 376 if '-v' not in arguments and '--verbose' not in arguments: | 360 if '-v' not in arguments and '--verbose' not in arguments: |
| 377 arguments.append('--verbose') | 361 arguments.append('--verbose') |
| 378 | 362 |
| 379 # Generate the command line for the perf trybots | 363 # Generate the command line for the perf trybots |
| 380 target_arch = 'ia32' | 364 target_arch = 'ia32' |
| 381 if any(arg == '--chrome-root' or arg.startswith('--chrome-root=') for arg | 365 if any(arg == '--chrome-root' or arg.startswith('--chrome-root=') for arg |
| 382 in arguments): | 366 in arguments): |
| 383 raise ValueError( | 367 raise ValueError( |
| 384 'Trybot does not suport --chrome-root option set directly ' | 368 'Trybot does not suport --chrome-root option set directly ' |
| 385 'through command line since it may contain references to your local ' | 369 'through command line since it may contain references to your local ' |
| 386 'directory') | 370 'directory') |
| 387 if bot_platform in ['win', 'win-x64']: | 371 |
| 388 arguments.insert(0, 'python tools\\perf\\run_benchmark') | 372 arguments.insert(0, 'src/tools/perf/run_benchmark') |
| 389 else: | |
| 390 arguments.insert(0, './tools/perf/run_benchmark') | |
| 391 | 373 |
| 392 if bot_platform == 'android': | 374 if bot_platform == 'android': |
| 393 arguments.insert(1, '--browser=android-chromium') | 375 arguments.insert(1, '--browser=android-chromium') |
| 394 elif any('x64' in bot for bot in self._builder_names[bot_platform]): | 376 elif any('x64' in bot for bot in self._builder_names[bot_platform]): |
| 395 arguments.insert(1, '--browser=release_x64') | 377 arguments.insert(1, '--browser=release_x64') |
| 396 target_arch = 'x64' | 378 target_arch = 'x64' |
| 397 else: | 379 else: |
| 398 arguments.insert(1, '--browser=release') | 380 arguments.insert(1, '--browser=release') |
| 399 | 381 |
| 400 command = ' '.join(arguments) | 382 command = ' '.join(arguments) |
| 401 | 383 |
| 402 return { | 384 return { |
| 403 'command': command, | 385 'command': command, |
| 404 'repeat_count': '1', | 386 'repeat_count': '1', |
| 405 'max_time_minutes': '120', | 387 'max_time_minutes': '120', |
| 406 'truncate_percent': '0', | 388 'truncate_percent': '0', |
| 407 'target_arch': target_arch, | 389 'target_arch': target_arch, |
| 408 } | 390 } |
| 409 | 391 |
| 410 def _AttemptTryjob(self, cfg_file_path, arguments): | 392 def _GetRepoAndBranchName(self, repo_path): |
| 411 """Attempts to run a tryjob from the current directory. | 393 """Gets the repository name and working branch name. |
| 412 | |
| 413 This is run once for chromium, and if it returns NO_CHANGES, once for blink. | |
| 414 | 394 |
| 415 Args: | 395 Args: |
| 416 cfg_file_path: Path to the config file for the try job. | 396 repo_path: Path to the repository. |
| 417 | 397 |
| 418 Returns: | 398 Returns: |
| 419 Returns SUCCESS if a tryjob was sent, NO_CHANGES if there was nothing to | 399 Repository name and branch name as tuple. |
| 420 try, ERROR if a tryjob was attempted but an error encountered. | 400 |
| 401 Raises: |
| 402 TrybotError: This exception is raised for the following cases, |
| 403 1. Try job is for non-git repository or in invalid branch. |
| 404 2. Un-committed changes in the current branch. |
| 405 3. No local commits in the current branch. |
| 421 """ | 406 """ |
| 422 source_repo = 'chromium' | 407 # If command runs successfully, then the output will be repo root path. |
| 423 if cfg_file_path == BLINK_CONFIG_FILENAME: | 408 # and current branch name. |
| 424 source_repo = 'blink' | 409 output = RunGit(['rev-parse', '--abbrev-ref', '--show-toplevel', 'HEAD'], |
| 410 ('%s is not a git repository, must be in a git repository ' |
| 411 'to send changes to trybots' % os.getcwd())) |
| 425 | 412 |
| 426 # TODO(prasadv): This method is quite long, we should consider refactor | 413 repo_info = output.split() |
| 427 # this by extracting to helper methods. | 414 # Assuming the base directory name is same as repo project name set in |
| 428 returncode, original_branchname, err = _RunProcess( | 415 # codereviews.settings file. |
| 429 [_GIT_CMD, 'rev-parse', '--abbrev-ref', 'HEAD']) | 416 repo_name = os.path.basename(repo_info[0]).strip() |
| 430 if returncode: | 417 branch_name = repo_info[1].strip() |
| 431 msg = 'Must be in a git repository to send changes to trybots.' | |
| 432 if err: | |
| 433 msg += '\nGit error: %s' % err | |
| 434 logging.error(msg) | |
| 435 return ERROR | |
| 436 | 418 |
| 437 original_branchname = original_branchname.strip() | 419 if branch_name == 'HEAD': |
| 420 raise TrybotError('Not on a valid branch, looks like branch ' |
| 421 'is dettached. [branch:%s]' % branch_name) |
| 438 | 422 |
| 439 # Check if the tree is dirty: make sure the index is up to date and then | 423 # Check if the tree is dirty: make sure the index is up to date and then |
| 440 # run diff-index | 424 # run diff-index |
| 441 _RunProcess([_GIT_CMD, 'update-index', '--refresh', '-q']) | 425 RunGit(['update-index', '--refresh', '-q'], ignore_return_code=True) |
| 442 returncode, out, err = _RunProcess([_GIT_CMD, 'diff-index', 'HEAD']) | 426 output = RunGit(['diff-index', 'HEAD']) |
| 443 if out: | 427 if output: |
| 444 logging.error( | 428 raise TrybotError( |
| 445 'Cannot send a try job with a dirty tree. Commit locally first.') | 429 'Cannot send a try job with a dirty tree. Please commit ' |
| 446 return ERROR | 430 'your changes locally first in %s repository.' % repo_path) |
| 447 | 431 |
| 448 # Make sure the tree does have local commits. | 432 # Make sure the tree does have local commits. |
| 449 returncode, out, err = _RunProcess( | 433 output = RunGit(['footers', 'HEAD']) |
| 450 [_GIT_CMD, 'log', 'origin/master..HEAD']) | 434 if output: |
| 451 if not out: | 435 raise TrybotError('No local changes found in %s repository.' % repo_path) |
| 452 return NO_CHANGES | |
| 453 | 436 |
| 454 # Create/check out the telemetry-tryjob branch, and edit the configs | 437 return (repo_name, branch_name) |
| 455 # for the tryjob there. | 438 |
| 456 returncode, out, err = _RunProcess( | 439 def _GetBaseGitHashForRepo(self, branch_name, git_url): |
| 457 [_GIT_CMD, 'checkout', '-b', 'telemetry-tryjob']) | 440 """Gets the base revision for the repo on which local changes are made.""" |
| 458 if returncode: | 441 while not self._IsRepoSupported(branch_name, git_url): |
| 459 logging.error('Error creating branch telemetry-tryjob. ' | 442 branch_name = RunGit([ |
| 460 'Please delete it if it exists.\n%s', err) | 443 'rev-parse', '--abbrev-ref', '%s@{upstream}' % branch_name]) |
| 461 return ERROR | 444 return RunGit(['rev-parse', '%s@{upstream}' % branch_name]) |
| 462 try: | 445 |
| 463 returncode, out, err = _RunProcess( | 446 def _IsRepoSupported(self, current_branch, repo_git_url): |
| 464 [_GIT_CMD, 'branch', '--set-upstream-to', 'origin/master']) | 447 cur_remote = RunGit(['config', 'branch.%s.remote'% current_branch]) |
| 465 if returncode: | 448 if cur_remote == '.': |
| 466 logging.error('Error in git branch --set-upstream-to: %s', err) | 449 return False |
| 467 return ERROR | 450 cur_remote_url= RunGit(['config', 'remote.%s.url' % cur_remote]) |
| 468 for bot_platform in self._builder_names: | 451 if cur_remote_url.lower() == repo_git_url: |
| 469 if not self._builder_names[bot_platform]: | 452 return True |
| 470 logging.warning('No builder is found for %s', bot_platform) | 453 raise TrybotError('Remote url on remote %s is not recognized ' |
| 471 continue | 454 'on branch: %s' % (cur_remote, cur_remote_url)) |
| 472 try: | 455 |
| 473 results, output = self._UpdateConfigAndRunTryjob( | 456 def _AttemptTryjob(self, repo_path, options, extra_args): |
| 474 bot_platform, cfg_file_path, arguments) | 457 """Attempts to run a tryjob from a repo directory. |
| 475 if results == ERROR: | 458 |
| 476 logging.error(output) | 459 Args: |
| 477 return ERROR | 460 repo_path: Path to the repository. |
| 478 elif results == NO_CHANGES: | 461 options: Command line arguments to run benchmark. |
| 479 print ('Skip the try job run on %s because it has been tried in ' | 462 extra_args: Extra arugments to run benchmark. |
| 480 'previous try job run. ' % bot_platform) | 463 """ |
| 481 else: | 464 repo_name, branch_name = self._GetRepoAndBranchName(repo_path) |
| 482 print ('Uploaded %s try job to rietveld for %s platform. ' | 465 |
| 483 'View progress at %s' % (source_repo, bot_platform, output)) | 466 repo_info = REPO_INFO_MAP.get(repo_name, None) |
| 484 except TrybotError, err: | 467 if not repo_info: |
| 485 print err | 468 raise TrybotError('Unsupported repository %s' % repo_name) |
| 486 logging.error(err) | 469 |
| 487 finally: | 470 rietveld_url = self._UploadPatchToRietveld(repo_name) |
| 488 # Checkout original branch and delete telemetry-tryjob branch. | 471 print ('\nUploaded patch to rietveld.' |
| 489 # TODO(prasadv): This finally block could be extracted out to be a | 472 '\n\tRepo Name: %s\n\tPath: %s\n\tBranch: %s' % ( |
| 490 # separate function called _CleanupBranch. | 473 repo_name, repo_path, branch_name)) |
| 491 returncode, out, err = _RunProcess( | 474 |
| 492 [_GIT_CMD, 'checkout', original_branchname]) | 475 deps_override = None |
| 493 if returncode: | 476 if repo_name != 'src': |
| 494 logging.error('Could not check out %s. Please check it out and ' | 477 if not options.deps_revision: |
| 495 'manually delete the telemetry-tryjob branch. ' | 478 options.deps_revision = self._GetBaseGitHashForRepo( |
| 496 ': %s', original_branchname, err) | 479 branch_name, repo_info.get('url')) |
| 497 return ERROR # pylint: disable=lost-exception | 480 deps_override = {repo_info.get('src'): options.deps_revision} |
| 498 logging.info('Checked out original branch: %s', original_branchname) | 481 |
| 499 returncode, out, err = _RunProcess( | 482 arguments = [options.benchmark_name] + extra_args |
| 500 [_GIT_CMD, 'branch', '-D', 'telemetry-tryjob']) | 483 |
| 501 if returncode: | 484 rietveld_url = self._UploadPatchToRietveld(repo_name) |
| 502 logging.error('Could not delete telemetry-tryjob branch. ' | 485 print ('\nUploaded try job to rietveld.\n' |
| 503 'Please delete it manually: %s', err) | 486 'view progress here %s.\n\tRepo Name: %s\n\tPath: %s\n\tBranch: %s' %
( |
| 504 return ERROR # pylint: disable=lost-exception | 487 rietveld_url, repo_name, repo_path, branch_name)) |
| 505 logging.info('Deleted temp branch: telemetry-tryjob') | 488 |
| 506 return SUCCESS | 489 for bot_platform in self._builder_names: |
| 490 if not self._builder_names[bot_platform]: |
| 491 logging.warning('No builder is found for %s', bot_platform) |
| 492 continue |
| 493 try: |
| 494 self._RunTryJob(bot_platform, arguments, deps_override) |
| 495 print ('Perf Try job sent to rietveld for %s platform.\n' % |
| 496 bot_platform) |
| 497 except TrybotError, err: |
| 498 print err |
| 499 |
| 500 def _UploadPatchToRietveld(self, repo_name): |
| 501 # Upload the CL to rietveld and run a try job. |
| 502 output = RunGit(['cl', 'upload', '-f', '--bypass-hooks', '-m', |
| 503 'CL for %s perf tryjob' % repo_name], |
| 504 'Could not upload to rietveld for %s' % repo_name) |
| 505 |
| 506 match = re.search(r'https://codereview.chromium.org/[\d]+', output) |
| 507 if not match: |
| 508 raise TrybotError('Could not upload CL to rietveld for %s! Output %s' % |
| 509 (repo_name, output)) |
| 510 return match.group(0) |
| 511 |
| 512 |
| 513 def _RunTryJob(self, bot_platform, arguments, deps_override): |
| 514 """Generates perf try config, uploads changes and excutes perf try job. |
| 515 |
| 516 Args: |
| 517 bot_platform: Name of the platform to be generated. |
| 518 arguments: Command line arguments. |
| 519 deps_override: DEPS revision if needs to be overridden. |
| 520 |
| 521 Raises: |
| 522 TrybotError: When trybot fails to upload CL or run git try. |
| 523 """ |
| 524 config = self._GetPerfConfig(bot_platform, arguments) |
| 525 |
| 526 # Generate git try command for available bots. |
| 527 git_try_command = ['cl', 'try', '-m', 'tryserver.chromium.perf'] |
| 528 |
| 529 # Add Perf Test config to git try --properties arg. |
| 530 git_try_command.extend(['-p', 'perf_try_config=%s' % json.dumps(config)]) |
| 531 |
| 532 # Add deps overrides to git try --properties arg. |
| 533 if deps_override: |
| 534 git_try_command.extend([ |
| 535 '-p', 'deps_revision_overrides=%s' % json.dumps(deps_override)]) |
| 536 |
| 537 for bot in self._builder_names[bot_platform]: |
| 538 git_try_command.extend(['-b', bot]) |
| 539 |
| 540 RunGit(git_try_command, 'Could not try CL for %s' % bot_platform) |
| OLD | NEW |