| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
| 4 # Use of this source code is governed by a BSD-style license that can be | |
| 5 # found in the LICENSE file. | |
| 6 | |
| 7 import collections | |
| 8 import copy | |
| 9 import json | |
| 10 import os | |
| 11 import pipes | |
| 12 import re | |
| 13 import subprocess | |
| 14 import sys | |
| 15 | |
| 16 import bb_utils | |
| 17 | |
| 18 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) | |
| 19 from pylib import constants | |
| 20 | |
| 21 | |
| 22 CHROMIUM_COVERAGE_BUCKET = 'chromium-code-coverage' | |
| 23 | |
| 24 _BotConfig = collections.namedtuple( | |
| 25 'BotConfig', ['bot_id', 'host_obj', 'test_obj']) | |
| 26 | |
| 27 HostConfig = collections.namedtuple( | |
| 28 'HostConfig', | |
| 29 ['script', 'host_steps', 'extra_args', 'extra_gyp_defines', 'target_arch']) | |
| 30 | |
| 31 TestConfig = collections.namedtuple('Tests', ['script', 'tests', 'extra_args']) | |
| 32 | |
| 33 | |
| 34 def BotConfig(bot_id, host_object, test_object=None): | |
| 35 return _BotConfig(bot_id, host_object, test_object) | |
| 36 | |
| 37 | |
| 38 def DictDiff(d1, d2): | |
| 39 diff = [] | |
| 40 for key in sorted(set(d1.keys() + d2.keys())): | |
| 41 if key in d1 and d1[key] != d2.get(key): | |
| 42 diff.append('- %s=%s' % (key, pipes.quote(d1[key]))) | |
| 43 if key in d2 and d2[key] != d1.get(key): | |
| 44 diff.append('+ %s=%s' % (key, pipes.quote(d2[key]))) | |
| 45 return '\n'.join(diff) | |
| 46 | |
| 47 | |
| 48 def GetEnvironment(host_obj, testing, extra_env_vars=None): | |
| 49 init_env = dict(os.environ) | |
| 50 init_env['GYP_GENERATORS'] = 'ninja' | |
| 51 if extra_env_vars: | |
| 52 init_env.update(extra_env_vars) | |
| 53 envsetup_cmd = '. build/android/envsetup.sh' | |
| 54 if testing: | |
| 55 # Skip envsetup to avoid presubmit dependence on android deps. | |
| 56 print 'Testing mode - skipping "%s"' % envsetup_cmd | |
| 57 envsetup_cmd = ':' | |
| 58 else: | |
| 59 print 'Running %s' % envsetup_cmd | |
| 60 proc = subprocess.Popen(['bash', '-exc', | |
| 61 envsetup_cmd + ' >&2; python build/android/buildbot/env_to_json.py'], | |
| 62 stdout=subprocess.PIPE, stderr=subprocess.PIPE, | |
| 63 cwd=bb_utils.CHROME_SRC, env=init_env) | |
| 64 json_env, envsetup_output = proc.communicate() | |
| 65 if proc.returncode != 0: | |
| 66 print >> sys.stderr, 'FATAL Failure in envsetup.' | |
| 67 print >> sys.stderr, envsetup_output | |
| 68 sys.exit(1) | |
| 69 env = json.loads(json_env) | |
| 70 env['GYP_DEFINES'] = env.get('GYP_DEFINES', '') + \ | |
| 71 ' OS=android fastbuild=1 use_goma=1 gomadir=%s' % bb_utils.GOMA_DIR | |
| 72 if host_obj.target_arch: | |
| 73 env['GYP_DEFINES'] += ' target_arch=%s' % host_obj.target_arch | |
| 74 extra_gyp = host_obj.extra_gyp_defines | |
| 75 if extra_gyp: | |
| 76 env['GYP_DEFINES'] += ' %s' % extra_gyp | |
| 77 if re.search('(asan|clang)=1', extra_gyp): | |
| 78 env.pop('CXX_target', None) | |
| 79 | |
| 80 # Bots checkout chrome in /b/build/slave/<name>/build/src | |
| 81 build_internal_android = os.path.abspath(os.path.join( | |
| 82 bb_utils.CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal', | |
| 83 'scripts', 'slave', 'android')) | |
| 84 if os.path.exists(build_internal_android): | |
| 85 env['PATH'] = os.pathsep.join([build_internal_android, env['PATH']]) | |
| 86 return env | |
| 87 | |
| 88 | |
| 89 def GetCommands(options, bot_config): | |
| 90 """Get a formatted list of commands. | |
| 91 | |
| 92 Args: | |
| 93 options: Options object. | |
| 94 bot_config: A BotConfig named tuple. | |
| 95 host_step_script: Host step script. | |
| 96 device_step_script: Device step script. | |
| 97 Returns: | |
| 98 list of Command objects. | |
| 99 """ | |
| 100 property_args = bb_utils.EncodeProperties(options) | |
| 101 commands = [[bot_config.host_obj.script, | |
| 102 '--steps=%s' % ','.join(bot_config.host_obj.host_steps)] + | |
| 103 property_args + (bot_config.host_obj.extra_args or [])] | |
| 104 | |
| 105 test_obj = bot_config.test_obj | |
| 106 if test_obj: | |
| 107 run_test_cmd = [test_obj.script] + property_args | |
| 108 for test in test_obj.tests: | |
| 109 run_test_cmd.extend(['-f', test]) | |
| 110 if test_obj.extra_args: | |
| 111 run_test_cmd.extend(test_obj.extra_args) | |
| 112 commands.append(run_test_cmd) | |
| 113 return commands | |
| 114 | |
| 115 | |
| 116 def GetBotStepMap(): | |
| 117 compile_step = ['compile'] | |
| 118 chrome_proxy_tests = ['chrome_proxy'] | |
| 119 python_unittests = ['python_unittests'] | |
| 120 std_host_tests = ['check_webview_licenses'] | |
| 121 std_build_steps = ['compile', 'zip_build'] | |
| 122 std_test_steps = ['extract_build'] | |
| 123 std_tests = ['ui', 'unit'] | |
| 124 telemetry_tests = ['telemetry_perf_unittests'] | |
| 125 telemetry_tests_user_build = ['telemetry_unittests', | |
| 126 'telemetry_perf_unittests'] | |
| 127 trial_tests = [ | |
| 128 'base_junit_tests', | |
| 129 'components_browsertests', | |
| 130 'gfx_unittests', | |
| 131 'gl_unittests', | |
| 132 ] | |
| 133 flakiness_server = ( | |
| 134 '--flakiness-server=%s' % constants.UPSTREAM_FLAKINESS_SERVER) | |
| 135 experimental = ['--experimental'] | |
| 136 bisect_chrome_output_dir = os.path.abspath( | |
| 137 os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, | |
| 138 os.pardir, 'bisect', 'src', 'out')) | |
| 139 B = BotConfig | |
| 140 H = (lambda steps, extra_args=None, extra_gyp=None, target_arch=None: | |
| 141 HostConfig('build/android/buildbot/bb_host_steps.py', steps, extra_args, | |
| 142 extra_gyp, target_arch)) | |
| 143 T = (lambda tests, extra_args=None: | |
| 144 TestConfig('build/android/buildbot/bb_device_steps.py', tests, | |
| 145 extra_args)) | |
| 146 | |
| 147 bot_configs = [ | |
| 148 # Main builders | |
| 149 B('main-builder-dbg', H(std_build_steps + std_host_tests)), | |
| 150 B('main-builder-rel', H(std_build_steps)), | |
| 151 B('main-clang-builder', | |
| 152 H(compile_step, extra_gyp='clang=1 component=shared_library')), | |
| 153 B('main-clobber', H(compile_step)), | |
| 154 B('main-tests-rel', H(std_test_steps), | |
| 155 T(std_tests + telemetry_tests + chrome_proxy_tests, | |
| 156 ['--cleanup', flakiness_server])), | |
| 157 B('main-tests', H(std_test_steps), | |
| 158 T(std_tests, ['--cleanup', flakiness_server])), | |
| 159 | |
| 160 # Other waterfalls | |
| 161 B('asan-builder-tests', H(compile_step, | |
| 162 extra_gyp='asan=1 component=shared_library'), | |
| 163 T(std_tests, ['--asan', '--asan-symbolize'])), | |
| 164 B('blink-try-builder', H(compile_step)), | |
| 165 B('chromedriver-fyi-tests-dbg', H(std_test_steps), | |
| 166 T(['chromedriver'], | |
| 167 ['--install=ChromeShell', '--install=ChromeDriverWebViewShell', | |
| 168 '--skip-wipe', '--disable-location', '--cleanup'])), | |
| 169 B('fyi-x86-builder-dbg', | |
| 170 H(compile_step + std_host_tests, experimental, target_arch='ia32')), | |
| 171 B('fyi-builder-dbg', | |
| 172 H(std_build_steps + std_host_tests, experimental, | |
| 173 extra_gyp='emma_coverage=1')), | |
| 174 B('x86-builder-dbg', | |
| 175 H(compile_step + std_host_tests, target_arch='ia32')), | |
| 176 B('fyi-builder-rel', H(std_build_steps, experimental)), | |
| 177 B('fyi-tests', H(std_test_steps), | |
| 178 T(std_tests + python_unittests, | |
| 179 ['--experimental', flakiness_server, | |
| 180 '--coverage-bucket', CHROMIUM_COVERAGE_BUCKET, | |
| 181 '--cleanup'])), | |
| 182 B('user-build-fyi-tests-dbg', H(std_test_steps), | |
| 183 T(sorted(telemetry_tests_user_build + trial_tests))), | |
| 184 B('fyi-component-builder-tests-dbg', | |
| 185 H(compile_step, extra_gyp='component=shared_library'), | |
| 186 T(std_tests, ['--experimental', flakiness_server])), | |
| 187 B('gpu-builder-tests-dbg', | |
| 188 H(compile_step), | |
| 189 T(['gpu'], ['--install=ContentShell'])), | |
| 190 # Pass empty T([]) so that logcat monitor and device status check are run. | |
| 191 B('perf-bisect-builder-tests-dbg', | |
| 192 H(['bisect_perf_regression']), | |
| 193 T([], ['--chrome-output-dir', bisect_chrome_output_dir])), | |
| 194 B('perf-tests-rel', H(std_test_steps), | |
| 195 T([], ['--install=ChromeShell', '--cleanup'])), | |
| 196 B('webkit-latest-webkit-tests', H(std_test_steps), | |
| 197 T(['webkit_layout', 'webkit'], ['--cleanup', '--auto-reconnect'])), | |
| 198 B('webkit-latest-contentshell', H(compile_step), | |
| 199 T(['webkit_layout'], ['--auto-reconnect'])), | |
| 200 B('builder-unit-tests', H(compile_step), T(['unit'])), | |
| 201 | |
| 202 # Generic builder config (for substring match). | |
| 203 B('builder', H(std_build_steps)), | |
| 204 ] | |
| 205 | |
| 206 bot_map = dict((config.bot_id, config) for config in bot_configs) | |
| 207 | |
| 208 # These bots have identical configuration to ones defined earlier. | |
| 209 copy_map = [ | |
| 210 ('lkgr-clobber', 'main-clobber'), | |
| 211 ('try-builder-dbg', 'main-builder-dbg'), | |
| 212 ('try-builder-rel', 'main-builder-rel'), | |
| 213 ('try-clang-builder', 'main-clang-builder'), | |
| 214 ('try-fyi-builder-dbg', 'fyi-builder-dbg'), | |
| 215 ('try-x86-builder-dbg', 'x86-builder-dbg'), | |
| 216 ('try-tests-rel', 'main-tests-rel'), | |
| 217 ('try-tests', 'main-tests'), | |
| 218 ('try-fyi-tests', 'fyi-tests'), | |
| 219 ('webkit-latest-tests', 'main-tests'), | |
| 220 ] | |
| 221 for to_id, from_id in copy_map: | |
| 222 assert to_id not in bot_map | |
| 223 # pylint: disable=W0212 | |
| 224 bot_map[to_id] = copy.deepcopy(bot_map[from_id])._replace(bot_id=to_id) | |
| 225 | |
| 226 # Trybots do not upload to flakiness dashboard. They should be otherwise | |
| 227 # identical in configuration to their trunk building counterparts. | |
| 228 test_obj = bot_map[to_id].test_obj | |
| 229 if to_id.startswith('try') and test_obj: | |
| 230 extra_args = test_obj.extra_args | |
| 231 if extra_args and flakiness_server in extra_args: | |
| 232 extra_args.remove(flakiness_server) | |
| 233 return bot_map | |
| 234 | |
| 235 | |
| 236 # Return an object from the map, looking first for an exact id match. | |
| 237 # If this fails, look for an id which is a substring of the specified id. | |
| 238 # Choose the longest of all substring matches. | |
| 239 # pylint: disable=W0622 | |
| 240 def GetBestMatch(id_map, id): | |
| 241 config = id_map.get(id) | |
| 242 if not config: | |
| 243 substring_matches = [x for x in id_map.iterkeys() if x in id] | |
| 244 if substring_matches: | |
| 245 max_id = max(substring_matches, key=len) | |
| 246 print 'Using config from id="%s" (substring match).' % max_id | |
| 247 config = id_map[max_id] | |
| 248 return config | |
| 249 | |
| 250 | |
| 251 def GetRunBotOptParser(): | |
| 252 parser = bb_utils.GetParser() | |
| 253 parser.add_option('--bot-id', help='Specify bot id directly.') | |
| 254 parser.add_option('--testing', action='store_true', | |
| 255 help='For testing: print, but do not run commands') | |
| 256 | |
| 257 return parser | |
| 258 | |
| 259 | |
| 260 def GetBotConfig(options, bot_step_map): | |
| 261 bot_id = options.bot_id or options.factory_properties.get('android_bot_id') | |
| 262 if not bot_id: | |
| 263 print (sys.stderr, | |
| 264 'A bot id must be specified through option or factory_props.') | |
| 265 return | |
| 266 | |
| 267 bot_config = GetBestMatch(bot_step_map, bot_id) | |
| 268 if not bot_config: | |
| 269 print 'Error: config for id="%s" cannot be inferred.' % bot_id | |
| 270 return bot_config | |
| 271 | |
| 272 | |
| 273 def RunBotCommands(options, commands, env): | |
| 274 print 'Environment changes:' | |
| 275 print DictDiff(dict(os.environ), env) | |
| 276 | |
| 277 for command in commands: | |
| 278 print bb_utils.CommandToString(command) | |
| 279 sys.stdout.flush() | |
| 280 if options.testing: | |
| 281 env['BUILDBOT_TESTING'] = '1' | |
| 282 return_code = subprocess.call(command, cwd=bb_utils.CHROME_SRC, env=env) | |
| 283 if return_code != 0: | |
| 284 return return_code | |
| 285 | |
| 286 | |
| 287 def main(argv): | |
| 288 proc = subprocess.Popen( | |
| 289 ['/bin/hostname', '-f'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| 290 hostname_stdout, hostname_stderr = proc.communicate() | |
| 291 if proc.returncode == 0: | |
| 292 print 'Running on: ' + hostname_stdout | |
| 293 else: | |
| 294 print >> sys.stderr, 'WARNING: failed to run hostname' | |
| 295 print >> sys.stderr, hostname_stdout | |
| 296 print >> sys.stderr, hostname_stderr | |
| 297 sys.exit(1) | |
| 298 | |
| 299 parser = GetRunBotOptParser() | |
| 300 options, args = parser.parse_args(argv[1:]) | |
| 301 if args: | |
| 302 parser.error('Unused args: %s' % args) | |
| 303 | |
| 304 bot_config = GetBotConfig(options, GetBotStepMap()) | |
| 305 if not bot_config: | |
| 306 sys.exit(1) | |
| 307 | |
| 308 print 'Using config:', bot_config | |
| 309 | |
| 310 commands = GetCommands(options, bot_config) | |
| 311 for command in commands: | |
| 312 print 'Will run: ', bb_utils.CommandToString(command) | |
| 313 print | |
| 314 | |
| 315 env = GetEnvironment(bot_config.host_obj, options.testing) | |
| 316 return RunBotCommands(options, commands, env) | |
| 317 | |
| 318 | |
| 319 if __name__ == '__main__': | |
| 320 sys.exit(main(sys.argv)) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2392643003:
Removes files from //build that we don't need (Closed)
