Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(199)

Side by Side Diff: build/android/buildbot/bb_run_bot.py

Issue 2101243005: Add a snapshot of flutter/engine/src/build to our sdk (Closed) Base URL: git@github.com:dart-lang/sdk.git@master
Patch Set: add README.dart Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « build/android/buildbot/bb_host_steps.py ('k') | build/android/buildbot/bb_utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 #
3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
6
7 import collections
8 import copy
9 import json
10 import os
11 import pipes
12 import re
13 import subprocess
14 import sys
15
16 import bb_utils
17
18 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
19 from pylib import constants
20
21
22 CHROMIUM_COVERAGE_BUCKET = 'chromium-code-coverage'
23
24 _BotConfig = collections.namedtuple(
25 'BotConfig', ['bot_id', 'host_obj', 'test_obj'])
26
27 HostConfig = collections.namedtuple(
28 'HostConfig',
29 ['script', 'host_steps', 'extra_args', 'extra_gyp_defines', 'target_arch'])
30
31 TestConfig = collections.namedtuple('Tests', ['script', 'tests', 'extra_args'])
32
33
34 def BotConfig(bot_id, host_object, test_object=None):
35 return _BotConfig(bot_id, host_object, test_object)
36
37
38 def DictDiff(d1, d2):
39 diff = []
40 for key in sorted(set(d1.keys() + d2.keys())):
41 if key in d1 and d1[key] != d2.get(key):
42 diff.append('- %s=%s' % (key, pipes.quote(d1[key])))
43 if key in d2 and d2[key] != d1.get(key):
44 diff.append('+ %s=%s' % (key, pipes.quote(d2[key])))
45 return '\n'.join(diff)
46
47
48 def GetEnvironment(host_obj, testing, extra_env_vars=None):
49 init_env = dict(os.environ)
50 init_env['GYP_GENERATORS'] = 'ninja'
51 if extra_env_vars:
52 init_env.update(extra_env_vars)
53 envsetup_cmd = '. build/android/envsetup.sh'
54 if testing:
55 # Skip envsetup to avoid presubmit dependence on android deps.
56 print 'Testing mode - skipping "%s"' % envsetup_cmd
57 envsetup_cmd = ':'
58 else:
59 print 'Running %s' % envsetup_cmd
60 proc = subprocess.Popen(['bash', '-exc',
61 envsetup_cmd + ' >&2; python build/android/buildbot/env_to_json.py'],
62 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
63 cwd=bb_utils.CHROME_SRC, env=init_env)
64 json_env, envsetup_output = proc.communicate()
65 if proc.returncode != 0:
66 print >> sys.stderr, 'FATAL Failure in envsetup.'
67 print >> sys.stderr, envsetup_output
68 sys.exit(1)
69 env = json.loads(json_env)
70 env['GYP_DEFINES'] = env.get('GYP_DEFINES', '') + \
71 ' OS=android fastbuild=1 use_goma=1 gomadir=%s' % bb_utils.GOMA_DIR
72 if host_obj.target_arch:
73 env['GYP_DEFINES'] += ' target_arch=%s' % host_obj.target_arch
74 extra_gyp = host_obj.extra_gyp_defines
75 if extra_gyp:
76 env['GYP_DEFINES'] += ' %s' % extra_gyp
77 if re.search('(asan|clang)=1', extra_gyp):
78 env.pop('CXX_target', None)
79
80 # Bots checkout chrome in /b/build/slave/<name>/build/src
81 build_internal_android = os.path.abspath(os.path.join(
82 bb_utils.CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal',
83 'scripts', 'slave', 'android'))
84 if os.path.exists(build_internal_android):
85 env['PATH'] = os.pathsep.join([build_internal_android, env['PATH']])
86 return env
87
88
89 def GetCommands(options, bot_config):
90 """Get a formatted list of commands.
91
92 Args:
93 options: Options object.
94 bot_config: A BotConfig named tuple.
95 host_step_script: Host step script.
96 device_step_script: Device step script.
97 Returns:
98 list of Command objects.
99 """
100 property_args = bb_utils.EncodeProperties(options)
101 commands = [[bot_config.host_obj.script,
102 '--steps=%s' % ','.join(bot_config.host_obj.host_steps)] +
103 property_args + (bot_config.host_obj.extra_args or [])]
104
105 test_obj = bot_config.test_obj
106 if test_obj:
107 run_test_cmd = [test_obj.script] + property_args
108 for test in test_obj.tests:
109 run_test_cmd.extend(['-f', test])
110 if test_obj.extra_args:
111 run_test_cmd.extend(test_obj.extra_args)
112 commands.append(run_test_cmd)
113 return commands
114
115
116 def GetBotStepMap():
117 compile_step = ['compile']
118 chrome_proxy_tests = ['chrome_proxy']
119 python_unittests = ['python_unittests']
120 std_host_tests = ['check_webview_licenses']
121 std_build_steps = ['compile', 'zip_build']
122 std_test_steps = ['extract_build']
123 std_tests = ['ui', 'unit']
124 telemetry_tests = ['telemetry_perf_unittests']
125 telemetry_tests_user_build = ['telemetry_unittests',
126 'telemetry_perf_unittests']
127 trial_tests = [
128 'base_junit_tests',
129 'components_browsertests',
130 'gfx_unittests',
131 'gl_unittests',
132 ]
133 flakiness_server = (
134 '--flakiness-server=%s' % constants.UPSTREAM_FLAKINESS_SERVER)
135 experimental = ['--experimental']
136 bisect_chrome_output_dir = os.path.abspath(
137 os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
138 os.pardir, 'bisect', 'src', 'out'))
139 B = BotConfig
140 H = (lambda steps, extra_args=None, extra_gyp=None, target_arch=None:
141 HostConfig('build/android/buildbot/bb_host_steps.py', steps, extra_args,
142 extra_gyp, target_arch))
143 T = (lambda tests, extra_args=None:
144 TestConfig('build/android/buildbot/bb_device_steps.py', tests,
145 extra_args))
146
147 bot_configs = [
148 # Main builders
149 B('main-builder-dbg', H(std_build_steps + std_host_tests)),
150 B('main-builder-rel', H(std_build_steps)),
151 B('main-clang-builder',
152 H(compile_step, extra_gyp='clang=1 component=shared_library')),
153 B('main-clobber', H(compile_step)),
154 B('main-tests-rel', H(std_test_steps),
155 T(std_tests + telemetry_tests + chrome_proxy_tests,
156 ['--cleanup', flakiness_server])),
157 B('main-tests', H(std_test_steps),
158 T(std_tests, ['--cleanup', flakiness_server])),
159
160 # Other waterfalls
161 B('asan-builder-tests', H(compile_step,
162 extra_gyp='asan=1 component=shared_library'),
163 T(std_tests, ['--asan', '--asan-symbolize'])),
164 B('blink-try-builder', H(compile_step)),
165 B('chromedriver-fyi-tests-dbg', H(std_test_steps),
166 T(['chromedriver'],
167 ['--install=ChromeShell', '--install=ChromeDriverWebViewShell',
168 '--skip-wipe', '--disable-location', '--cleanup'])),
169 B('fyi-x86-builder-dbg',
170 H(compile_step + std_host_tests, experimental, target_arch='ia32')),
171 B('fyi-builder-dbg',
172 H(std_build_steps + std_host_tests, experimental,
173 extra_gyp='emma_coverage=1')),
174 B('x86-builder-dbg',
175 H(compile_step + std_host_tests, target_arch='ia32')),
176 B('fyi-builder-rel', H(std_build_steps, experimental)),
177 B('fyi-tests', H(std_test_steps),
178 T(std_tests + python_unittests,
179 ['--experimental', flakiness_server,
180 '--coverage-bucket', CHROMIUM_COVERAGE_BUCKET,
181 '--cleanup'])),
182 B('user-build-fyi-tests-dbg', H(std_test_steps),
183 T(sorted(telemetry_tests_user_build + trial_tests))),
184 B('fyi-component-builder-tests-dbg',
185 H(compile_step, extra_gyp='component=shared_library'),
186 T(std_tests, ['--experimental', flakiness_server])),
187 B('gpu-builder-tests-dbg',
188 H(compile_step),
189 T(['gpu'], ['--install=ContentShell'])),
190 # Pass empty T([]) so that logcat monitor and device status check are run.
191 B('perf-bisect-builder-tests-dbg',
192 H(['bisect_perf_regression']),
193 T([], ['--chrome-output-dir', bisect_chrome_output_dir])),
194 B('perf-tests-rel', H(std_test_steps),
195 T([], ['--install=ChromeShell', '--cleanup'])),
196 B('webkit-latest-webkit-tests', H(std_test_steps),
197 T(['webkit_layout', 'webkit'], ['--cleanup', '--auto-reconnect'])),
198 B('webkit-latest-contentshell', H(compile_step),
199 T(['webkit_layout'], ['--auto-reconnect'])),
200 B('builder-unit-tests', H(compile_step), T(['unit'])),
201
202 # Generic builder config (for substring match).
203 B('builder', H(std_build_steps)),
204 ]
205
206 bot_map = dict((config.bot_id, config) for config in bot_configs)
207
208 # These bots have identical configuration to ones defined earlier.
209 copy_map = [
210 ('lkgr-clobber', 'main-clobber'),
211 ('try-builder-dbg', 'main-builder-dbg'),
212 ('try-builder-rel', 'main-builder-rel'),
213 ('try-clang-builder', 'main-clang-builder'),
214 ('try-fyi-builder-dbg', 'fyi-builder-dbg'),
215 ('try-x86-builder-dbg', 'x86-builder-dbg'),
216 ('try-tests-rel', 'main-tests-rel'),
217 ('try-tests', 'main-tests'),
218 ('try-fyi-tests', 'fyi-tests'),
219 ('webkit-latest-tests', 'main-tests'),
220 ]
221 for to_id, from_id in copy_map:
222 assert to_id not in bot_map
223 # pylint: disable=W0212
224 bot_map[to_id] = copy.deepcopy(bot_map[from_id])._replace(bot_id=to_id)
225
226 # Trybots do not upload to flakiness dashboard. They should be otherwise
227 # identical in configuration to their trunk building counterparts.
228 test_obj = bot_map[to_id].test_obj
229 if to_id.startswith('try') and test_obj:
230 extra_args = test_obj.extra_args
231 if extra_args and flakiness_server in extra_args:
232 extra_args.remove(flakiness_server)
233 return bot_map
234
235
236 # Return an object from the map, looking first for an exact id match.
237 # If this fails, look for an id which is a substring of the specified id.
238 # Choose the longest of all substring matches.
239 # pylint: disable=W0622
240 def GetBestMatch(id_map, id):
241 config = id_map.get(id)
242 if not config:
243 substring_matches = [x for x in id_map.iterkeys() if x in id]
244 if substring_matches:
245 max_id = max(substring_matches, key=len)
246 print 'Using config from id="%s" (substring match).' % max_id
247 config = id_map[max_id]
248 return config
249
250
251 def GetRunBotOptParser():
252 parser = bb_utils.GetParser()
253 parser.add_option('--bot-id', help='Specify bot id directly.')
254 parser.add_option('--testing', action='store_true',
255 help='For testing: print, but do not run commands')
256
257 return parser
258
259
260 def GetBotConfig(options, bot_step_map):
261 bot_id = options.bot_id or options.factory_properties.get('android_bot_id')
262 if not bot_id:
263 print (sys.stderr,
264 'A bot id must be specified through option or factory_props.')
265 return
266
267 bot_config = GetBestMatch(bot_step_map, bot_id)
268 if not bot_config:
269 print 'Error: config for id="%s" cannot be inferred.' % bot_id
270 return bot_config
271
272
273 def RunBotCommands(options, commands, env):
274 print 'Environment changes:'
275 print DictDiff(dict(os.environ), env)
276
277 for command in commands:
278 print bb_utils.CommandToString(command)
279 sys.stdout.flush()
280 if options.testing:
281 env['BUILDBOT_TESTING'] = '1'
282 return_code = subprocess.call(command, cwd=bb_utils.CHROME_SRC, env=env)
283 if return_code != 0:
284 return return_code
285
286
287 def main(argv):
288 proc = subprocess.Popen(
289 ['/bin/hostname', '-f'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
290 hostname_stdout, hostname_stderr = proc.communicate()
291 if proc.returncode == 0:
292 print 'Running on: ' + hostname_stdout
293 else:
294 print >> sys.stderr, 'WARNING: failed to run hostname'
295 print >> sys.stderr, hostname_stdout
296 print >> sys.stderr, hostname_stderr
297 sys.exit(1)
298
299 parser = GetRunBotOptParser()
300 options, args = parser.parse_args(argv[1:])
301 if args:
302 parser.error('Unused args: %s' % args)
303
304 bot_config = GetBotConfig(options, GetBotStepMap())
305 if not bot_config:
306 sys.exit(1)
307
308 print 'Using config:', bot_config
309
310 commands = GetCommands(options, bot_config)
311 for command in commands:
312 print 'Will run: ', bb_utils.CommandToString(command)
313 print
314
315 env = GetEnvironment(bot_config.host_obj, options.testing)
316 return RunBotCommands(options, commands, env)
317
318
319 if __name__ == '__main__':
320 sys.exit(main(sys.argv))
OLDNEW
« no previous file with comments | « build/android/buildbot/bb_host_steps.py ('k') | build/android/buildbot/bb_utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698