Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(300)

Side by Side Diff: appengine/swarming/handlers_bot.py

Issue 2969513002: Add a default Isolate gRPC proxy in config (Closed)
Patch Set: Fix problem in tests Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | appengine/swarming/proto/config.proto » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2015 The LUCI Authors. All rights reserved. 1 # Copyright 2015 The LUCI Authors. All rights reserved.
2 # Use of this source code is governed under the Apache License, Version 2.0 2 # Use of this source code is governed under the Apache License, Version 2.0
3 # that can be found in the LICENSE file. 3 # that can be found in the LICENSE file.
4 4
5 """Internal bot API handlers.""" 5 """Internal bot API handlers."""
6 6
7 import base64 7 import base64
8 import json 8 import json
9 import logging 9 import logging
10 10
(...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after
85 85
86 Do not warn about unexpected keys. 86 Do not warn about unexpected keys.
87 """ 87 """
88 actual_keys = frozenset(actual_keys) 88 actual_keys = frozenset(actual_keys)
89 missing = minimum_keys - actual_keys 89 missing = minimum_keys - actual_keys
90 if missing: 90 if missing:
91 msg_missing = (' missing: %s' % sorted(missing)) if missing else '' 91 msg_missing = (' missing: %s' % sorted(missing)) if missing else ''
92 return 'Unexpected %s%s; did you make a typo?' % (name, msg_missing) 92 return 'Unexpected %s%s; did you make a typo?' % (name, msg_missing)
93 93
94 94
95 def get_bot_contact_server(request):
96 """Gets the server contacted by the bot.
97
98 Usually, this is the URL of the Swarming server itself, but if the bot
99 is communicating to the server by a gRPC intermediary, this will be the
100 IP address of the gRPC endpoint. The Native API will have an http or
101 https protocol, while gRPC endpoints will have a fake "grpc://" protocol.
102 This is to help consumers of this information (mainly the bot code
103 generators) distinguish between native and gRPC bots.
104 """
105 server = request.host_url
106 if 'luci-grpc' in request.headers:
107 server = 'grpc://%s' % request.headers['luci-grpc']
108 return server
109
110
111 class _BotApiHandler(auth.ApiHandler): 95 class _BotApiHandler(auth.ApiHandler):
112 """Like ApiHandler, but also implements machine authentication.""" 96 """Like ApiHandler, but also implements machine authentication."""
113 97
114 # Bots are passing credentials through special headers (not cookies), no need 98 # Bots are passing credentials through special headers (not cookies), no need
115 # for XSRF tokens. 99 # for XSRF tokens.
116 xsrf_token_enforce_on = () 100 xsrf_token_enforce_on = ()
117 101
118 @classmethod 102 @classmethod
119 def get_auth_methods(cls, conf): 103 def get_auth_methods(cls, conf):
120 return [auth.machine_authentication, auth.oauth_authentication] 104 return [auth.machine_authentication, auth.oauth_authentication]
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after
167 151
168 # TODO(vadimsh): Remove is_ip_whitelisted_machine check once all bots are 152 # TODO(vadimsh): Remove is_ip_whitelisted_machine check once all bots are
169 # using auth for bootstrap and updating. 153 # using auth for bootstrap and updating.
170 if (not acl.is_bootstrapper() and 154 if (not acl.is_bootstrapper() and
171 not acl.is_ip_whitelisted_machine() and 155 not acl.is_ip_whitelisted_machine() and
172 not (bot_id and bot_auth.is_authenticated_bot(bot_id, machine_type))): 156 not (bot_id and bot_auth.is_authenticated_bot(bot_id, machine_type))):
173 raise auth.AuthorizationError('Not allowed to access the bot code') 157 raise auth.AuthorizationError('Not allowed to access the bot code')
174 158
175 return bot_code.generate_bootstrap_token() if generate_token else None 159 return bot_code.generate_bootstrap_token() if generate_token else None
176 160
177 def get_bot_contact_server(self):
178 """Gets the server contacted by the bot."""
179 return get_bot_contact_server(self.request)
180
181 161
182 class BootstrapHandler(_BotAuthenticatingHandler): 162 class BootstrapHandler(_BotAuthenticatingHandler):
183 """Returns python code to run to bootstrap a swarming bot.""" 163 """Returns python code to run to bootstrap a swarming bot."""
184 164
185 @auth.public # auth inside check_bot_code_access() 165 @auth.public # auth inside check_bot_code_access()
186 def get(self): 166 def get(self):
187 # We must pass a bootstrap token (generating it, if necessary) to 167 # We must pass a bootstrap token (generating it, if necessary) to
188 # get_bootstrap(...), since bootstrap.py uses tokens exclusively (it can't 168 # get_bootstrap(...), since bootstrap.py uses tokens exclusively (it can't
189 # transparently pass OAuth headers to /bot_code). 169 # transparently pass OAuth headers to /bot_code).
190 bootstrap_token = self.check_bot_code_access( 170 bootstrap_token = self.check_bot_code_access(
191 bot_id=None, generate_token=True) 171 bot_id=None, generate_token=True)
192 self.response.headers['Content-Type'] = 'text/x-python' 172 self.response.headers['Content-Type'] = 'text/x-python'
193 self.response.headers['Content-Disposition'] = ( 173 self.response.headers['Content-Disposition'] = (
194 'attachment; filename="swarming_bot_bootstrap.py"') 174 'attachment; filename="swarming_bot_bootstrap.py"')
195 self.response.out.write( 175 self.response.out.write(
196 bot_code.get_bootstrap(self.request.host_url, bootstrap_token).content) 176 bot_code.get_bootstrap(self.request.host_url, bootstrap_token).content)
197 177
198 178
199 class BotCodeHandler(_BotAuthenticatingHandler): 179 class BotCodeHandler(_BotAuthenticatingHandler):
200 """Returns a zip file with all the files required by a bot. 180 """Returns a zip file with all the files required by a bot.
201 181
202 Optionally specify the hash version to download. If so, the returned data is 182 Optionally specify the hash version to download. If so, the returned data is
203 cacheable. 183 cacheable.
204 """ 184 """
205 185
206 @auth.public # auth inside check_bot_code_access() 186 @auth.public # auth inside check_bot_code_access()
207 def get(self, version=None): 187 def get(self, version=None):
208 server = self.get_bot_contact_server() 188 server = self.request.host_url
209 self.check_bot_code_access( 189 self.check_bot_code_access(
210 bot_id=self.request.get('bot_id'), generate_token=False) 190 bot_id=self.request.get('bot_id'), generate_token=False)
211 if version: 191 if version:
212 expected, _ = bot_code.get_bot_version(server) 192 expected, _ = bot_code.get_bot_version(server)
213 if version != expected: 193 if version != expected:
214 # This can happen when the server is rapidly updated. 194 # This can happen when the server is rapidly updated.
215 logging.error('Requested Swarming bot %s, have %s', version, expected) 195 logging.error('Requested Swarming bot %s, have %s', version, expected)
216 self.abort(404) 196 self.abort(404)
217 self.response.headers['Cache-Control'] = 'public, max-age=3600' 197 self.response.headers['Cache-Control'] = 'public, max-age=3600'
218 else: 198 else:
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after
376 return result 356 return result
377 357
378 # Look for admin enforced quarantine. 358 # Look for admin enforced quarantine.
379 if bool(bot_settings and bot_settings.quarantined): 359 if bool(bot_settings and bot_settings.quarantined):
380 result.quarantined_msg = 'Quarantined by admin' 360 result.quarantined_msg = 'Quarantined by admin'
381 return result 361 return result
382 362
383 task_queues.assert_bot(dimensions) 363 task_queues.assert_bot(dimensions)
384 return result 364 return result
385 365
386 def get_bot_contact_server(self):
387 """Gets the server contacted by the bot."""
388 return get_bot_contact_server(self.request)
389
390 366
391 class BotHandshakeHandler(_BotBaseHandler): 367 class BotHandshakeHandler(_BotBaseHandler):
392 """First request to be called to get initial data like bot code version. 368 """First request to be called to get initial data like bot code version.
393 369
394 The bot is server-controlled so the server doesn't have to support multiple 370 The bot is server-controlled so the server doesn't have to support multiple
395 API version. When running a task, the bot sync the the version specific URL. 371 API version. When running a task, the bot sync the the version specific URL.
396 Once a bot finishes its currently running task, it'll be immediately upgraded 372 Once a bot finishes its currently running task, it'll be immediately upgraded
397 on its next poll. 373 on its next poll.
398 374
399 This endpoint does not return commands to the bot, for example to upgrade 375 This endpoint does not return commands to the bot, for example to upgrade
(...skipping 15 matching lines...) Expand all
415 res = self._process() 391 res = self._process()
416 bot_management.bot_event( 392 bot_management.bot_event(
417 event_type='bot_connected', bot_id=res.bot_id, 393 event_type='bot_connected', bot_id=res.bot_id,
418 external_ip=self.request.remote_addr, 394 external_ip=self.request.remote_addr,
419 authenticated_as=auth.get_peer_identity().to_bytes(), 395 authenticated_as=auth.get_peer_identity().to_bytes(),
420 dimensions=res.dimensions, state=res.state, 396 dimensions=res.dimensions, state=res.state,
421 version=res.version, quarantined=bool(res.quarantined_msg), 397 version=res.version, quarantined=bool(res.quarantined_msg),
422 task_id='', task_name=None, message=res.quarantined_msg) 398 task_id='', task_name=None, message=res.quarantined_msg)
423 399
424 data = { 400 data = {
425 'bot_version': bot_code.get_bot_version(self.get_bot_contact_server())[0], 401 'bot_version': bot_code.get_bot_version(self.request.host_url)[0],
426 'server_version': utils.get_app_version(), 402 'server_version': utils.get_app_version(),
427 'bot_group_cfg_version': res.bot_group_cfg.version, 403 'bot_group_cfg_version': res.bot_group_cfg.version,
428 'bot_group_cfg': { 404 'bot_group_cfg': {
429 # Let the bot know its server-side dimensions (from bots.cfg file). 405 # Let the bot know its server-side dimensions (from bots.cfg file).
430 'dimensions': res.bot_group_cfg.dimensions, 406 'dimensions': res.bot_group_cfg.dimensions,
431 }, 407 },
432 } 408 }
433 if res.bot_group_cfg.bot_config_script_content: 409 if res.bot_group_cfg.bot_config_script_content:
434 logging.info( 410 logging.info(
435 'Injecting %s: %d bytes', 411 'Injecting %s: %d bytes',
(...skipping 177 matching lines...) Expand 10 before | Expand all | Expand 10 after
613 external_ip=self.request.remote_addr, 589 external_ip=self.request.remote_addr,
614 authenticated_as=auth.get_peer_identity().to_bytes(), 590 authenticated_as=auth.get_peer_identity().to_bytes(),
615 dimensions=res.dimensions, state=res.state, 591 dimensions=res.dimensions, state=res.state,
616 version=res.version, quarantined=quarantined, 592 version=res.version, quarantined=quarantined,
617 task_id=task_id, task_name=task_name, message=res.quarantined_msg) 593 task_id=task_id, task_name=task_name, message=res.quarantined_msg)
618 594
619 # Bot version is host-specific because the host URL is embedded in 595 # Bot version is host-specific because the host URL is embedded in
620 # swarming_bot.zip 596 # swarming_bot.zip
621 logging.debug('Fetching bot code version') 597 logging.debug('Fetching bot code version')
622 expected_version, _ = bot_code.get_bot_version( 598 expected_version, _ = bot_code.get_bot_version(
623 self.get_bot_contact_server()) 599 self.request.host_url)
624 if res.version != expected_version: 600 if res.version != expected_version:
625 bot_event('request_update') 601 bot_event('request_update')
626 self._cmd_update(expected_version) 602 self._cmd_update(expected_version)
627 return 603 return
628 if quarantined: 604 if quarantined:
629 bot_event('request_sleep') 605 bot_event('request_sleep')
630 self._cmd_sleep(sleep_streak, quarantined) 606 self._cmd_sleep(sleep_streak, quarantined)
631 return 607 return
632 608
633 # If the server-side per-bot config for the bot has changed, we need 609 # If the server-side per-bot config for the bot has changed, we need
(...skipping 435 matching lines...) Expand 10 before | Expand all | Expand 10 after
1069 ('/swarming/api/v1/bot/poll', BotPollHandler), 1045 ('/swarming/api/v1/bot/poll', BotPollHandler),
1070 ('/swarming/api/v1/bot/server_ping', ServerPingHandler), 1046 ('/swarming/api/v1/bot/server_ping', ServerPingHandler),
1071 ('/swarming/api/v1/bot/task_update', BotTaskUpdateHandler), 1047 ('/swarming/api/v1/bot/task_update', BotTaskUpdateHandler),
1072 ('/swarming/api/v1/bot/task_update/<task_id:[a-f0-9]+>', 1048 ('/swarming/api/v1/bot/task_update/<task_id:[a-f0-9]+>',
1073 BotTaskUpdateHandler), 1049 BotTaskUpdateHandler),
1074 ('/swarming/api/v1/bot/task_error', BotTaskErrorHandler), 1050 ('/swarming/api/v1/bot/task_error', BotTaskErrorHandler),
1075 ('/swarming/api/v1/bot/task_error/<task_id:[a-f0-9]+>', 1051 ('/swarming/api/v1/bot/task_error/<task_id:[a-f0-9]+>',
1076 BotTaskErrorHandler), 1052 BotTaskErrorHandler),
1077 ] 1053 ]
1078 return [webapp2.Route(*i) for i in routes] 1054 return [webapp2.Route(*i) for i in routes]
OLDNEW
« no previous file with comments | « no previous file | appengine/swarming/proto/config.proto » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698