Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(13)

Side by Side Diff: appengine/swarming/handlers_api.py

Issue 1458553003: Delete old APIs on both Swarming and Isolate servers. (Closed) Base URL: git@github.com:luci/luci-py.git@1_warning
Patch Set: Rebasing on HEAD Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | appengine/swarming/handlers_api_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # Copyright 2014 The Swarming Authors. All rights reserved.
2 # Use of this source code is governed by the Apache v2.0 license that can be
3 # found in the LICENSE file.
4
5 """Swarming client REST APIs handlers."""
6
7 import base64
8 import datetime
9 import json
10 import logging
11 import textwrap
12
13 import webapp2
14
15 from google.appengine.api import app_identity
16 from google.appengine.api import datastore_errors
17 from google.appengine.datastore import datastore_query
18 from google.appengine import runtime
19 from google.appengine.ext import ndb
20
21 from components import auth
22 from components import ereporter2
23 from components import utils
24 from server import acl
25 from server import config
26 from server import bot_code
27 from server import bot_management
28 from server import stats
29 from server import task_pack
30 from server import task_request
31 from server import task_result
32 from server import task_scheduler
33 from server import task_to_run
34
35
36 def has_unexpected_subset_keys(expected_keys, minimum_keys, actual_keys, name):
37 """Returns an error if unexpected keys are present or expected keys are
38 missing.
39
40 Accepts optional keys.
41
42 This is important to catch typos.
43 """
44 actual_keys = frozenset(actual_keys)
45 superfluous = actual_keys - expected_keys
46 missing = minimum_keys - actual_keys
47 if superfluous or missing:
48 msg_missing = (' missing: %s' % sorted(missing)) if missing else ''
49 msg_superfluous = (
50 (' superfluous: %s' % sorted(superfluous)) if superfluous else '')
51 return 'Unexpected %s%s%s; did you make a typo?' % (
52 name, msg_missing, msg_superfluous)
53
54
55 def log_unexpected_subset_keys(
56 expected_keys, minimum_keys, actual_keys, request, source, name):
57 """Logs an error if unexpected keys are present or expected keys are missing.
58
59 Accepts optional keys.
60
61 This is important to catch typos.
62 """
63 message = has_unexpected_subset_keys(
64 expected_keys, minimum_keys, actual_keys, name)
65 if message:
66 ereporter2.log_request(request, source=source, message=message)
67 return message
68
69
70 def log_unexpected_keys(expected_keys, actual_keys, request, source, name):
71 """Logs an error if unexpected keys are present or expected keys are missing.
72 """
73 return log_unexpected_subset_keys(
74 expected_keys, expected_keys, actual_keys, request, source, name)
75
76
77 def process_doc(handler):
78 lines = handler.__doc__.rstrip().splitlines()
79 rest = textwrap.dedent('\n'.join(lines[1:]))
80 return '\n'.join((lines[0], rest)).rstrip()
81
82
83 ### New Client APIs.
84
85
86 class ClientApiListHandler(auth.ApiHandler):
87 """All query handlers"""
88
89 @auth.public
90 def get(self):
91 logging.error('Unexpected old client')
92 # Hard to make it any simpler.
93 prefix = '/swarming/api/v1/client/'
94 data = {
95 r.template[len(prefix):]: process_doc(r.handler) for r in get_routes()
96 if r.template.startswith(prefix) and hasattr(r.handler, 'get')
97 }
98 self.send_response(data)
99
100
101 class ClientHandshakeHandler(auth.ApiHandler):
102 """First request to be called to get initial data like XSRF token.
103
104 Request body is a JSON dict:
105 {
106 # TODO(maruel): Add useful data.
107 }
108
109 Response body is a JSON dict:
110 {
111 "server_version": "138-193f1f3",
112 "xsrf_token": "......",
113 }
114 """
115
116 # This handler is called to get XSRF token, there's nothing to enforce yet.
117 xsrf_token_enforce_on = ()
118
119 EXPECTED_KEYS = frozenset()
120
121 @auth.require_xsrf_token_request
122 @auth.require(acl.is_bot_or_user)
123 def post(self):
124 logging.error('Unexpected old client')
125 request = self.parse_body()
126 log_unexpected_keys(
127 self.EXPECTED_KEYS, request, self.request, 'client', 'keys')
128 data = {
129 # This access token will be used to validate each subsequent request.
130 'server_version': utils.get_app_version(),
131 'xsrf_token': self.generate_xsrf_token(),
132 }
133 self.send_response(data)
134
135
136 class ClientTaskResultBase(auth.ApiHandler):
137 """Implements the common base code for task related query APIs."""
138
139 def get_result_key(self, task_id):
140 logging.error('Unexpected old client')
141 # TODO(maruel): Users can only request their own task. Privileged users can
142 # request any task.
143 key = None
144 summary_key = None
145 try:
146 key = task_pack.unpack_result_summary_key(task_id)
147 summary_key = key
148 except ValueError:
149 try:
150 key = task_pack.unpack_run_result_key(task_id)
151 summary_key = task_pack.run_result_key_to_result_summary_key(key)
152 except ValueError:
153 self.abort_with_error(400, error='Invalid key')
154 return key, summary_key
155
156 def get_result_entity(self, task_id):
157 key, _ = self.get_result_key(task_id)
158 result = key.get()
159 if not result:
160 self.abort_with_error(404, error='Task not found')
161 return result
162
163
164 class ClientTaskResultHandler(ClientTaskResultBase):
165 """Task's result meta data"""
166
167 @auth.require(acl.is_bot_or_user)
168 def get(self, task_id):
169 logging.error('Unexpected old client')
170 result = self.get_result_entity(task_id)
171 self.send_response(utils.to_json_encodable(result))
172
173
174 class ClientTaskResultRequestHandler(ClientTaskResultBase):
175 """Task's request details"""
176
177 @auth.require(acl.is_bot_or_user)
178 def get(self, task_id):
179 logging.error('Unexpected old client')
180 _, summary_key = self.get_result_key(task_id)
181 request_key = task_pack.result_summary_key_to_request_key(summary_key)
182 self.send_response(utils.to_json_encodable(request_key.get()))
183
184
185 class ClientTaskResultOutputHandler(ClientTaskResultBase):
186 """Task's output for a single command"""
187
188 @auth.require(acl.is_bot_or_user)
189 def get(self, task_id, command_index):
190 logging.error('Unexpected old client')
191 result = self.get_result_entity(task_id)
192 output = result.get_command_output_async(int(command_index)).get_result()
193 if output:
194 output = output.decode('utf-8', 'replace')
195 # JSON then reencodes to ascii compatible encoded strings, which explodes
196 # the size.
197 data = {
198 'output': output,
199 }
200 self.send_response(utils.to_json_encodable(data))
201
202
203 class ClientTaskResultOutputAllHandler(ClientTaskResultBase):
204 """All output from all commands in a task"""
205
206 @auth.require(acl.is_bot_or_user)
207 def get(self, task_id):
208 logging.error('Unexpected old client')
209 result = self.get_result_entity(task_id)
210 # JSON then reencodes to ascii compatible encoded strings, which explodes
211 # the size.
212 data = {
213 'outputs': [
214 i.decode('utf-8', 'replace') if i else i
215 for i in result.get_outputs()
216 ],
217 }
218 self.send_response(utils.to_json_encodable(data))
219
220
221 class ClientApiTasksHandler(auth.ApiHandler):
222 """Requests all TaskResultSummary with filters.
223
224 It is specifically a GET with query parameters for simplicity instead of a
225 JSON POST.
226
227 Arguments:
228 name: Search by task name; str or None.
229 tag: Search by task tag, can be used mulitple times; list(str) or None.
230 cursor: Continue a previous query; str or None.
231 limit: Maximum number of items to return.
232 sort: Ordering: 'created_ts', 'modified_ts', 'completed_ts', 'abandoned_ts'.
233 Defaults to 'created_ts'.
234 state: Filtering: 'all', 'pending', 'running', 'pending_running',
235 'completed', 'completed_success', 'completed_failure', 'bot_died',
236 'expired', 'canceled'. Defaults to 'all'.
237
238 In particular, one of `name`, `tag` or `state` can be used
239 exclusively.
240 """
241 EXPECTED = {'cursor', 'limit', 'name', 'sort', 'state', 'tag'}
242
243 @auth.require(acl.is_privileged_user)
244 def get(self):
245 logging.error('Unexpected old client')
246 extra = frozenset(self.request.GET) - self.EXPECTED
247 if extra:
248 self.abort_with_error(
249 400,
250 error='Extraneous query parameters. Did you make a typo? %s' %
251 ','.join(sorted(extra)))
252
253 # Use a similar query to /user/tasks.
254 name = self.request.get('name')
255 tags = self.request.get_all('tag')
256 cursor_str = self.request.get('cursor')
257 limit = int(self.request.get('limit', 100))
258 sort = self.request.get('sort', 'created_ts')
259 state = self.request.get('state', 'all')
260
261 uses = bool(name) + bool(tags) + bool(state!='all')
262 if uses > 1:
263 self.abort_with_error(
264 400, error='Only one of name, tag (1 or many) or state can be used')
265
266 items, cursor_str, sort, state = task_result.get_tasks(
267 limit, cursor_str, sort, state, tags, name)
268 data = {
269 'cursor': cursor_str,
270 'items': items,
271 'limit': limit,
272 'sort': sort,
273 'state': state,
274 }
275 self.send_response(utils.to_json_encodable(data))
276
277
278 class ClientApiTasksCountHandler(auth.ApiHandler):
279 """Counts number of tasks in a given state.
280
281 Can be used to estimate pending queue size.
282
283 Args:
284 interval: How far back into the past to search for tasks (seconds).
285 state: Filtering: 'all', 'pending', 'running', 'pending_running',
286 'completed', 'completed_success', 'completed_failure', 'bot_died',
287 'expired', 'canceled'. Defaults to 'all'.
288 """
289 EXPECTED = {'interval', 'state', 'tag'}
290
291 VALID_STATES = {
292 'all',
293 'bot_died',
294 'canceled',
295 'completed',
296 'completed_failure',
297 'completed_success',
298 'expired',
299 'pending',
300 'pending_running',
301 'running',
302 'timed_out',
303 }
304
305 @auth.require(acl.is_privileged_user)
306 def get(self):
307 logging.error('Unexpected old client')
308 extra = frozenset(self.request.GET) - self.EXPECTED
309 if extra:
310 self.abort_with_error(
311 400,
312 error='Extraneous query parameters. Did you make a typo? %s' %
313 ','.join(sorted(extra)))
314
315 interval = self.request.get('interval', 24 * 3600)
316 state = self.request.get('state', 'all')
317 tags = self.request.get_all('tag')
318
319 try:
320 interval = int(interval)
321 if interval <= 0:
322 raise ValueError()
323 except ValueError:
324 self.abort_with_error(
325 400, error='"interval" must be a positive integer number of seconds')
326
327 if state not in self.VALID_STATES:
328 self.abort_with_error(
329 400,
330 error='Invalid state "%s", expecting on of %s' %
331 (state, ', '.join(sorted(self.VALID_STATES))))
332
333 cutoff = utils.utcnow() - datetime.timedelta(seconds=interval)
334 query = task_result.get_result_summaries_query(
335 cutoff, None, 'created_ts', state, tags)
336 self.send_response(utils.to_json_encodable({'count': query.count()}))
337
338
339 class ClientApiBots(auth.ApiHandler):
340 """Bots known to the server"""
341
342 ACCEPTABLE_FILTERS = (
343 'quarantined',
344 'is_dead',
345 )
346
347 @auth.require(acl.is_privileged_user)
348 def get(self):
349 logging.error('Unexpected old client')
350 now = utils.utcnow()
351 limit = int(self.request.get('limit', 1000))
352 filter_by = self.request.get('filter')
353 if filter_by and filter_by not in self.ACCEPTABLE_FILTERS:
354 self.abort_with_error(400, error='Invalid filter query parameter')
355
356 q = bot_management.BotInfo.query()
357
358 if not filter_by:
359 q = q.order(bot_management.BotInfo.key)
360 recheck = lambda _: True
361 elif filter_by == 'quarantined':
362 q = q.order(bot_management.BotInfo.key)
363 q = q.filter(bot_management.BotInfo.quarantined == True)
364 recheck = lambda b: b.quarantined
365 elif filter_by == 'is_dead':
366 # The first sort key must be the same as used in the filter, otherwise
367 # datastore raises BadRequestError.
368 deadline = now - datetime.timedelta(
369 seconds=config.settings().bot_death_timeout_secs)
370 q = q.order(bot_management.BotInfo.last_seen_ts)
371 q = q.filter(bot_management.BotInfo.last_seen_ts < deadline)
372 recheck = lambda b: b.last_seen_ts < deadline
373 else:
374 raise AssertionError('Impossible')
375
376 cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
377 bots, cursor, more = q.fetch_page(limit, start_cursor=cursor)
378 data = {
379 'cursor': cursor.urlsafe() if cursor and more else None,
380 'death_timeout': config.settings().bot_death_timeout_secs,
381 'items': [b.to_dict_with_now(now) for b in bots if recheck(b)],
382 'limit': limit,
383 'now': now,
384 }
385 self.send_response(utils.to_json_encodable(data))
386
387
388 class ClientApiBot(auth.ApiHandler):
389 """Bot's meta data"""
390
391 @auth.require(acl.is_privileged_user)
392 def get(self, bot_id):
393 logging.error('Unexpected old client')
394 bot = bot_management.get_info_key(bot_id).get()
395 if not bot:
396 self.abort_with_error(404, error='Bot not found')
397 now = utils.utcnow()
398 self.send_response(utils.to_json_encodable(bot.to_dict_with_now(now)))
399
400 @auth.require(acl.is_admin)
401 def delete(self, bot_id):
402 # Only delete BotInfo, not BotRoot, BotEvent nor BotSettings.
403 bot_key = bot_management.get_info_key(bot_id)
404 found = False
405 if bot_key.get():
406 bot_key.delete()
407 found = True
408 self.send_response({'deleted': bool(found)})
409
410
411 class ClientApiBotTask(auth.ApiHandler):
412 """Tasks executed on a specific bot"""
413
414 @auth.require(acl.is_privileged_user)
415 def get(self, bot_id):
416 logging.error('Unexpected old client')
417 limit = int(self.request.get('limit', 100))
418 cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
419 run_results, cursor, more = task_result.TaskRunResult.query(
420 task_result.TaskRunResult.bot_id == bot_id).order(
421 -task_result.TaskRunResult.started_ts).fetch_page(
422 limit, start_cursor=cursor)
423 now = utils.utcnow()
424 data = {
425 'cursor': cursor.urlsafe() if cursor and more else None,
426 'items': run_results,
427 'limit': limit,
428 'now': now,
429 }
430 self.send_response(utils.to_json_encodable(data))
431
432
433 class ClientApiServer(auth.ApiHandler):
434 """Server details"""
435
436 @auth.require(acl.is_privileged_user)
437 def get(self):
438 logging.error('Unexpected old client')
439 data = {
440 'bot_version': bot_code.get_bot_version(self.request.host_url),
441 }
442 self.send_response(utils.to_json_encodable(data))
443
444
445 class ClientRequestHandler(auth.ApiHandler):
446 """Creates a new request, returns the task id.
447
448 Argument:
449 - data: dict with:
450 - name
451 - parent_task_id*
452 - properties
453 - commands
454 - data
455 - dimensions
456 - env
457 - execution_timeout_secs
458 - grace_period_secs*
459 - idempotent*
460 - io_timeout_secs
461 - priority
462 - scheduling_expiration_secs
463 - tags
464 - user
465
466 * are optional.
467 """
468 # Parameters for make_request().
469 # The content of the 'data' parameter. This relates to the context of the
470 # request, e.g. who wants to run a task.
471 _REQUIRED_DATA_KEYS = frozenset(
472 ['name', 'priority', 'properties', 'scheduling_expiration_secs', 'tags',
473 'user'])
474 _EXPECTED_DATA_KEYS = frozenset(
475 ['name', 'parent_task_id', 'priority', 'properties',
476 'scheduling_expiration_secs', 'tags', 'user'])
477 # The content of 'properties' inside the 'data' parameter. This relates to the
478 # task itself, e.g. what to run.
479 _REQUIRED_PROPERTIES_KEYS= frozenset(
480 ['commands', 'data', 'dimensions', 'env', 'execution_timeout_secs',
481 'io_timeout_secs'])
482 _EXPECTED_PROPERTIES_KEYS = frozenset(
483 ['commands', 'data', 'dimensions', 'env', 'execution_timeout_secs',
484 'grace_period_secs', 'idempotent', 'io_timeout_secs'])
485
486 @auth.require(acl.is_bot_or_user)
487 def post(self):
488 logging.error('Unexpected old client')
489 data = self.parse_body()
490 msg = log_unexpected_subset_keys(
491 self._EXPECTED_DATA_KEYS, self._REQUIRED_DATA_KEYS, data, self.request,
492 'client', 'request keys')
493 if msg:
494 self.abort_with_error(400, error=msg)
495 data_properties = data['properties']
496 msg = log_unexpected_subset_keys(
497 self._EXPECTED_PROPERTIES_KEYS, self._REQUIRED_PROPERTIES_KEYS,
498 data_properties, self.request, 'client', 'request properties keys')
499 if msg:
500 self.abort_with_error(400, error=msg)
501
502 # Class TaskProperties takes care of making everything deterministic.
503 properties = task_request.TaskProperties(
504 commands=data_properties['commands'],
505 data=data_properties['data'],
506 dimensions=data_properties['dimensions'],
507 env=data_properties['env'],
508 execution_timeout_secs=data_properties['execution_timeout_secs'],
509 grace_period_secs=data_properties.get('grace_period_secs', 30),
510 idempotent=data_properties.get('idempotent', False),
511 io_timeout_secs=data_properties['io_timeout_secs'])
512
513 now = utils.utcnow()
514 expiration_ts = now + datetime.timedelta(
515 seconds=data['scheduling_expiration_secs'])
516 request = task_request.TaskRequest(
517 created_ts=now,
518 expiration_ts=expiration_ts,
519 name=data['name'],
520 parent_task_id=data.get('parent_task_id'),
521 priority=data['priority'],
522 properties=properties,
523 tags=data['tags'],
524 user=data['user'] or '')
525
526 try:
527 request = task_request.make_request(request, acl.is_bot_or_admin())
528 except (
529 AttributeError, datastore_errors.BadValueError, TypeError,
530 ValueError) as e:
531 self.abort_with_error(400, error=str(e))
532
533 result_summary = task_scheduler.schedule_request(request)
534 data = {
535 'request': request.to_dict(),
536 'task_id': task_pack.pack_result_summary_key(result_summary.key),
537 }
538 self.send_response(utils.to_json_encodable(data))
539
540
541 class ClientCancelHandler(auth.ApiHandler):
542 """Cancels a task."""
543
544 # TODO(maruel): Allow privileged users to cancel, and users to cancel their
545 # own task.
546 @auth.require(acl.is_admin)
547 def post(self):
548 logging.error('Unexpected old client')
549 request = self.parse_body()
550 task_id = request.get('task_id')
551 summary_key = task_pack.unpack_result_summary_key(task_id)
552
553 ok, was_running = task_scheduler.cancel_task(summary_key)
554 out = {
555 'ok': ok,
556 'was_running': was_running,
557 }
558 self.send_response(out)
559
560
561 def get_routes():
562 routes = [
563 ('/swarming/api/v1/client/bots', ClientApiBots),
564 ('/swarming/api/v1/client/bot/<bot_id:[^/]+>', ClientApiBot),
565 ('/swarming/api/v1/client/bot/<bot_id:[^/]+>/tasks', ClientApiBotTask),
566 ('/swarming/api/v1/client/cancel', ClientCancelHandler),
567 ('/swarming/api/v1/client/handshake', ClientHandshakeHandler),
568 ('/swarming/api/v1/client/list', ClientApiListHandler),
569 ('/swarming/api/v1/client/request', ClientRequestHandler),
570 ('/swarming/api/v1/client/server', ClientApiServer),
571 ('/swarming/api/v1/client/task/<task_id:[0-9a-f]+>',
572 ClientTaskResultHandler),
573 ('/swarming/api/v1/client/task/<task_id:[0-9a-f]+>/request',
574 ClientTaskResultRequestHandler),
575 ('/swarming/api/v1/client/task/<task_id:[0-9a-f]+>/output/'
576 '<command_index:[0-9]+>',
577 ClientTaskResultOutputHandler),
578 ('/swarming/api/v1/client/task/<task_id:[0-9a-f]+>/output/all',
579 ClientTaskResultOutputAllHandler),
580 ('/swarming/api/v1/client/tasks', ClientApiTasksHandler),
581 ('/swarming/api/v1/client/tasks/count', ClientApiTasksCountHandler),
582 ]
583 return [webapp2.Route(*i) for i in routes]
OLDNEW
« no previous file with comments | « no previous file | appengine/swarming/handlers_api_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698