| OLD | NEW |
| 1 # Copyright 2013 The LUCI Authors. All rights reserved. | 1 # Copyright 2013 The LUCI Authors. All rights reserved. |
| 2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
| 3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
| 4 | 4 |
| 5 """Main entry point for Swarming service. | 5 """Main entry point for Swarming service. |
| 6 | 6 |
| 7 This file contains the URL handlers for all the Swarming service URLs, | 7 This file contains the URL handlers for all the Swarming service URLs, |
| 8 implemented using the webapp2 framework. | 8 implemented using the webapp2 framework. |
| 9 """ | 9 """ |
| 10 | 10 |
| 11 import collections | 11 import collections |
| 12 import datetime | 12 import datetime |
| 13 import itertools | 13 import itertools |
| 14 import os | 14 import os |
| 15 import re | 15 import re |
| 16 | 16 |
| 17 import webapp2 | 17 import webapp2 |
| 18 | 18 |
| 19 from google.appengine import runtime | 19 from google.appengine import runtime |
| 20 from google.appengine.api import search | |
| 21 from google.appengine.api import users | 20 from google.appengine.api import users |
| 22 from google.appengine.datastore import datastore_query | 21 from google.appengine.datastore import datastore_query |
| 23 from google.appengine.ext import ndb | 22 from google.appengine.ext import ndb |
| 24 | 23 |
| 25 import handlers_bot | 24 import handlers_bot |
| 26 import handlers_backend | 25 import handlers_backend |
| 27 import mapreduce_jobs | 26 import mapreduce_jobs |
| 28 import template | 27 import template |
| 29 from components import auth | 28 from components import auth |
| 29 from components import datastore_utils |
| 30 from components import utils | 30 from components import utils |
| 31 from server import acl | 31 from server import acl |
| 32 from server import bot_code | 32 from server import bot_code |
| 33 from server import bot_management | 33 from server import bot_management |
| 34 from server import config | 34 from server import config |
| 35 from server import stats_gviz | 35 from server import stats_gviz |
| 36 from server import task_pack | 36 from server import task_pack |
| 37 from server import task_request | 37 from server import task_request |
| 38 from server import task_result | 38 from server import task_result |
| 39 from server import task_scheduler | 39 from server import task_scheduler |
| (...skipping 365 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 405 'Using this order resets order to \'Created\'.'), | 405 'Using this order resets order to \'Created\'.'), |
| 406 ('canceled', 'Canceled', | 406 ('canceled', 'Canceled', |
| 407 'The task was explictly canceled by a user before it started ' | 407 'The task was explictly canceled by a user before it started ' |
| 408 'executing. Using this order resets order to \'Created\'.'), | 408 'executing. Using this order resets order to \'Created\'.'), |
| 409 ], | 409 ], |
| 410 ] | 410 ] |
| 411 | 411 |
| 412 @auth.autologin | 412 @auth.autologin |
| 413 @auth.require(acl.is_user) | 413 @auth.require(acl.is_user) |
| 414 def get(self): | 414 def get(self): |
| 415 """Handles both ndb.Query searches and search.Index().search() queries. | |
| 416 | |
| 417 If |task_name| is set or not affects the meaning of |cursor|. When set, the | |
| 418 cursor is for search.Index, otherwise the cursor is for a ndb.Query. | |
| 419 """ | |
| 420 cursor_str = self.request.get('cursor') | 415 cursor_str = self.request.get('cursor') |
| 421 limit = int(self.request.get('limit', 100)) | 416 limit = int(self.request.get('limit', 100)) |
| 422 sort = self.request.get('sort', self.SORT_CHOICES[0][0]) | 417 sort = self.request.get('sort', self.SORT_CHOICES[0][0]) |
| 423 state = self.request.get('state', self.STATE_CHOICES[0][0][0]) | 418 state = self.request.get('state', self.STATE_CHOICES[0][0][0]) |
| 424 task_name = self.request.get('task_name', '').strip() | |
| 425 counts = self.request.get('counts', '').strip() | 419 counts = self.request.get('counts', '').strip() |
| 426 task_tags = [ | 420 task_tags = [ |
| 427 line for line in self.request.get('task_tag', '').splitlines() if line | 421 line for line in self.request.get('task_tag', '').splitlines() if line |
| 428 ] | 422 ] |
| 429 | 423 |
| 430 if not any(sort == i[0] for i in self.SORT_CHOICES): | 424 if not any(sort == i[0] for i in self.SORT_CHOICES): |
| 431 self.abort(400, 'Invalid sort') | 425 self.abort(400, 'Invalid sort') |
| 432 if not any(any(state == i[0] for i in j) for j in self.STATE_CHOICES): | 426 if not any(any(state == i[0] for i in j) for j in self.STATE_CHOICES): |
| 433 self.abort(400, 'Invalid state') | 427 self.abort(400, 'Invalid state') |
| 434 | 428 |
| 435 if sort != 'created_ts': | 429 if sort != 'created_ts': |
| 436 # Zap all filters in this case to reduce the number of required indexes. | 430 # Zap all filters in this case to reduce the number of required indexes. |
| 437 # Revisit according to the user requests. | 431 # Revisit according to the user requests. |
| 438 state = 'all' | 432 state = 'all' |
| 439 | 433 |
| 440 now = utils.utcnow() | 434 now = utils.utcnow() |
| 441 # "Temporarily" disable the count. This is too slow on the prod server | 435 # "Temporarily" disable the count. This is too slow on the prod server |
| 442 # (>10s). The fix is to have the web page do a XHR query to get the values | 436 # (>10s). The fix is to have the web page do a XHR query to get the values |
| 443 # asynchronously. | 437 # asynchronously. |
| 444 counts_future = None | 438 counts_future = None |
| 445 if counts == 'true': | 439 if counts == 'true': |
| 446 counts_future = self._get_counts_future(now) | 440 counts_future = self._get_counts_future(now) |
| 447 | 441 |
| 448 # This call is synchronous. | |
| 449 try: | 442 try: |
| 450 tasks, cursor_str, sort, state = task_result.get_tasks( | 443 if task_tags: |
| 451 limit, cursor_str, sort, state, task_tags, task_name) | 444 # Enforce created_ts when tags are used. |
| 445 sort = 'created_ts' |
| 446 query = task_result.get_result_summaries_query( |
| 447 None, None, sort, state, task_tags) |
| 448 tasks, cursor_str = datastore_utils.fetch_page(query, limit, cursor_str) |
| 452 | 449 |
| 453 # Prefetch the TaskRequest all at once, so that ndb's in-process cache has | 450 # Prefetch the TaskRequest all at once, so that ndb's in-process cache has |
| 454 # it instead of fetching them one at a time indirectly when using | 451 # it instead of fetching them one at a time indirectly when using |
| 455 # TaskResultSummary.request_key.get(). | 452 # TaskResultSummary.request_key.get(). |
| 456 futures = ndb.get_multi_async(t.request_key for t in tasks) | 453 futures = ndb.get_multi_async(t.request_key for t in tasks) |
| 457 | 454 |
| 458 # Evaluate the counts to print the filtering columns with the associated | 455 # Evaluate the counts to print the filtering columns with the associated |
| 459 # numbers. | 456 # numbers. |
| 460 state_choices = self._get_state_choices(counts_future) | 457 state_choices = self._get_state_choices(counts_future) |
| 461 except (search.QueryError, ValueError) as e: | 458 except ValueError as e: |
| 462 self.abort(400, str(e)) | 459 self.abort(400, str(e)) |
| 463 | 460 |
| 464 def safe_sum(items): | 461 def safe_sum(items): |
| 465 return sum(items, datetime.timedelta()) | 462 return sum(items, datetime.timedelta()) |
| 466 | 463 |
| 467 def avg(items): | 464 def avg(items): |
| 468 if not items: | 465 if not items: |
| 469 return 0. | 466 return 0. |
| 470 return safe_sum(items) / len(items) | 467 return safe_sum(items) / len(items) |
| 471 | 468 |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 507 'limit': limit, | 504 'limit': limit, |
| 508 'now': now, | 505 'now': now, |
| 509 'pending_average': avg(pendings), | 506 'pending_average': avg(pendings), |
| 510 'pending_median': median(pendings), | 507 'pending_median': median(pendings), |
| 511 'pending_sum': safe_sum(pendings), | 508 'pending_sum': safe_sum(pendings), |
| 512 'show_footer': bool(pendings or durations), | 509 'show_footer': bool(pendings or durations), |
| 513 'sort': sort, | 510 'sort': sort, |
| 514 'sort_choices': self.SORT_CHOICES, | 511 'sort_choices': self.SORT_CHOICES, |
| 515 'state': state, | 512 'state': state, |
| 516 'state_choices': state_choices, | 513 'state_choices': state_choices, |
| 517 'task_name': task_name, | |
| 518 'task_tag': '\n'.join(task_tags), | 514 'task_tag': '\n'.join(task_tags), |
| 519 'tasks': tasks, | 515 'tasks': tasks, |
| 520 'total_cost_usd': total_cost_usd, | 516 'total_cost_usd': total_cost_usd, |
| 521 'total_cost_saved_usd': total_cost_saved_usd, | 517 'total_cost_saved_usd': total_cost_saved_usd, |
| 522 'total_saved': total_saved, | 518 'total_saved': total_saved, |
| 523 'total_saved_percent': total_saved_percent, | 519 'total_saved_percent': total_saved_percent, |
| 524 'xsrf_token': self.generate_xsrf_token(), | 520 'xsrf_token': self.generate_xsrf_token(), |
| 525 } | 521 } |
| 526 # TODO(maruel): If admin or if the user is task's .user, show the Cancel | 522 # TODO(maruel): If admin or if the user is task's .user, show the Cancel |
| 527 # button. Do not show otherwise. | 523 # button. Do not show otherwise. |
| (...skipping 260 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 788 # If running on a local dev server, allow bots to connect without prior | 784 # If running on a local dev server, allow bots to connect without prior |
| 789 # groups configuration. Useful when running smoke test. | 785 # groups configuration. Useful when running smoke test. |
| 790 if utils.is_local_dev_server(): | 786 if utils.is_local_dev_server(): |
| 791 acl.bootstrap_dev_server_acls() | 787 acl.bootstrap_dev_server_acls() |
| 792 | 788 |
| 793 # TODO(maruel): Split backend into a separate module. For now add routes here. | 789 # TODO(maruel): Split backend into a separate module. For now add routes here. |
| 794 routes.extend(handlers_backend.get_routes()) | 790 routes.extend(handlers_backend.get_routes()) |
| 795 routes.extend(handlers_bot.get_routes()) | 791 routes.extend(handlers_bot.get_routes()) |
| 796 | 792 |
| 797 return webapp2.WSGIApplication(routes, debug=debug) | 793 return webapp2.WSGIApplication(routes, debug=debug) |
| OLD | NEW |