| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # coding: utf-8 | |
| 3 # Copyright 2014 The Swarming Authors. All rights reserved. | |
| 4 # Use of this source code is governed by the Apache v2.0 license that can be | |
| 5 # found in the LICENSE file. | |
| 6 | |
| 7 import base64 | |
| 8 import datetime | |
| 9 import logging | |
| 10 import os | |
| 11 import random | |
| 12 import sys | |
| 13 import unittest | |
| 14 | |
| 15 # Setups environment. | |
| 16 import test_env_handlers | |
| 17 | |
| 18 import webapp2 | |
| 19 import webtest | |
| 20 | |
| 21 import handlers_api | |
| 22 import handlers_bot | |
| 23 from components import ereporter2 | |
| 24 from components import utils | |
| 25 from server import config | |
| 26 from server import bot_code | |
| 27 from server import bot_management | |
| 28 from server import task_result | |
| 29 | |
| 30 | |
| 31 class ClientApiTest(test_env_handlers.AppTestBase): | |
| 32 def setUp(self): | |
| 33 super(ClientApiTest, self).setUp() | |
| 34 # By default requests in tests are coming from bot with fake IP. | |
| 35 routes = handlers_bot.get_routes() + handlers_api.get_routes() | |
| 36 app = webapp2.WSGIApplication(routes, debug=True) | |
| 37 self.app = webtest.TestApp( | |
| 38 app, | |
| 39 extra_environ={ | |
| 40 'REMOTE_ADDR': self.source_ip, | |
| 41 'SERVER_SOFTWARE': os.environ['SERVER_SOFTWARE'], | |
| 42 }) | |
| 43 self.mock( | |
| 44 ereporter2, 'log_request', | |
| 45 lambda *args, **kwargs: self.fail('%s, %s' % (args, kwargs))) | |
| 46 # Client API test cases run by default as user. | |
| 47 self.set_as_user() | |
| 48 | |
| 49 def get_client_token(self): | |
| 50 """Gets the XSRF token for client after handshake.""" | |
| 51 headers = {'X-XSRF-Token-Request': '1'} | |
| 52 params = {} | |
| 53 response = self.app.post_json( | |
| 54 '/swarming/api/v1/client/handshake', | |
| 55 headers=headers, | |
| 56 params=params).json | |
| 57 return response['xsrf_token'].encode('ascii') | |
| 58 | |
| 59 def test_list(self): | |
| 60 self.set_as_anonymous() | |
| 61 response = self.app.get('/swarming/api/v1/client/list').json | |
| 62 expected = { | |
| 63 u'bot/<bot_id:[^/]+>': u'Bot\'s meta data', | |
| 64 u'bot/<bot_id:[^/]+>/tasks': u'Tasks executed on a specific bot', | |
| 65 u'bots': u'Bots known to the server', | |
| 66 u'list': u'All query handlers', | |
| 67 u'server': u'Server details', | |
| 68 u'task/<task_id:[0-9a-f]+>': u'Task\'s result meta data', | |
| 69 u'task/<task_id:[0-9a-f]+>/output/<command_index:[0-9]+>': | |
| 70 u'Task\'s output for a single command', | |
| 71 u'task/<task_id:[0-9a-f]+>/output/all': | |
| 72 u'All output from all commands in a task', | |
| 73 u'task/<task_id:[0-9a-f]+>/request': u'Task\'s request details', | |
| 74 u'tasks': handlers_api.process_doc(handlers_api.ClientApiTasksHandler), | |
| 75 u'tasks/count': handlers_api.process_doc( | |
| 76 handlers_api.ClientApiTasksCountHandler), | |
| 77 } | |
| 78 self.assertEqual(expected, response) | |
| 79 | |
| 80 def test_handshake(self): | |
| 81 # Bare minimum: | |
| 82 headers = {'X-XSRF-Token-Request': '1'} | |
| 83 params = {} | |
| 84 response = self.app.post_json( | |
| 85 '/swarming/api/v1/client/handshake', | |
| 86 headers=headers, params=params).json | |
| 87 self.assertEqual( | |
| 88 [u'server_version', u'xsrf_token'], sorted(response)) | |
| 89 self.assertTrue(response['xsrf_token']) | |
| 90 self.assertEqual(u'v1a', response['server_version']) | |
| 91 | |
| 92 def test_handshake_extra(self): | |
| 93 errors = [] | |
| 94 def add_error(request, source, message): | |
| 95 self.assertTrue(request) | |
| 96 self.assertEqual('client', source) | |
| 97 errors.append(message) | |
| 98 self.mock(ereporter2, 'log_request', add_error) | |
| 99 headers = {'X-XSRF-Token-Request': '1'} | |
| 100 params = { | |
| 101 # Works with unknown items but logs an error. This permits catching typos. | |
| 102 'foo': 1, | |
| 103 } | |
| 104 response = self.app.post_json( | |
| 105 '/swarming/api/v1/client/handshake', | |
| 106 headers=headers, params=params).json | |
| 107 self.assertEqual( | |
| 108 [u'server_version', u'xsrf_token'], sorted(response)) | |
| 109 self.assertTrue(response['xsrf_token']) | |
| 110 self.assertEqual(u'v1a', response['server_version']) | |
| 111 expected = [ | |
| 112 'Unexpected keys superfluous: [u\'foo\']; did you make a typo?', | |
| 113 ] | |
| 114 self.assertEqual(expected, errors) | |
| 115 | |
| 116 def test_request_invalid(self): | |
| 117 record = [] | |
| 118 self.mock( | |
| 119 ereporter2, 'log_request', | |
| 120 lambda *args, **kwargs: record.append((args, kwargs))) | |
| 121 headers = {'X-XSRF-Token-Request': '1'} | |
| 122 response = self.app.post_json( | |
| 123 '/swarming/api/v1/client/handshake', headers=headers, params={}).json | |
| 124 params = { | |
| 125 'foo': 'bar', | |
| 126 'properties': {}, | |
| 127 'scheduling_expiration_secs': 30, | |
| 128 'tags': [], | |
| 129 } | |
| 130 headers = {'X-XSRF-Token': str(response['xsrf_token'])} | |
| 131 response = self.app.post_json( | |
| 132 '/swarming/api/v1/client/request', | |
| 133 headers=headers, params=params, status=400).json | |
| 134 expected = { | |
| 135 u'error': | |
| 136 u'Unexpected request keys missing: ' | |
| 137 u'[\'name\', \'priority\', \'user\'] superfluous: [u\'foo\']; ' | |
| 138 u'did you make a typo?', | |
| 139 } | |
| 140 self.assertEqual(expected, response) | |
| 141 | |
| 142 def test_request_invalid_lower_level(self): | |
| 143 headers = {'X-XSRF-Token-Request': '1'} | |
| 144 response = self.app.post_json( | |
| 145 '/swarming/api/v1/client/handshake', headers=headers, params={}).json | |
| 146 params = { | |
| 147 'name': 'job1', | |
| 148 'priority': 200, | |
| 149 'properties': { | |
| 150 'commands': [], | |
| 151 'data': [], | |
| 152 'dimensions': {}, | |
| 153 'env': {}, | |
| 154 'execution_timeout_secs': 10, | |
| 155 'io_timeout_secs': 10, | |
| 156 }, | |
| 157 'scheduling_expiration_secs': 30, | |
| 158 'tags': ['foo:bar'], | |
| 159 'user': 'joe@localhost', | |
| 160 } | |
| 161 headers = {'X-XSRF-Token': str(response['xsrf_token'])} | |
| 162 response = self.app.post_json( | |
| 163 '/swarming/api/v1/client/request', | |
| 164 headers=headers, params=params, status=400).json | |
| 165 self.assertEqual({u'error': u'use one of command or inputs_ref'}, response) | |
| 166 | |
| 167 def test_request(self): | |
| 168 self.mock(random, 'getrandbits', lambda _: 0x88) | |
| 169 now = datetime.datetime(2010, 1, 2, 3, 4, 5) | |
| 170 self.mock_now(now) | |
| 171 str_now = unicode(now.strftime(utils.DATETIME_FORMAT)) | |
| 172 headers = {'X-XSRF-Token-Request': '1'} | |
| 173 response = self.app.post_json( | |
| 174 '/swarming/api/v1/client/handshake', headers=headers, params={}).json | |
| 175 params = { | |
| 176 'name': 'job1', | |
| 177 'priority': 200, | |
| 178 'properties': { | |
| 179 'commands': [['rm', '-rf', '/']], | |
| 180 'data': [], | |
| 181 'dimensions': {}, | |
| 182 'env': {}, | |
| 183 'execution_timeout_secs': 30, | |
| 184 'io_timeout_secs': 30, | |
| 185 }, | |
| 186 'scheduling_expiration_secs': 30, | |
| 187 'tags': ['foo:bar'], | |
| 188 'user': 'joe@localhost', | |
| 189 } | |
| 190 headers = {'X-XSRF-Token': str(response['xsrf_token'])} | |
| 191 response = self.app.post_json( | |
| 192 '/swarming/api/v1/client/request', | |
| 193 headers=headers, params=params).json | |
| 194 expected = { | |
| 195 u'request': { | |
| 196 u'authenticated': [u'user', u'user@example.com'], | |
| 197 u'created_ts': str_now, | |
| 198 u'expiration_ts': unicode( | |
| 199 (now + datetime.timedelta(seconds=30)).strftime( | |
| 200 utils.DATETIME_FORMAT)), | |
| 201 u'name': u'job1', | |
| 202 u'parent_task_id': None, | |
| 203 u'priority': 200, | |
| 204 u'properties': { | |
| 205 u'commands': [[u'rm', u'-rf', u'/']], | |
| 206 u'data': [], | |
| 207 u'dimensions': {}, | |
| 208 u'env': {}, | |
| 209 u'execution_timeout_secs': 30, | |
| 210 u'extra_args': [], | |
| 211 u'grace_period_secs': 30, | |
| 212 u'idempotent': False, | |
| 213 u'inputs_ref': None, | |
| 214 u'io_timeout_secs': 30, | |
| 215 }, | |
| 216 u'properties_hash': None, | |
| 217 u'pubsub_topic': None, | |
| 218 u'pubsub_userdata': None, | |
| 219 u'tags': [ | |
| 220 u'foo:bar', | |
| 221 u'priority:200', | |
| 222 u'user:joe@localhost', | |
| 223 ], | |
| 224 u'user': u'joe@localhost', | |
| 225 }, | |
| 226 u'task_id': u'5cee488008810', | |
| 227 } | |
| 228 self.assertEqual(expected, response) | |
| 229 | |
| 230 def test_cancel(self): | |
| 231 self.mock(random, 'getrandbits', lambda _: 0x88) | |
| 232 now = datetime.datetime(2010, 1, 2, 3, 4, 5) | |
| 233 self.mock_now(now) | |
| 234 str_now = unicode(now.strftime(utils.DATETIME_FORMAT)) | |
| 235 self.set_as_admin() | |
| 236 token = self.get_client_token() | |
| 237 _, task_id = self.client_create_task_raw() | |
| 238 params = { | |
| 239 'task_id': task_id, | |
| 240 } | |
| 241 response = self.post_with_token( | |
| 242 '/swarming/api/v1/client/cancel', params, token) | |
| 243 expected = { | |
| 244 u'ok': True, | |
| 245 u'was_running': False, | |
| 246 } | |
| 247 self.assertEqual(expected, response) | |
| 248 response = self.app.get( | |
| 249 '/swarming/api/v1/client/task/' + task_id).json | |
| 250 expected = { | |
| 251 u'abandoned_ts': str_now, | |
| 252 u'bot_dimensions': None, | |
| 253 u'bot_id': None, | |
| 254 u'bot_version': None, | |
| 255 u'children_task_ids': [], | |
| 256 u'completed_ts': None, | |
| 257 u'costs_usd': [], | |
| 258 u'cost_saved_usd': None, | |
| 259 u'created_ts': str_now, | |
| 260 u'deduped_from': None, | |
| 261 u'durations': [], | |
| 262 u'exit_codes': [], | |
| 263 u'failure': False, | |
| 264 u'id': task_id, | |
| 265 u'internal_failure': False, | |
| 266 u'modified_ts': str_now, | |
| 267 u'name': u'hi', | |
| 268 u'outputs_ref': None, | |
| 269 u'properties_hash': None, | |
| 270 u'server_versions': [], | |
| 271 u'started_ts': None, | |
| 272 u'state': task_result.State.CANCELED, | |
| 273 u'tags': [u'os:Amiga', u'priority:10', u'user:joe@localhost'], | |
| 274 u'try_number': None, | |
| 275 u'user': u'joe@localhost', | |
| 276 } | |
| 277 self.assertEqual(expected, response) | |
| 278 | |
| 279 def test_get_task_metadata_unknown(self): | |
| 280 response = self.app.get( | |
| 281 '/swarming/api/v1/client/task/12300', status=404).json | |
| 282 self.assertEqual({u'error': u'Task not found'}, response) | |
| 283 | |
| 284 def test_get_task_metadata(self): | |
| 285 self.mock(random, 'getrandbits', lambda _: 0x88) | |
| 286 now = datetime.datetime(2010, 1, 2, 3, 4, 5) | |
| 287 self.mock_now(now) | |
| 288 str_now = unicode(now.strftime(utils.DATETIME_FORMAT)) | |
| 289 _, task_id = self.client_create_task_raw() | |
| 290 response = self.app.get( | |
| 291 '/swarming/api/v1/client/task/' + task_id).json | |
| 292 expected = { | |
| 293 u'abandoned_ts': None, | |
| 294 u'bot_dimensions': None, | |
| 295 u'bot_id': None, | |
| 296 u'bot_version': None, | |
| 297 u'children_task_ids': [], | |
| 298 u'completed_ts': None, | |
| 299 u'costs_usd': [], | |
| 300 u'cost_saved_usd': None, | |
| 301 u'created_ts': str_now, | |
| 302 u'deduped_from': None, | |
| 303 u'durations': [], | |
| 304 u'exit_codes': [], | |
| 305 u'failure': False, | |
| 306 u'id': u'5cee488008810', | |
| 307 u'internal_failure': False, | |
| 308 u'modified_ts': str_now, | |
| 309 u'name': u'hi', | |
| 310 u'outputs_ref': None, | |
| 311 u'properties_hash': None, | |
| 312 u'server_versions': [], | |
| 313 u'started_ts': None, | |
| 314 u'state': task_result.State.PENDING, | |
| 315 u'tags': [u'os:Amiga', u'priority:100', u'user:joe@localhost'], | |
| 316 u'try_number': None, | |
| 317 u'user': u'joe@localhost', | |
| 318 } | |
| 319 self.assertEqual(expected, response) | |
| 320 self.assertEqual('0', task_id[-1]) | |
| 321 | |
| 322 # No bot started yet. | |
| 323 run_id = task_id[:-1] + '1' | |
| 324 response = self.app.get( | |
| 325 '/swarming/api/v1/client/task/' + run_id, status=404).json | |
| 326 self.assertEqual({u'error': u'Task not found'}, response) | |
| 327 | |
| 328 self.set_as_bot() | |
| 329 self.bot_poll('bot1') | |
| 330 | |
| 331 self.set_as_user() | |
| 332 response = self.app.get( | |
| 333 '/swarming/api/v1/client/task/' + run_id).json | |
| 334 expected = { | |
| 335 u'abandoned_ts': None, | |
| 336 u'bot_dimensions': {u'id': [u'bot1'], u'os': [u'Amiga']}, | |
| 337 u'bot_id': u'bot1', | |
| 338 u'bot_version': self.bot_version, | |
| 339 u'children_task_ids': [], | |
| 340 u'completed_ts': None, | |
| 341 u'cost_usd': 0., | |
| 342 u'durations': [], | |
| 343 u'exit_codes': [], | |
| 344 u'failure': False, | |
| 345 u'id': u'5cee488008811', | |
| 346 u'internal_failure': False, | |
| 347 u'modified_ts': str_now, | |
| 348 u'outputs_ref': None, | |
| 349 u'server_versions': [u'v1a'], | |
| 350 u'started_ts': str_now, | |
| 351 u'state': task_result.State.RUNNING, | |
| 352 u'try_number': 1, | |
| 353 } | |
| 354 self.assertEqual(expected, response) | |
| 355 | |
| 356 def test_get_task_metadata_denied(self): | |
| 357 # Asserts that a non-public task can not be seen by an anonymous user. | |
| 358 _, task_id = self.client_create_task_raw() | |
| 359 | |
| 360 self.set_as_anonymous() | |
| 361 self.app.get('/swarming/api/v1/client/task/' + task_id, status=403) | |
| 362 self.assertEqual('0', task_id[-1]) | |
| 363 | |
| 364 def test_get_task_output(self): | |
| 365 self.client_create_task_raw() | |
| 366 | |
| 367 self.set_as_bot() | |
| 368 task_id = self.bot_run_task() | |
| 369 | |
| 370 self.set_as_privileged_user() | |
| 371 run_id = task_id[:-1] + '1' | |
| 372 response = self.app.get( | |
| 373 '/swarming/api/v1/client/task/%s/output/0' % task_id).json | |
| 374 self.assertEqual({'output': u'rÉsult string'}, response) | |
| 375 response = self.app.get( | |
| 376 '/swarming/api/v1/client/task/%s/output/0' % run_id).json | |
| 377 self.assertEqual({'output': u'rÉsult string'}, response) | |
| 378 | |
| 379 response = self.app.get( | |
| 380 '/swarming/api/v1/client/task/%s/output/1' % task_id).json | |
| 381 self.assertEqual({'output': None}, response) | |
| 382 response = self.app.get( | |
| 383 '/swarming/api/v1/client/task/%s/output/1' % run_id).json | |
| 384 self.assertEqual({'output': None}, response) | |
| 385 | |
| 386 def test_get_task_output_empty(self): | |
| 387 _, task_id = self.client_create_task_raw() | |
| 388 response = self.app.get( | |
| 389 '/swarming/api/v1/client/task/%s/output/0' % task_id).json | |
| 390 self.assertEqual({'output': None}, response) | |
| 391 | |
| 392 run_id = task_id[:-1] + '1' | |
| 393 response = self.app.get( | |
| 394 '/swarming/api/v1/client/task/%s/output/0' % run_id, status=404).json | |
| 395 self.assertEqual({u'error': u'Task not found'}, response) | |
| 396 | |
| 397 def test_task_deduped(self): | |
| 398 _, task_id_1 = self.client_create_task_raw(properties=dict(idempotent=True)) | |
| 399 | |
| 400 self.set_as_bot() | |
| 401 task_id_bot = self.bot_run_task() | |
| 402 self.assertEqual(task_id_1, task_id_bot[:-1] + '0') | |
| 403 self.assertEqual('1', task_id_bot[-1:]) | |
| 404 | |
| 405 # Create a second task. Results will be returned immediately without the bot | |
| 406 # running anything. | |
| 407 self.set_as_user() | |
| 408 _, task_id_2 = self.client_create_task_raw( | |
| 409 name='second', user='jack@localhost', properties=dict(idempotent=True)) | |
| 410 | |
| 411 self.set_as_bot() | |
| 412 resp = self.bot_poll() | |
| 413 self.assertEqual('sleep', resp['cmd']) | |
| 414 | |
| 415 self.set_as_user() | |
| 416 # Look at the results. It's the same as the previous run, even if task_id_2 | |
| 417 # was never executed. | |
| 418 response = self.app.get( | |
| 419 '/swarming/api/v1/client/task/%s/output/all' % task_id_2).json | |
| 420 self.assertEqual({'outputs': [u'rÉsult string']}, response) | |
| 421 | |
| 422 def test_get_task_output_all(self): | |
| 423 self.client_create_task_raw() | |
| 424 | |
| 425 self.set_as_bot() | |
| 426 token, _ = self.get_bot_token() | |
| 427 res = self.bot_poll() | |
| 428 task_id = res['manifest']['task_id'] | |
| 429 params = { | |
| 430 'cost_usd': 0.1, | |
| 431 'duration': 0.1, | |
| 432 'exit_code': 0, | |
| 433 'id': 'bot1', | |
| 434 'output': base64.b64encode('result string'), | |
| 435 'output_chunk_start': 0, | |
| 436 'task_id': task_id, | |
| 437 } | |
| 438 response = self.post_with_token( | |
| 439 '/swarming/api/v1/bot/task_update', params, token) | |
| 440 self.assertEqual({u'ok': True}, response) | |
| 441 | |
| 442 self.set_as_privileged_user() | |
| 443 run_id = task_id[:-1] + '1' | |
| 444 response = self.app.get( | |
| 445 '/swarming/api/v1/client/task/%s/output/all' % task_id).json | |
| 446 self.assertEqual({'outputs': [u'result string']}, response) | |
| 447 response = self.app.get( | |
| 448 '/swarming/api/v1/client/task/%s/output/all' % run_id).json | |
| 449 self.assertEqual({'outputs': [u'result string']}, response) | |
| 450 | |
| 451 def test_get_task_output_all_empty(self): | |
| 452 _, task_id = self.client_create_task_raw() | |
| 453 response = self.app.get( | |
| 454 '/swarming/api/v1/client/task/%s/output/all' % task_id).json | |
| 455 self.assertEqual({'outputs': []}, response) | |
| 456 | |
| 457 run_id = task_id[:-1] + '1' | |
| 458 response = self.app.get( | |
| 459 '/swarming/api/v1/client/task/%s/output/all' % run_id, status=404).json | |
| 460 self.assertEqual({u'error': u'Task not found'}, response) | |
| 461 | |
| 462 def test_get_task_request(self): | |
| 463 now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6) | |
| 464 self.mock_now(now) | |
| 465 _, task_id = self.client_create_task_raw() | |
| 466 response = self.app.get( | |
| 467 '/swarming/api/v1/client/task/%s/request' % task_id).json | |
| 468 expected = { | |
| 469 u'authenticated': [u'user', u'user@example.com'], | |
| 470 u'created_ts': unicode(now.strftime(utils.DATETIME_FORMAT)), | |
| 471 u'expiration_ts': unicode( | |
| 472 (now + datetime.timedelta(days=1)).strftime(utils.DATETIME_FORMAT)), | |
| 473 u'name': u'hi', | |
| 474 u'parent_task_id': None, | |
| 475 u'priority': 100, | |
| 476 u'properties': { | |
| 477 u'commands': [[u'python', u'run_test.py']], | |
| 478 u'data': [], | |
| 479 u'dimensions': {u'os': u'Amiga'}, | |
| 480 u'env': {}, | |
| 481 u'execution_timeout_secs': 3600, | |
| 482 u'extra_args': [], | |
| 483 u'grace_period_secs': 30, | |
| 484 u'idempotent': False, | |
| 485 u'inputs_ref': None, | |
| 486 u'io_timeout_secs': 1200, | |
| 487 }, | |
| 488 u'properties_hash': None, | |
| 489 u'pubsub_topic': None, | |
| 490 u'pubsub_userdata': None, | |
| 491 u'tags': [u'os:Amiga', u'priority:100', u'user:joe@localhost'], | |
| 492 u'user': u'joe@localhost', | |
| 493 } | |
| 494 self.assertEqual(expected, response) | |
| 495 | |
| 496 def test_tasks(self): | |
| 497 # Create two tasks, one deduped. | |
| 498 self.mock(random, 'getrandbits', lambda _: 0x66) | |
| 499 now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6) | |
| 500 now_str = unicode(now.strftime(utils.DATETIME_FORMAT)) | |
| 501 self.mock_now(now) | |
| 502 self.client_create_task_raw( | |
| 503 name='first', tags=['project:yay', 'commit:post', 'os:Win'], | |
| 504 properties=dict(idempotent=True)) | |
| 505 self.set_as_bot() | |
| 506 self.bot_run_task() | |
| 507 | |
| 508 self.set_as_user() | |
| 509 self.mock(random, 'getrandbits', lambda _: 0x88) | |
| 510 now_60 = self.mock_now(now, 60) | |
| 511 now_60_str = unicode(now_60.strftime(utils.DATETIME_FORMAT)) | |
| 512 self.client_create_task_raw( | |
| 513 name='second', user='jack@localhost', | |
| 514 tags=['project:yay', 'commit:pre', 'os:Win'], | |
| 515 properties=dict(idempotent=True)) | |
| 516 | |
| 517 self.set_as_privileged_user() | |
| 518 expected_first = { | |
| 519 u'abandoned_ts': None, | |
| 520 u'bot_dimensions': {u'id': [u'bot1'], u'os': [u'Amiga']}, | |
| 521 u'bot_id': u'bot1', | |
| 522 u'bot_version': self.bot_version, | |
| 523 u'children_task_ids': [], | |
| 524 u'completed_ts': now_str, | |
| 525 u'costs_usd': [0.1], | |
| 526 u'cost_saved_usd': None, | |
| 527 u'created_ts': now_str, | |
| 528 u'deduped_from': None, | |
| 529 u'durations': [0.1], | |
| 530 u'exit_codes': [0], | |
| 531 u'failure': False, | |
| 532 u'id': u'5cee488006610', | |
| 533 u'internal_failure': False, | |
| 534 u'modified_ts': now_str, | |
| 535 u'name': u'first', | |
| 536 u'outputs_ref': None, | |
| 537 u'properties_hash': u'8771754ee465a689f19c87f2d21ea0d9b8dd4f64', | |
| 538 u'server_versions': [u'v1a'], | |
| 539 u'started_ts': now_str, | |
| 540 u'state': task_result.State.COMPLETED, | |
| 541 u'tags': [ | |
| 542 u'commit:post', | |
| 543 u'os:Amiga', | |
| 544 u'os:Win', | |
| 545 u'priority:100', | |
| 546 u'project:yay', | |
| 547 u'user:joe@localhost', | |
| 548 ], | |
| 549 u'try_number': 1, | |
| 550 u'user': u'joe@localhost', | |
| 551 } | |
| 552 expected_second = { | |
| 553 u'abandoned_ts': None, | |
| 554 u'bot_dimensions': {u'id': [u'bot1'], u'os': [u'Amiga']}, | |
| 555 u'bot_id': u'bot1', | |
| 556 u'bot_version': self.bot_version, | |
| 557 u'children_task_ids': [], | |
| 558 u'completed_ts': now_str, | |
| 559 u'costs_usd': [], | |
| 560 u'cost_saved_usd': 0.1, | |
| 561 u'created_ts': now_60_str, | |
| 562 u'deduped_from': u'5cee488006611', | |
| 563 u'durations': [0.1], | |
| 564 u'exit_codes': [0], | |
| 565 u'failure': False, | |
| 566 u'id': u'5cfcee8008810', | |
| 567 u'internal_failure': False, | |
| 568 u'modified_ts': now_60_str, | |
| 569 u'name': u'second', | |
| 570 u'outputs_ref': None, | |
| 571 u'properties_hash': None, | |
| 572 u'server_versions': [u'v1a'], | |
| 573 u'started_ts': now_str, | |
| 574 u'state': task_result.State.COMPLETED, | |
| 575 u'tags': [ | |
| 576 u'commit:pre', | |
| 577 u'os:Amiga', | |
| 578 u'os:Win', | |
| 579 u'priority:100', | |
| 580 u'project:yay', | |
| 581 u'user:jack@localhost', | |
| 582 ], | |
| 583 u'try_number': 0, | |
| 584 u'user': u'jack@localhost', | |
| 585 } | |
| 586 | |
| 587 expected = { | |
| 588 u'cursor': None, | |
| 589 u'items': [expected_second, expected_first], | |
| 590 u'limit': 100, | |
| 591 u'sort': u'created_ts', | |
| 592 u'state': u'all', | |
| 593 } | |
| 594 resource = '/swarming/api/v1/client/tasks' | |
| 595 self.assertEqual(expected, self.app.get(resource).json) | |
| 596 | |
| 597 # It has a cursor even if there's only one element because of Search API. | |
| 598 expected = { | |
| 599 u'items': [expected_second], | |
| 600 u'limit': 100, | |
| 601 u'sort': u'created_ts', | |
| 602 u'state': u'all', | |
| 603 } | |
| 604 actual = self.app.get(resource + '?name=second').json | |
| 605 self.assertTrue(actual.pop('cursor')) | |
| 606 self.assertEqual(expected, actual) | |
| 607 | |
| 608 expected = { | |
| 609 u'cursor': None, | |
| 610 u'items': [], | |
| 611 u'limit': 100, | |
| 612 u'sort': u'created_ts', | |
| 613 u'state': u'all', | |
| 614 } | |
| 615 self.assertEqual(expected, self.app.get(resource + '?&tag=foo:bar').json) | |
| 616 | |
| 617 expected = { | |
| 618 u'cursor': None, | |
| 619 u'items': [expected_second], | |
| 620 u'limit': 100, | |
| 621 u'sort': u'created_ts', | |
| 622 u'state': u'all', | |
| 623 } | |
| 624 actual = self.app.get(resource + '?tag=project:yay&tag=commit:pre').json | |
| 625 self.assertEqual(expected, actual) | |
| 626 | |
| 627 # Test output from deduped task. | |
| 628 for task in expected['items']: | |
| 629 response = self.app.get( | |
| 630 '/swarming/api/v1/client/task/%s/output/all' % task['id']).json | |
| 631 self.assertEqual({u'outputs': [u'r\xc9sult string']}, response) | |
| 632 | |
| 633 def test_tasks_fail(self): | |
| 634 self.app.get('/swarming/api/v1/client/tasks?tags=a:b', status=403) | |
| 635 self.set_as_privileged_user() | |
| 636 # It's 'tag', not 'tags'. | |
| 637 self.app.get('/swarming/api/v1/client/tasks?tags=a:b', status=400) | |
| 638 | |
| 639 def test_count(self): | |
| 640 now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6) | |
| 641 | |
| 642 # Task in completed state. | |
| 643 self.set_as_user() | |
| 644 self.mock_now(now) | |
| 645 self.client_create_task_raw( | |
| 646 name='first', tags=['project:yay', 'commit:post', 'os:Win'], | |
| 647 properties=dict(idempotent=False)) | |
| 648 self.set_as_bot() | |
| 649 self.bot_run_task() | |
| 650 | |
| 651 # Task in pending state. | |
| 652 self.set_as_user() | |
| 653 self.mock_now(now, 60) | |
| 654 self.client_create_task_raw( | |
| 655 name='second', user='jack@localhost', | |
| 656 tags=['project:yay', 'commit:pre', 'os:Win'], | |
| 657 properties=dict(idempotent=False)) | |
| 658 | |
| 659 self.set_as_privileged_user() | |
| 660 | |
| 661 # Default 24h cutoff interval. | |
| 662 result = self.app.get('/swarming/api/v1/client/tasks/count').json | |
| 663 self.assertEqual({'count': 2}, result) | |
| 664 | |
| 665 # Test cutoff. | |
| 666 result = self.app.get( | |
| 667 '/swarming/api/v1/client/tasks/count?interval=30').json | |
| 668 self.assertEqual({'count': 1}, result) | |
| 669 | |
| 670 # Test filter by state. | |
| 671 result = self.app.get( | |
| 672 '/swarming/api/v1/client/tasks/count?state=pending').json | |
| 673 self.assertEqual({'count': 1}, result) | |
| 674 | |
| 675 # Test filter by tag. | |
| 676 result = self.app.get( | |
| 677 '/swarming/api/v1/client/tasks/count?' | |
| 678 'tag=project:yay&tag=commit:pre').json | |
| 679 self.assertEqual({'count': 1}, result) | |
| 680 | |
| 681 def test_api_bots(self): | |
| 682 self.set_as_privileged_user() | |
| 683 self.mock_now(datetime.datetime(2010, 1, 2, 3, 4, 5, 6)) | |
| 684 now_str = lambda: unicode(utils.utcnow().strftime(utils.DATETIME_FORMAT)) | |
| 685 bot_management.bot_event( | |
| 686 event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4', | |
| 687 dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65}, | |
| 688 version='123456789', quarantined=False, task_id=None, task_name=None) | |
| 689 bot1_dict = { | |
| 690 u'dimensions': {u'foo': [u'bar'], u'id': [u'id1']}, | |
| 691 u'external_ip': u'8.8.4.4', | |
| 692 u'first_seen_ts': now_str(), | |
| 693 u'id': u'id1', | |
| 694 u'is_dead': False, | |
| 695 u'last_seen_ts': now_str(), | |
| 696 u'quarantined': False, | |
| 697 u'state': {u'ram': 65}, | |
| 698 u'task_id': None, | |
| 699 u'task_name': None, | |
| 700 u'version': u'123456789', | |
| 701 } | |
| 702 | |
| 703 actual = self.app.get('/swarming/api/v1/client/bots', status=200).json | |
| 704 expected = { | |
| 705 u'items': [bot1_dict], | |
| 706 u'cursor': None, | |
| 707 u'death_timeout': config.settings().bot_death_timeout_secs, | |
| 708 u'limit': 1000, | |
| 709 u'now': now_str(), | |
| 710 } | |
| 711 self.assertEqual(expected, actual) | |
| 712 | |
| 713 # Test with limit. | |
| 714 actual = self.app.get( | |
| 715 '/swarming/api/v1/client/bots?limit=1', status=200).json | |
| 716 expected['limit'] = 1 | |
| 717 self.assertEqual(expected, actual) | |
| 718 | |
| 719 # Advance time to make bot1 dead to test filtering for dead bots. | |
| 720 self.mock_now(datetime.datetime(2011, 1, 2, 3, 4, 5, 6)) | |
| 721 bot1_dict['is_dead'] = True | |
| 722 expected['now'] = now_str() | |
| 723 | |
| 724 # Use quarantined bot to check filtering by 'quarantined' flag. | |
| 725 bot_management.bot_event( | |
| 726 event_type='bot_connected', bot_id='id2', external_ip='8.8.4.4', | |
| 727 dimensions={'foo': ['bar'], 'id': ['id2']}, state={'ram': 65}, | |
| 728 version='123456789', quarantined=True, task_id=None, task_name=None) | |
| 729 bot2_dict = { | |
| 730 u'dimensions': {u'foo': [u'bar'], u'id': [u'id2']}, | |
| 731 u'external_ip': u'8.8.4.4', | |
| 732 u'first_seen_ts': now_str(), | |
| 733 u'id': u'id2', | |
| 734 u'is_dead': False, | |
| 735 u'last_seen_ts': now_str(), | |
| 736 u'quarantined': True, | |
| 737 u'state': {u'ram': 65}, | |
| 738 u'task_id': None, | |
| 739 u'task_name': None, | |
| 740 u'version': u'123456789', | |
| 741 } | |
| 742 | |
| 743 # Test limit + cursor: start the query. | |
| 744 actual = self.app.get( | |
| 745 '/swarming/api/v1/client/bots?limit=1', status=200).json | |
| 746 expected['cursor'] = actual['cursor'] | |
| 747 expected['items'] = [bot1_dict] | |
| 748 self.assertTrue(actual['cursor']) | |
| 749 self.assertEqual(expected, actual) | |
| 750 | |
| 751 # Test limit + cursor: continue the query. | |
| 752 actual = self.app.get( | |
| 753 '/swarming/api/v1/client/bots?limit=1&cursor=%s' % actual['cursor'], | |
| 754 status=200).json | |
| 755 expected['cursor'] = None | |
| 756 expected['items'] = [bot2_dict] | |
| 757 self.assertEqual(expected, actual) | |
| 758 | |
| 759 # Filtering by 'quarantined'. | |
| 760 actual = self.app.get( | |
| 761 '/swarming/api/v1/client/bots?filter=quarantined', | |
| 762 status=200).json | |
| 763 expected['limit'] = 1000 | |
| 764 expected['cursor'] = None | |
| 765 expected['items'] = [bot2_dict] | |
| 766 self.assertEqual(expected, actual) | |
| 767 | |
| 768 # Filtering by 'is_dead'. | |
| 769 actual = self.app.get( | |
| 770 '/swarming/api/v1/client/bots?filter=is_dead', | |
| 771 status=200).json | |
| 772 expected['limit'] = 1000 | |
| 773 expected['cursor'] = None | |
| 774 expected['items'] = [bot1_dict] | |
| 775 self.assertEqual(expected, actual) | |
| 776 | |
| 777 def test_api_bot(self): | |
| 778 self.set_as_privileged_user() | |
| 779 now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6) | |
| 780 now_str = unicode(now.strftime(utils.DATETIME_FORMAT)) | |
| 781 self.mock_now(now) | |
| 782 bot_management.bot_event( | |
| 783 event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4', | |
| 784 dimensions={'foo': ['bar'], 'id': ['id1']}, state={'ram': 65}, | |
| 785 version='123456789', quarantined=False, task_id=None, task_name=None) | |
| 786 | |
| 787 actual = self.app.get('/swarming/api/v1/client/bot/id1', status=200).json | |
| 788 expected = { | |
| 789 u'dimensions': {u'foo': [u'bar'], u'id': [u'id1']}, | |
| 790 u'external_ip': u'8.8.4.4', | |
| 791 u'first_seen_ts': now_str, | |
| 792 u'id': u'id1', | |
| 793 u'is_dead': False, | |
| 794 u'last_seen_ts': now_str, | |
| 795 u'quarantined': False, | |
| 796 u'state': {u'ram': 65}, | |
| 797 u'task_id': None, | |
| 798 u'task_name': None, | |
| 799 u'version': u'123456789', | |
| 800 } | |
| 801 self.assertEqual(expected, actual) | |
| 802 | |
| 803 def test_api_bot_delete(self): | |
| 804 self.set_as_admin() | |
| 805 now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6) | |
| 806 self.mock_now(now) | |
| 807 state = { | |
| 808 'dict': {'random': 'values'}, | |
| 809 'float': 0., | |
| 810 'list': ['of', 'things'], | |
| 811 'str': u'uni', | |
| 812 } | |
| 813 bot_management.bot_event( | |
| 814 event_type='bot_connected', bot_id='id1', external_ip='8.8.4.4', | |
| 815 dimensions={'foo': ['bar'], 'id': ['id1']}, state=state, | |
| 816 version='123456789', quarantined=False, task_id=None, task_name=None) | |
| 817 | |
| 818 token = self.get_client_token() | |
| 819 actual = self.app.delete( | |
| 820 '/swarming/api/v1/client/bot/id1', | |
| 821 status=200, | |
| 822 headers={'X-XSRF-Token': str(token)}).json | |
| 823 expected = { | |
| 824 u'deleted': True, | |
| 825 } | |
| 826 self.assertEqual(expected, actual) | |
| 827 | |
| 828 actual = self.app.get('/swarming/api/v1/client/bot/id1', status=404).json | |
| 829 expected = { | |
| 830 u'error': u'Bot not found', | |
| 831 } | |
| 832 self.assertEqual(expected, actual) | |
| 833 | |
| 834 def test_api_bot_tasks_empty(self): | |
| 835 self.set_as_privileged_user() | |
| 836 now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6) | |
| 837 self.mock_now(now) | |
| 838 actual = self.app.get('/swarming/api/v1/client/bot/id1/tasks').json | |
| 839 expected = { | |
| 840 u'cursor': None, | |
| 841 u'limit': 100, | |
| 842 u'now': now.strftime(utils.DATETIME_FORMAT), | |
| 843 u'items': [], | |
| 844 } | |
| 845 self.assertEqual(expected, actual) | |
| 846 | |
| 847 def test_api_bot_tasks(self): | |
| 848 self.mock(random, 'getrandbits', lambda _: 0x88) | |
| 849 now = datetime.datetime(2010, 1, 2, 3, 4, 5, 6) | |
| 850 now_str = unicode(now.strftime(utils.DATETIME_FORMAT)) | |
| 851 self.mock_now(now) | |
| 852 | |
| 853 self.set_as_bot() | |
| 854 self.client_create_task_raw() | |
| 855 token, _ = self.get_bot_token() | |
| 856 res = self.bot_poll() | |
| 857 self.bot_complete_task(token, task_id=res['manifest']['task_id']) | |
| 858 | |
| 859 now_1 = self.mock_now(now, 1) | |
| 860 now_1_str = unicode(now_1.strftime(utils.DATETIME_FORMAT)) | |
| 861 self.mock(random, 'getrandbits', lambda _: 0x55) | |
| 862 self.client_create_task_raw(name='ho') | |
| 863 token, _ = self.get_bot_token() | |
| 864 res = self.bot_poll() | |
| 865 self.bot_complete_task( | |
| 866 token, exit_code=1, task_id=res['manifest']['task_id']) | |
| 867 | |
| 868 self.set_as_privileged_user() | |
| 869 actual = self.app.get('/swarming/api/v1/client/bot/bot1/tasks?limit=1').json | |
| 870 expected = { | |
| 871 u'limit': 1, | |
| 872 u'now': now_1_str, | |
| 873 u'items': [ | |
| 874 { | |
| 875 u'abandoned_ts': None, | |
| 876 u'bot_dimensions': {u'id': [u'bot1'], u'os': [u'Amiga']}, | |
| 877 u'bot_id': u'bot1', | |
| 878 u'bot_version': self.bot_version, | |
| 879 u'children_task_ids': [], | |
| 880 u'completed_ts': now_1_str, | |
| 881 u'cost_usd': 0.1, | |
| 882 u'durations': [0.1], | |
| 883 u'exit_codes': [1], | |
| 884 u'failure': True, | |
| 885 u'id': u'5cee870005511', | |
| 886 u'internal_failure': False, | |
| 887 u'modified_ts': now_1_str, | |
| 888 u'outputs_ref': None, | |
| 889 u'server_versions': [u'v1a'], | |
| 890 u'started_ts': now_1_str, | |
| 891 u'state': task_result.State.COMPLETED, | |
| 892 u'try_number': 1, | |
| 893 }, | |
| 894 ], | |
| 895 } | |
| 896 cursor = actual.pop('cursor') | |
| 897 self.assertEqual(expected, actual) | |
| 898 | |
| 899 actual = self.app.get( | |
| 900 '/swarming/api/v1/client/bot/bot1/tasks?limit=1&cursor=' + cursor).json | |
| 901 expected = { | |
| 902 u'cursor': None, | |
| 903 u'limit': 1, | |
| 904 u'now': now_1_str, | |
| 905 u'items': [ | |
| 906 { | |
| 907 u'abandoned_ts': None, | |
| 908 u'bot_dimensions': {u'id': [u'bot1'], u'os': [u'Amiga']}, | |
| 909 u'bot_id': u'bot1', | |
| 910 u'bot_version': self.bot_version, | |
| 911 u'children_task_ids': [], | |
| 912 u'completed_ts': now_str, | |
| 913 u'cost_usd': 0.1, | |
| 914 u'durations': [0.1], | |
| 915 u'exit_codes': [0], | |
| 916 u'failure': False, | |
| 917 u'id': u'5cee488008811', | |
| 918 u'internal_failure': False, | |
| 919 u'modified_ts': now_str, | |
| 920 u'outputs_ref': None, | |
| 921 u'server_versions': [u'v1a'], | |
| 922 u'started_ts': now_str, | |
| 923 u'state': task_result.State.COMPLETED, | |
| 924 u'try_number': 1, | |
| 925 }, | |
| 926 ], | |
| 927 } | |
| 928 self.assertEqual(expected, actual) | |
| 929 | |
| 930 def test_api_bot_missing(self): | |
| 931 self.set_as_privileged_user() | |
| 932 self.app.get('/swarming/api/v1/client/bot/unknown', status=404) | |
| 933 | |
| 934 def test_api_server(self): | |
| 935 self.set_as_privileged_user() | |
| 936 actual = self.app.get('/swarming/api/v1/client/server').json | |
| 937 expected = { | |
| 938 'bot_version': bot_code.get_bot_version('http://localhost'), | |
| 939 } | |
| 940 self.assertEqual(expected, actual) | |
| 941 | |
| 942 | |
| 943 if __name__ == '__main__': | |
| 944 if '-v' in sys.argv: | |
| 945 unittest.TestCase.maxDiff = None | |
| 946 logging.basicConfig( | |
| 947 level=logging.DEBUG if '-v' in sys.argv else logging.CRITICAL, | |
| 948 format='%(levelname)-7s %(filename)s:%(lineno)3d %(message)s') | |
| 949 unittest.main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1458553003:
Delete old APIs on both Swarming and Isolate servers. (Closed)
Base URL: git@github.com:luci/luci-py.git@1_warning