| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # coding=utf-8 | 2 # coding=utf-8 |
| 3 # Copyright 2013 The LUCI Authors. All rights reserved. | 3 # Copyright 2013 The LUCI Authors. All rights reserved. |
| 4 # Use of this source code is governed by the Apache v2.0 license that can be | 4 # Use of this source code is governed by the Apache v2.0 license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 import base64 | 7 import base64 |
| 8 import json | 8 import json |
| 9 import logging | 9 import logging |
| 10 import os | 10 import os |
| (...skipping 20 matching lines...) Expand all Loading... |
| 31 import fake_swarming | 31 import fake_swarming |
| 32 import task_runner | 32 import task_runner |
| 33 | 33 |
| 34 CLIENT_DIR = os.path.normpath( | 34 CLIENT_DIR = os.path.normpath( |
| 35 os.path.join(test_env_bot_code.BOT_DIR, '..', '..', '..', 'client')) | 35 os.path.join(test_env_bot_code.BOT_DIR, '..', '..', '..', 'client')) |
| 36 | 36 |
| 37 sys.path.insert(0, os.path.join(CLIENT_DIR, 'tests')) | 37 sys.path.insert(0, os.path.join(CLIENT_DIR, 'tests')) |
| 38 import isolateserver_mock | 38 import isolateserver_mock |
| 39 | 39 |
| 40 | 40 |
| 41 def get_manifest(script=None, inputs_ref=None, **kwargs): | 41 def get_manifest(script=None, isolated=None, **kwargs): |
| 42 isolated_input = isolated and isolated.get('input') |
| 42 out = { | 43 out = { |
| 43 'bot_id': 'localhost', | 44 'bot_id': 'localhost', |
| 44 'command': | 45 'command': |
| 45 [sys.executable, '-u', '-c', script] if not inputs_ref else None, | 46 [sys.executable, '-u', '-c', script] if not isolated_input else None, |
| 46 'env': {}, | 47 'env': {}, |
| 47 'extra_args': [], | 48 'extra_args': [], |
| 48 'grace_period': 30., | 49 'grace_period': 30., |
| 49 'hard_timeout': 10., | 50 'hard_timeout': 10., |
| 50 'inputs_ref': inputs_ref, | |
| 51 'io_timeout': 10., | 51 'io_timeout': 10., |
| 52 'isolated': isolated, |
| 52 'task_id': 23, | 53 'task_id': 23, |
| 53 } | 54 } |
| 54 out.update(kwargs) | 55 out.update(kwargs) |
| 55 return out | 56 return out |
| 56 | 57 |
| 57 | 58 |
| 58 class TestTaskRunnerBase(net_utils.TestCase): | 59 class TestTaskRunnerBase(net_utils.TestCase): |
| 59 def setUp(self): | 60 def setUp(self): |
| 60 super(TestTaskRunnerBase, self).setUp() | 61 super(TestTaskRunnerBase, self).setUp() |
| 61 self.root_dir = tempfile.mkdtemp(prefix='task_runner') | 62 self.root_dir = tempfile.mkdtemp(prefix='task_runner') |
| (...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 175 | 176 |
| 176 manifest = os.path.join(self.root_dir, 'manifest') | 177 manifest = os.path.join(self.root_dir, 'manifest') |
| 177 with open(manifest, 'wb') as f: | 178 with open(manifest, 'wb') as f: |
| 178 data = { | 179 data = { |
| 179 'bot_id': 'localhost', | 180 'bot_id': 'localhost', |
| 180 'command': ['a'], | 181 'command': ['a'], |
| 181 'env': {'d': 'e'}, | 182 'env': {'d': 'e'}, |
| 182 'extra_args': [], | 183 'extra_args': [], |
| 183 'grace_period': 30., | 184 'grace_period': 30., |
| 184 'hard_timeout': 10, | 185 'hard_timeout': 10, |
| 185 'inputs_ref': None, | |
| 186 'io_timeout': 11, | 186 'io_timeout': 11, |
| 187 'isolated': None, |
| 187 'task_id': 23, | 188 'task_id': 23, |
| 188 } | 189 } |
| 189 json.dump(data, f) | 190 json.dump(data, f) |
| 190 | 191 |
| 191 out_file = os.path.join(self.root_dir, 'work', 'task_runner_out.json') | 192 out_file = os.path.join(self.root_dir, 'work', 'task_runner_out.json') |
| 192 task_runner.load_and_run(manifest, server, 3600., time.time(), out_file, 1) | 193 task_runner.load_and_run(manifest, server, 3600., time.time(), out_file, 1) |
| 193 expected = { | 194 expected = { |
| 194 u'exit_code': 1, | 195 u'exit_code': 1, |
| 195 u'hard_timeout': False, | 196 u'hard_timeout': False, |
| 196 u'io_timeout': False, | 197 u'io_timeout': False, |
| (...skipping 30 matching lines...) Expand all Loading... |
| 227 manifest = os.path.join(self.root_dir, 'manifest') | 228 manifest = os.path.join(self.root_dir, 'manifest') |
| 228 with open(manifest, 'wb') as f: | 229 with open(manifest, 'wb') as f: |
| 229 data = { | 230 data = { |
| 230 'bot_id': 'localhost', | 231 'bot_id': 'localhost', |
| 231 'command': None, | 232 'command': None, |
| 232 'env': {'d': 'e'}, | 233 'env': {'d': 'e'}, |
| 233 'extra_args': ['foo', 'bar'], | 234 'extra_args': ['foo', 'bar'], |
| 234 'grace_period': 30., | 235 'grace_period': 30., |
| 235 'hard_timeout': 10, | 236 'hard_timeout': 10, |
| 236 'io_timeout': 11, | 237 'io_timeout': 11, |
| 237 'inputs_ref': { | 238 'isolated': { |
| 238 'isolated': '123', | 239 'input': '123', |
| 239 'isolatedserver': 'http://localhost:1', | 240 'server': 'http://localhost:1', |
| 240 'namespace': 'default-gzip', | 241 'namespace': 'default-gzip', |
| 241 }, | 242 }, |
| 242 'task_id': 23, | 243 'task_id': 23, |
| 243 } | 244 } |
| 244 json.dump(data, f) | 245 json.dump(data, f) |
| 245 | 246 |
| 246 out_file = os.path.join(self.root_dir, 'work', 'task_runner_out.json') | 247 out_file = os.path.join(self.root_dir, 'work', 'task_runner_out.json') |
| 247 task_runner.load_and_run(manifest, server, 3600., time.time(), out_file, 1) | 248 task_runner.load_and_run(manifest, server, 3600., time.time(), out_file, 1) |
| 248 expected = { | 249 expected = { |
| 249 u'exit_code': 0, | 250 u'exit_code': 0, |
| (...skipping 20 matching lines...) Expand all Loading... |
| 270 | 271 |
| 271 def test_run_command_isolated(self): | 272 def test_run_command_isolated(self): |
| 272 # This runs the command for real. | 273 # This runs the command for real. |
| 273 self.requests( | 274 self.requests( |
| 274 cost_usd=1, exit_code=0, | 275 cost_usd=1, exit_code=0, |
| 275 outputs_ref={ | 276 outputs_ref={ |
| 276 u'isolated': u'123', | 277 u'isolated': u'123', |
| 277 u'isolatedserver': u'http://localhost:1', | 278 u'isolatedserver': u'http://localhost:1', |
| 278 u'namespace': u'default-gzip', | 279 u'namespace': u'default-gzip', |
| 279 }) | 280 }) |
| 280 task_details = self.get_task_details(inputs_ref={ | 281 task_details = self.get_task_details(isolated={ |
| 281 'isolated': '123', | 282 'input': '123', |
| 282 'isolatedserver': 'localhost:1', | 283 'server': 'localhost:1', |
| 283 'namespace': 'default-gzip', | 284 'namespace': 'default-gzip', |
| 284 }, extra_args=['foo', 'bar']) | 285 }, extra_args=['foo', 'bar']) |
| 285 # Mock running run_isolated with a script. | 286 # Mock running run_isolated with a script. |
| 286 SCRIPT_ISOLATED = ( | 287 SCRIPT_ISOLATED = ( |
| 287 'import json, sys;\n' | 288 'import json, sys;\n' |
| 288 'if len(sys.argv) != 2:\n' | 289 'if len(sys.argv) != 2:\n' |
| 289 ' raise Exception(sys.argv);\n' | 290 ' raise Exception(sys.argv);\n' |
| 290 'with open(sys.argv[1], \'wb\') as f:\n' | 291 'with open(sys.argv[1], \'wb\') as f:\n' |
| 291 ' json.dump({\n' | 292 ' json.dump({\n' |
| 292 ' \'exit_code\': 0,\n' | 293 ' \'exit_code\': 0,\n' |
| (...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 342 { | 343 { |
| 343 'bot_id': 'localhost', | 344 'bot_id': 'localhost', |
| 344 'command': [ | 345 'command': [ |
| 345 'executable_that_shouldnt_be_on_your_system', | 346 'executable_that_shouldnt_be_on_your_system', |
| 346 'thus_raising_OSError', | 347 'thus_raising_OSError', |
| 347 ], | 348 ], |
| 348 'env': {}, | 349 'env': {}, |
| 349 'extra_args': [], | 350 'extra_args': [], |
| 350 'grace_period': 30., | 351 'grace_period': 30., |
| 351 'hard_timeout': 6, | 352 'hard_timeout': 6, |
| 352 'inputs_ref': None, | |
| 353 'io_timeout': 6, | 353 'io_timeout': 6, |
| 354 'isolated': None, |
| 354 'task_id': 23, | 355 'task_id': 23, |
| 355 }) | 356 }) |
| 356 expected = { | 357 expected = { |
| 357 u'exit_code': -1, | 358 u'exit_code': -1, |
| 358 u'hard_timeout': False, | 359 u'hard_timeout': False, |
| 359 u'io_timeout': False, | 360 u'io_timeout': False, |
| 360 u'must_signal_internal_failure': None, | 361 u'must_signal_internal_failure': None, |
| 361 u'version': task_runner.OUT_VERSION, | 362 u'version': task_runner.OUT_VERSION, |
| 362 } | 363 } |
| 363 self.assertEqual(expected, self._run_command(task_details)) | 364 self.assertEqual(expected, self._run_command(task_details)) |
| (...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 451 ] | 452 ] |
| 452 self.expected_requests(requests) | 453 self.expected_requests(requests) |
| 453 task_details = task_runner.TaskDetails( | 454 task_details = task_runner.TaskDetails( |
| 454 { | 455 { |
| 455 'bot_id': 'localhost', | 456 'bot_id': 'localhost', |
| 456 'command': ['large', 'executable'], | 457 'command': ['large', 'executable'], |
| 457 'env': {'foo': 'bar'}, | 458 'env': {'foo': 'bar'}, |
| 458 'extra_args': [], | 459 'extra_args': [], |
| 459 'grace_period': 30., | 460 'grace_period': 30., |
| 460 'hard_timeout': 60, | 461 'hard_timeout': 60, |
| 461 'inputs_ref': None, | |
| 462 'io_timeout': 60, | 462 'io_timeout': 60, |
| 463 'isolated': None, |
| 463 'task_id': 23, | 464 'task_id': 23, |
| 464 }) | 465 }) |
| 465 expected = { | 466 expected = { |
| 466 u'exit_code': 0, | 467 u'exit_code': 0, |
| 467 u'hard_timeout': False, | 468 u'hard_timeout': False, |
| 468 u'io_timeout': False, | 469 u'io_timeout': False, |
| 469 u'must_signal_internal_failure': None, | 470 u'must_signal_internal_failure': None, |
| 470 u'version': task_runner.OUT_VERSION, | 471 u'version': task_runner.OUT_VERSION, |
| 471 } | 472 } |
| 472 self.assertEqual(expected, self._run_command(task_details)) | 473 self.assertEqual(expected, self._run_command(task_details)) |
| (...skipping 343 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 816 'command': ['python', 'parent.py'], | 817 'command': ['python', 'parent.py'], |
| 817 'files': { | 818 'files': { |
| 818 name: { | 819 name: { |
| 819 'h': server.add_content_compressed('default-gzip', content), | 820 'h': server.add_content_compressed('default-gzip', content), |
| 820 's': len(content), | 821 's': len(content), |
| 821 } for name, content in files.iteritems() | 822 } for name, content in files.iteritems() |
| 822 }, | 823 }, |
| 823 }) | 824 }) |
| 824 isolated_digest = server.add_content_compressed('default-gzip', isolated) | 825 isolated_digest = server.add_content_compressed('default-gzip', isolated) |
| 825 manifest = get_manifest( | 826 manifest = get_manifest( |
| 826 inputs_ref={ | 827 isolated={ |
| 827 'isolated': isolated_digest, | 828 'input': isolated_digest, |
| 828 'namespace': 'default-gzip', | 829 'namespace': 'default-gzip', |
| 829 'isolatedserver': server.url, | 830 'server': server.url, |
| 830 }) | 831 }) |
| 831 expected = { | 832 expected = { |
| 832 u'exit_code': 0, | 833 u'exit_code': 0, |
| 833 u'hard_timeout': False, | 834 u'hard_timeout': False, |
| 834 u'io_timeout': False, | 835 u'io_timeout': False, |
| 835 u'must_signal_internal_failure': None, | 836 u'must_signal_internal_failure': None, |
| 836 u'version': task_runner.OUT_VERSION, | 837 u'version': task_runner.OUT_VERSION, |
| 837 } | 838 } |
| 838 self.assertEqual(expected, self._load_and_run(manifest)) | 839 self.assertEqual(expected, self._load_and_run(manifest)) |
| 839 finally: | 840 finally: |
| (...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 938 'files': { | 939 'files': { |
| 939 name: { | 940 name: { |
| 940 'h': server.add_content_compressed('default-gzip', content), | 941 'h': server.add_content_compressed('default-gzip', content), |
| 941 's': len(content), | 942 's': len(content), |
| 942 } for name, content in files.iteritems() | 943 } for name, content in files.iteritems() |
| 943 }, | 944 }, |
| 944 }) | 945 }) |
| 945 isolated_digest = server.add_content_compressed('default-gzip', isolated) | 946 isolated_digest = server.add_content_compressed('default-gzip', isolated) |
| 946 try: | 947 try: |
| 947 manifest = get_manifest( | 948 manifest = get_manifest( |
| 948 inputs_ref={ | 949 isolated={ |
| 949 'isolated': isolated_digest, | 950 'input': isolated_digest, |
| 950 'namespace': 'default-gzip', | 951 'namespace': 'default-gzip', |
| 951 'isolatedserver': server.url, | 952 'server': server.url, |
| 952 }, | 953 }, |
| 953 # TODO(maruel): A bit cheezy, we'd want the I/O timeout to be just | 954 # TODO(maruel): A bit cheezy, we'd want the I/O timeout to be just |
| 954 # enough to have the time for the PID to be printed but not more. | 955 # enough to have the time for the PID to be printed but not more. |
| 955 io_timeout=1, | 956 io_timeout=1, |
| 956 grace_period=self.SHORT_TIME_OUT) | 957 grace_period=self.SHORT_TIME_OUT) |
| 957 expected = { | 958 expected = { |
| 958 u'exit_code': exit_code, | 959 u'exit_code': exit_code, |
| 959 u'hard_timeout': False, | 960 u'hard_timeout': False, |
| 960 u'io_timeout': True, | 961 u'io_timeout': True, |
| 961 u'must_signal_internal_failure': None, | 962 u'must_signal_internal_failure': None, |
| (...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1095 fix_encoding.fix_encoding() | 1096 fix_encoding.fix_encoding() |
| 1096 if '-v' in sys.argv: | 1097 if '-v' in sys.argv: |
| 1097 unittest.TestCase.maxDiff = None | 1098 unittest.TestCase.maxDiff = None |
| 1098 logging_utils.prepare_logging(None) | 1099 logging_utils.prepare_logging(None) |
| 1099 logging_utils.set_console_level( | 1100 logging_utils.set_console_level( |
| 1100 logging.DEBUG if '-v' in sys.argv else logging.CRITICAL+1) | 1101 logging.DEBUG if '-v' in sys.argv else logging.CRITICAL+1) |
| 1101 # Fix litteral text expectation. | 1102 # Fix litteral text expectation. |
| 1102 os.environ['LANG'] = 'en_US.UTF-8' | 1103 os.environ['LANG'] = 'en_US.UTF-8' |
| 1103 os.environ['LANGUAGE'] = 'en_US.UTF-8' | 1104 os.environ['LANGUAGE'] = 'en_US.UTF-8' |
| 1104 unittest.main() | 1105 unittest.main() |
| OLD | NEW |