| OLD | NEW |
| 1 # Copyright 2014 The LUCI Authors. All rights reserved. | 1 # Copyright 2014 The LUCI Authors. All rights reserved. |
| 2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
| 3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
| 4 | 4 |
| 5 """Generates the swarming_bot.zip archive for the bot. | 5 """Generates the swarming_bot.zip archive for the bot. |
| 6 | 6 |
| 7 Unlike the other source files, this file can be run from ../tools/bot_archive.py | 7 Unlike the other source files, this file can be run from ../tools/bot_archive.py |
| 8 stand-alone to generate a swarming_bot.zip for local testing so it doesn't | 8 stand-alone to generate a swarming_bot.zip for local testing so it doesn't |
| 9 import anything from the AppEngine SDK. | 9 import anything from the AppEngine SDK. |
| 10 | 10 |
| (...skipping 359 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 370 partial = os.path.sep.join(parts[:i]) | 370 partial = os.path.sep.join(parts[:i]) |
| 371 if os.path.isfile(partial): | 371 if os.path.isfile(partial): |
| 372 with open(partial) as f: | 372 with open(partial) as f: |
| 373 link = f.read() | 373 link = f.read() |
| 374 assert '\n' not in link and link, link | 374 assert '\n' not in link and link, link |
| 375 parts[i-1] = link | 375 parts[i-1] = link |
| 376 return os.path.normpath(os.path.sep.join(parts)) | 376 return os.path.normpath(os.path.sep.join(parts)) |
| 377 | 377 |
| 378 | 378 |
| 379 def yield_swarming_bot_files( | 379 def yield_swarming_bot_files( |
| 380 root_dir, host, host_version, additionals, enable_ts_monitoring): | 380 root_dir, host, host_version, additionals, settings): |
| 381 """Yields all the files to map as tuple(filename, content). | 381 """Yields all the files to map as tuple(filename, content). |
| 382 | 382 |
| 383 config.json is injected with json data about the server. | 383 config.json is injected with json data about the server. |
| 384 | 384 |
| 385 This function guarantees that the output is sorted by filename. | 385 This function guarantees that the output is sorted by filename. |
| 386 """ | 386 """ |
| 387 grpc_prefix = 'grpc://' | |
| 388 is_grpc = host.startswith(grpc_prefix) | |
| 389 if is_grpc: | |
| 390 host = host[len(grpc_prefix):] | |
| 391 | |
| 392 items = {i: None for i in FILES} | 387 items = {i: None for i in FILES} |
| 393 items.update(additionals) | 388 items.update(additionals) |
| 394 config = { | 389 config = { |
| 395 'enable_ts_monitoring': enable_ts_monitoring, | |
| 396 'is_grpc': is_grpc, | |
| 397 'server': host.rstrip('/'), | 390 'server': host.rstrip('/'), |
| 398 'server_version': host_version, | 391 'server_version': host_version, |
| 399 } | 392 } |
| 393 |
| 394 if settings: |
| 395 config['enable_ts_monitoring'] = settings.enable_ts_monitoring |
| 396 config['isolate_grpc_proxy'] = settings.bot_isolate_grpc_proxy |
| 397 |
| 400 items['config/config.json'] = json.dumps(config) | 398 items['config/config.json'] = json.dumps(config) |
| 399 logging.debug('Bot config.json: %s', items['config/config.json']) |
| 401 for item, content in sorted(items.iteritems()): | 400 for item, content in sorted(items.iteritems()): |
| 402 if content is not None: | 401 if content is not None: |
| 403 yield item, content | 402 yield item, content |
| 404 else: | 403 else: |
| 405 with open(resolve_symlink(os.path.join(root_dir, item)), 'rb') as f: | 404 with open(resolve_symlink(os.path.join(root_dir, item)), 'rb') as f: |
| 406 yield item, f.read() | 405 yield item, f.read() |
| 407 | 406 |
| 408 | 407 |
| 409 def get_swarming_bot_zip( | 408 def get_swarming_bot_zip( |
| 410 root_dir, host, host_version, additionals, enable_ts_monitoring): | 409 root_dir, host, host_version, additionals, settings): |
| 411 """Returns a zipped file of all the files a bot needs to run. | 410 """Returns a zipped file of all the files a bot needs to run. |
| 412 | 411 |
| 413 Arguments: | 412 Arguments: |
| 414 root_dir: directory swarming_bot. | 413 root_dir: directory swarming_bot. |
| 415 additionals: dict(filepath: content) of additional items to put into the zip | 414 additionals: dict(filepath: content) of additional items to put into the zip |
| 416 file, in addition to FILES and MAPPED. In practice, it's going to be a | 415 file, in addition to FILES and MAPPED. In practice, it's going to be a |
| 417 custom bot_config.py. | 416 custom bot_config.py. |
| 418 enable_ts_monitoring: bool if ts_mon should be enabled on the bot. | 417 enable_ts_monitoring: bool if ts_mon should be enabled on the bot. |
| 419 | 418 |
| 420 Returns: | 419 Returns: |
| 421 Tuple(str being the zipped file's content, bot version (SHA256) it | 420 Tuple(str being the zipped file's content, bot version (SHA256) it |
| 422 represents). | 421 represents). |
| 423 """ | 422 """ |
| 424 zip_memory_file = StringIO.StringIO() | 423 zip_memory_file = StringIO.StringIO() |
| 425 h = hashlib.sha256() | 424 h = hashlib.sha256() |
| 426 with zipfile.ZipFile(zip_memory_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: | 425 with zipfile.ZipFile(zip_memory_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: |
| 427 for name, content in yield_swarming_bot_files( | 426 for name, content in yield_swarming_bot_files( |
| 428 root_dir, host, host_version, additionals, enable_ts_monitoring): | 427 root_dir, host, host_version, additionals, settings): |
| 429 zip_file.writestr(name, content) | 428 zip_file.writestr(name, content) |
| 430 h.update(str(len(name))) | 429 h.update(str(len(name))) |
| 431 h.update(name) | 430 h.update(name) |
| 432 h.update(str(len(content))) | 431 h.update(str(len(content))) |
| 433 h.update(content) | 432 h.update(content) |
| 434 | 433 |
| 435 data = zip_memory_file.getvalue() | 434 data = zip_memory_file.getvalue() |
| 436 bot_version = h.hexdigest() | 435 bot_version = h.hexdigest() |
| 437 logging.info( | 436 logging.info( |
| 438 'get_swarming_bot_zip(%s) is %d bytes; %s', | 437 'get_swarming_bot_zip(%s) is %d bytes; %s', |
| 439 additionals.keys(), len(data), bot_version) | 438 additionals.keys(), len(data), bot_version) |
| 440 return data, bot_version | 439 return data, bot_version |
| 441 | 440 |
| 442 | 441 |
| 443 def get_swarming_bot_version( | 442 def get_swarming_bot_version( |
| 444 root_dir, host, host_version, additionals, enable_ts_monitoring): | 443 root_dir, host, host_version, additionals, settings): |
| 445 """Returns the SHA256 hash of the bot code, representing the version. | 444 """Returns the SHA256 hash of the bot code, representing the version. |
| 446 | 445 |
| 447 Arguments: | 446 Arguments: |
| 448 root_dir: directory swarming_bot. | 447 root_dir: directory swarming_bot. |
| 449 additionals: See get_swarming_bot_zip's doc. | 448 additionals: See get_swarming_bot_zip's doc. |
| 450 enable_ts_monitoring: bool if ts_mon should be enabled on the bot. | 449 enable_ts_monitoring: bool if ts_mon should be enabled on the bot. |
| 451 | 450 |
| 452 Returns: | 451 Returns: |
| 453 The SHA256 hash of the bot code. | 452 The SHA256 hash of the bot code. |
| 454 """ | 453 """ |
| 455 h = hashlib.sha256() | 454 h = hashlib.sha256() |
| 456 try: | 455 try: |
| 457 # TODO(maruel): Deduplicate from zip_package.genereate_version(). | 456 # TODO(maruel): Deduplicate from zip_package.genereate_version(). |
| 458 for name, content in yield_swarming_bot_files( | 457 for name, content in yield_swarming_bot_files( |
| 459 root_dir, host, host_version, additionals, enable_ts_monitoring): | 458 root_dir, host, host_version, additionals, settings): |
| 460 h.update(str(len(name))) | 459 h.update(str(len(name))) |
| 461 h.update(name) | 460 h.update(name) |
| 462 h.update(str(len(content))) | 461 h.update(str(len(content))) |
| 463 h.update(content) | 462 h.update(content) |
| 464 except IOError: | 463 except IOError: |
| 465 logging.warning('Missing expected file. Hash will be invalid.') | 464 logging.warning('Missing expected file. Hash will be invalid.') |
| 466 bot_version = h.hexdigest() | 465 bot_version = h.hexdigest() |
| 467 logging.info( | 466 logging.info( |
| 468 'get_swarming_bot_version(%s) = %s', sorted(additionals), bot_version) | 467 'get_swarming_bot_version(%s) = %s', sorted(additionals), bot_version) |
| 469 return bot_version | 468 return bot_version |
| OLD | NEW |