Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(465)

Side by Side Diff: appengine/swarming/server/bot_archive.py

Issue 2969513002: Add a default Isolate gRPC proxy in config (Closed)
Patch Set: Fix problem in tests Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « appengine/swarming/proto/config_pb2.py ('k') | appengine/swarming/server/bot_code.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2014 The LUCI Authors. All rights reserved. 1 # Copyright 2014 The LUCI Authors. All rights reserved.
2 # Use of this source code is governed under the Apache License, Version 2.0 2 # Use of this source code is governed under the Apache License, Version 2.0
3 # that can be found in the LICENSE file. 3 # that can be found in the LICENSE file.
4 4
5 """Generates the swarming_bot.zip archive for the bot. 5 """Generates the swarming_bot.zip archive for the bot.
6 6
7 Unlike the other source files, this file can be run from ../tools/bot_archive.py 7 Unlike the other source files, this file can be run from ../tools/bot_archive.py
8 stand-alone to generate a swarming_bot.zip for local testing so it doesn't 8 stand-alone to generate a swarming_bot.zip for local testing so it doesn't
9 import anything from the AppEngine SDK. 9 import anything from the AppEngine SDK.
10 10
(...skipping 359 matching lines...) Expand 10 before | Expand all | Expand 10 after
370 partial = os.path.sep.join(parts[:i]) 370 partial = os.path.sep.join(parts[:i])
371 if os.path.isfile(partial): 371 if os.path.isfile(partial):
372 with open(partial) as f: 372 with open(partial) as f:
373 link = f.read() 373 link = f.read()
374 assert '\n' not in link and link, link 374 assert '\n' not in link and link, link
375 parts[i-1] = link 375 parts[i-1] = link
376 return os.path.normpath(os.path.sep.join(parts)) 376 return os.path.normpath(os.path.sep.join(parts))
377 377
378 378
379 def yield_swarming_bot_files( 379 def yield_swarming_bot_files(
380 root_dir, host, host_version, additionals, enable_ts_monitoring): 380 root_dir, host, host_version, additionals, settings):
381 """Yields all the files to map as tuple(filename, content). 381 """Yields all the files to map as tuple(filename, content).
382 382
383 config.json is injected with json data about the server. 383 config.json is injected with json data about the server.
384 384
385 This function guarantees that the output is sorted by filename. 385 This function guarantees that the output is sorted by filename.
386 """ 386 """
387 grpc_prefix = 'grpc://'
388 is_grpc = host.startswith(grpc_prefix)
389 if is_grpc:
390 host = host[len(grpc_prefix):]
391
392 items = {i: None for i in FILES} 387 items = {i: None for i in FILES}
393 items.update(additionals) 388 items.update(additionals)
394 config = { 389 config = {
395 'enable_ts_monitoring': enable_ts_monitoring,
396 'is_grpc': is_grpc,
397 'server': host.rstrip('/'), 390 'server': host.rstrip('/'),
398 'server_version': host_version, 391 'server_version': host_version,
399 } 392 }
393
394 if settings:
395 config['enable_ts_monitoring'] = settings.enable_ts_monitoring
396 config['isolate_grpc_proxy'] = settings.bot_isolate_grpc_proxy
397
400 items['config/config.json'] = json.dumps(config) 398 items['config/config.json'] = json.dumps(config)
399 logging.debug('Bot config.json: %s', items['config/config.json'])
401 for item, content in sorted(items.iteritems()): 400 for item, content in sorted(items.iteritems()):
402 if content is not None: 401 if content is not None:
403 yield item, content 402 yield item, content
404 else: 403 else:
405 with open(resolve_symlink(os.path.join(root_dir, item)), 'rb') as f: 404 with open(resolve_symlink(os.path.join(root_dir, item)), 'rb') as f:
406 yield item, f.read() 405 yield item, f.read()
407 406
408 407
409 def get_swarming_bot_zip( 408 def get_swarming_bot_zip(
410 root_dir, host, host_version, additionals, enable_ts_monitoring): 409 root_dir, host, host_version, additionals, settings):
411 """Returns a zipped file of all the files a bot needs to run. 410 """Returns a zipped file of all the files a bot needs to run.
412 411
413 Arguments: 412 Arguments:
414 root_dir: directory swarming_bot. 413 root_dir: directory swarming_bot.
415 additionals: dict(filepath: content) of additional items to put into the zip 414 additionals: dict(filepath: content) of additional items to put into the zip
416 file, in addition to FILES and MAPPED. In practice, it's going to be a 415 file, in addition to FILES and MAPPED. In practice, it's going to be a
417 custom bot_config.py. 416 custom bot_config.py.
418 enable_ts_monitoring: bool if ts_mon should be enabled on the bot. 417 enable_ts_monitoring: bool if ts_mon should be enabled on the bot.
419 418
420 Returns: 419 Returns:
421 Tuple(str being the zipped file's content, bot version (SHA256) it 420 Tuple(str being the zipped file's content, bot version (SHA256) it
422 represents). 421 represents).
423 """ 422 """
424 zip_memory_file = StringIO.StringIO() 423 zip_memory_file = StringIO.StringIO()
425 h = hashlib.sha256() 424 h = hashlib.sha256()
426 with zipfile.ZipFile(zip_memory_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: 425 with zipfile.ZipFile(zip_memory_file, 'w', zipfile.ZIP_DEFLATED) as zip_file:
427 for name, content in yield_swarming_bot_files( 426 for name, content in yield_swarming_bot_files(
428 root_dir, host, host_version, additionals, enable_ts_monitoring): 427 root_dir, host, host_version, additionals, settings):
429 zip_file.writestr(name, content) 428 zip_file.writestr(name, content)
430 h.update(str(len(name))) 429 h.update(str(len(name)))
431 h.update(name) 430 h.update(name)
432 h.update(str(len(content))) 431 h.update(str(len(content)))
433 h.update(content) 432 h.update(content)
434 433
435 data = zip_memory_file.getvalue() 434 data = zip_memory_file.getvalue()
436 bot_version = h.hexdigest() 435 bot_version = h.hexdigest()
437 logging.info( 436 logging.info(
438 'get_swarming_bot_zip(%s) is %d bytes; %s', 437 'get_swarming_bot_zip(%s) is %d bytes; %s',
439 additionals.keys(), len(data), bot_version) 438 additionals.keys(), len(data), bot_version)
440 return data, bot_version 439 return data, bot_version
441 440
442 441
443 def get_swarming_bot_version( 442 def get_swarming_bot_version(
444 root_dir, host, host_version, additionals, enable_ts_monitoring): 443 root_dir, host, host_version, additionals, settings):
445 """Returns the SHA256 hash of the bot code, representing the version. 444 """Returns the SHA256 hash of the bot code, representing the version.
446 445
447 Arguments: 446 Arguments:
448 root_dir: directory swarming_bot. 447 root_dir: directory swarming_bot.
449 additionals: See get_swarming_bot_zip's doc. 448 additionals: See get_swarming_bot_zip's doc.
450 enable_ts_monitoring: bool if ts_mon should be enabled on the bot. 449 enable_ts_monitoring: bool if ts_mon should be enabled on the bot.
451 450
452 Returns: 451 Returns:
453 The SHA256 hash of the bot code. 452 The SHA256 hash of the bot code.
454 """ 453 """
455 h = hashlib.sha256() 454 h = hashlib.sha256()
456 try: 455 try:
457 # TODO(maruel): Deduplicate from zip_package.genereate_version(). 456 # TODO(maruel): Deduplicate from zip_package.genereate_version().
458 for name, content in yield_swarming_bot_files( 457 for name, content in yield_swarming_bot_files(
459 root_dir, host, host_version, additionals, enable_ts_monitoring): 458 root_dir, host, host_version, additionals, settings):
460 h.update(str(len(name))) 459 h.update(str(len(name)))
461 h.update(name) 460 h.update(name)
462 h.update(str(len(content))) 461 h.update(str(len(content)))
463 h.update(content) 462 h.update(content)
464 except IOError: 463 except IOError:
465 logging.warning('Missing expected file. Hash will be invalid.') 464 logging.warning('Missing expected file. Hash will be invalid.')
466 bot_version = h.hexdigest() 465 bot_version = h.hexdigest()
467 logging.info( 466 logging.info(
468 'get_swarming_bot_version(%s) = %s', sorted(additionals), bot_version) 467 'get_swarming_bot_version(%s) = %s', sorted(additionals), bot_version)
469 return bot_version 468 return bot_version
OLDNEW
« no previous file with comments | « appengine/swarming/proto/config_pb2.py ('k') | appengine/swarming/server/bot_code.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698