| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Archives or replays webpages and creates SKPs in a Google Storage location. | 6 """Archives or replays webpages and creates SKPs in a Google Storage location. |
| 7 | 7 |
| 8 To archive webpages and store SKP files (archives should be rarely updated): | 8 To archive webpages and store SKP files (archives should be rarely updated): |
| 9 | 9 |
| 10 cd ../buildbot/slave/skia_slave_scripts | 10 cd ../buildbot/slave/skia_slave_scripts |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 62 # and site_config. | 62 # and site_config. |
| 63 BUILDBOT_PATH = os.path.realpath(os.path.join( | 63 BUILDBOT_PATH = os.path.realpath(os.path.join( |
| 64 os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) | 64 os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) |
| 65 sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) | 65 sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) |
| 66 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', | 66 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', |
| 67 'scripts')) | 67 'scripts')) |
| 68 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', | 68 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', |
| 69 'site_config')) | 69 'site_config')) |
| 70 | 70 |
| 71 from utils import file_utils | 71 from utils import file_utils |
| 72 from utils import old_gs_utils as gs_utils | 72 from utils import gs_utils |
| 73 from utils import old_gs_utils |
| 73 from py.utils import misc | 74 from py.utils import misc |
| 74 from py.utils import shell_utils | 75 from py.utils import shell_utils |
| 75 | 76 |
| 76 from slave import slave_utils | 77 from slave import slave_utils |
| 77 | 78 |
| 78 from build_step import PLAYBACK_CANNED_ACL | |
| 79 from playback_dirs import ROOT_PLAYBACK_DIR_NAME | 79 from playback_dirs import ROOT_PLAYBACK_DIR_NAME |
| 80 from playback_dirs import SKPICTURES_DIR_NAME | 80 from playback_dirs import SKPICTURES_DIR_NAME |
| 81 | 81 |
| 82 | 82 |
| 83 # Local archive and SKP directories. | 83 # Local archive and SKP directories. |
| 84 LOCAL_PLAYBACK_ROOT_DIR = os.path.join( | 84 LOCAL_PLAYBACK_ROOT_DIR = os.path.join( |
| 85 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME) | 85 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME) |
| 86 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join( | 86 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join( |
| 87 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data') | 87 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data') |
| 88 TMP_SKP_DIR = tempfile.mkdtemp() | 88 TMP_SKP_DIR = tempfile.mkdtemp() |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 167 elif '*' in page_sets: | 167 elif '*' in page_sets: |
| 168 # Explode and return the glob. | 168 # Explode and return the glob. |
| 169 return glob.glob(page_sets) | 169 return glob.glob(page_sets) |
| 170 else: | 170 else: |
| 171 return page_sets.split(',') | 171 return page_sets.split(',') |
| 172 | 172 |
| 173 def Run(self): | 173 def Run(self): |
| 174 """Run the SkPicturePlayback BuildStep.""" | 174 """Run the SkPicturePlayback BuildStep.""" |
| 175 | 175 |
| 176 # Ensure the right .boto file is used by gsutil. | 176 # Ensure the right .boto file is used by gsutil. |
| 177 if not self._skip_all_gs_access and gs_utils.read_timestamp_file( | 177 if not self._skip_all_gs_access and old_gs_utils.read_timestamp_file( |
| 178 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 178 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
| 179 gs_base=self._dest_gsbase, | 179 gs_base=self._dest_gsbase, |
| 180 gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME, | 180 gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME, |
| 181 SKPICTURES_DIR_NAME)) == "0": | 181 SKPICTURES_DIR_NAME)) == "0": |
| 182 raise Exception( | 182 raise Exception( |
| 183 'Missing .boto file or .boto does not have the right credentials.' | 183 'Missing .boto file or .boto does not have the right credentials.' |
| 184 'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q' | 184 'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q' |
| 185 'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit ' | 185 'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit ' |
| 186 '(may have to request access). The .boto file will need to be placed ' | 186 '(may have to request access). The .boto file will need to be placed ' |
| 187 'under third_party/chromium_buildbot/site_config/') | 187 'under third_party/chromium_buildbot/site_config/') |
| 188 | 188 |
| (...skipping 150 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 339 if not self._skip_all_gs_access and self._upload_to_gs: | 339 if not self._skip_all_gs_access and self._upload_to_gs: |
| 340 print '\n\n=======Uploading to Google Storage=======\n\n' | 340 print '\n\n=======Uploading to Google Storage=======\n\n' |
| 341 # Copy the directory structure in the root directory into Google Storage. | 341 # Copy the directory structure in the root directory into Google Storage. |
| 342 dest_dir_name = ROOT_PLAYBACK_DIR_NAME | 342 dest_dir_name = ROOT_PLAYBACK_DIR_NAME |
| 343 if self._alternate_upload_dir: | 343 if self._alternate_upload_dir: |
| 344 dest_dir_name = self._alternate_upload_dir | 344 dest_dir_name = self._alternate_upload_dir |
| 345 gs_status = slave_utils.GSUtilCopyDir( | 345 gs_status = slave_utils.GSUtilCopyDir( |
| 346 src_dir=LOCAL_PLAYBACK_ROOT_DIR, | 346 src_dir=LOCAL_PLAYBACK_ROOT_DIR, |
| 347 gs_base=self._dest_gsbase, | 347 gs_base=self._dest_gsbase, |
| 348 dest_dir=dest_dir_name, | 348 dest_dir=dest_dir_name, |
| 349 gs_acl=PLAYBACK_CANNED_ACL) | 349 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL) |
| 350 if gs_status != 0: | 350 if gs_status != 0: |
| 351 raise Exception( | 351 raise Exception( |
| 352 'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % ( | 352 'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % ( |
| 353 gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase, | 353 gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase, |
| 354 ROOT_PLAYBACK_DIR_NAME)) | 354 ROOT_PLAYBACK_DIR_NAME)) |
| 355 self._SetGoogleReadACLs( | 355 self._SetGoogleReadACLs( |
| 356 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) | 356 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) |
| 357 | 357 |
| 358 # Add a timestamp file to the SKP directory in Google Storage so we can | 358 # Add a timestamp file to the SKP directory in Google Storage so we can |
| 359 # use directory level rsync like functionality. | 359 # use directory level rsync like functionality. |
| 360 gs_utils.write_timestamp_file( | 360 old_gs_utils.write_timestamp_file( |
| 361 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 361 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
| 362 timestamp_value=time.time(), | 362 timestamp_value=time.time(), |
| 363 gs_base=self._dest_gsbase, | 363 gs_base=self._dest_gsbase, |
| 364 gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME), | 364 gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME), |
| 365 gs_acl=PLAYBACK_CANNED_ACL, | 365 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL, |
| 366 local_dir=LOCAL_PLAYBACK_ROOT_DIR) | 366 local_dir=LOCAL_PLAYBACK_ROOT_DIR) |
| 367 | 367 |
| 368 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( | 368 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( |
| 369 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) | 369 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) |
| 370 else: | 370 else: |
| 371 print '\n\n=======Not Uploading to Google Storage=======\n\n' | 371 print '\n\n=======Not Uploading to Google Storage=======\n\n' |
| 372 print 'Generated resources are available in %s\n\n' % ( | 372 print 'Generated resources are available in %s\n\n' % ( |
| 373 LOCAL_PLAYBACK_ROOT_DIR) | 373 LOCAL_PLAYBACK_ROOT_DIR) |
| 374 | 374 |
| 375 return 0 | 375 return 0 |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 422 file_utils.create_clean_local_dir(self._local_skp_dir) | 422 file_utils.create_clean_local_dir(self._local_skp_dir) |
| 423 | 423 |
| 424 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename): | 424 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename): |
| 425 """Downloads the webpages archive and its required page set from GS.""" | 425 """Downloads the webpages archive and its required page set from GS.""" |
| 426 wpr_source = posixpath.join( | 426 wpr_source = posixpath.join( |
| 427 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', | 427 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', |
| 428 wpr_data_file) | 428 wpr_data_file) |
| 429 page_set_source = posixpath.join( | 429 page_set_source = posixpath.join( |
| 430 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', | 430 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', |
| 431 page_set_basename) | 431 page_set_basename) |
| 432 if (gs_utils.does_storage_object_exist(wpr_source) and | 432 if (old_gs_utils.does_storage_object_exist(wpr_source) and |
| 433 gs_utils.does_storage_object_exist(page_set_source)): | 433 old_gs_utils.does_storage_object_exist(page_set_source)): |
| 434 slave_utils.GSUtilDownloadFile( | 434 slave_utils.GSUtilDownloadFile( |
| 435 src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) | 435 src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) |
| 436 slave_utils.GSUtilDownloadFile( | 436 slave_utils.GSUtilDownloadFile( |
| 437 src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) | 437 src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) |
| 438 else: | 438 else: |
| 439 raise Exception('%s and %s do not exist in Google Storage!' % ( | 439 raise Exception('%s and %s do not exist in Google Storage!' % ( |
| 440 wpr_source, page_set_source)) | 440 wpr_source, page_set_source)) |
| 441 | 441 |
| 442 | 442 |
| 443 if '__main__' == __name__: | 443 if '__main__' == __name__: |
| 444 option_parser = optparse.OptionParser() | 444 option_parser = optparse.OptionParser() |
| 445 option_parser.add_option( | 445 option_parser.add_option( |
| 446 '', '--page_sets', | 446 '', '--page_sets', |
| 447 help='Specifies the page sets to use to archive. Supports globs.', | 447 help='Specifies the page sets to use to archive. Supports globs.', |
| 448 default='all') | 448 default='all') |
| 449 option_parser.add_option( | 449 option_parser.add_option( |
| 450 '', '--skip_all_gs_access', action='store_true', | 450 '', '--skip_all_gs_access', action='store_true', |
| 451 help='All Google Storage interactions will be skipped if this flag is ' | 451 help='All Google Storage interactions will be skipped if this flag is ' |
| 452 'specified. This is useful for cases where the user does not have ' | 452 'specified. This is useful for cases where the user does not have ' |
| 453 'the required .boto file but would like to generate webpage ' | 453 'the required .boto file but would like to generate webpage ' |
| 454 'archives and SKPs from the Skia page sets.', | 454 'archives and SKPs from the Skia page sets.', |
| 455 default=False) | 455 default=False) |
| 456 option_parser.add_option( | 456 option_parser.add_option( |
| 457 '', '--record', action='store_true', | 457 '', '--record', action='store_true', |
| 458 help='Specifies whether a new website archive should be created.', | 458 help='Specifies whether a new website archive should be created.', |
| 459 default=False) | 459 default=False) |
| 460 option_parser.add_option( | 460 option_parser.add_option( |
| 461 '', '--dest_gsbase', | 461 '', '--dest_gsbase', |
| 462 help='gs:// bucket_name, the bucket to upload the file to.', | 462 help='gs:// bucket_name, the bucket to upload the file to.', |
| 463 default=gs_utils.DEFAULT_DEST_GSBASE) | 463 default=old_gs_utils.DEFAULT_DEST_GSBASE) |
| 464 option_parser.add_option( | 464 option_parser.add_option( |
| 465 '', '--skia_tools', | 465 '', '--skia_tools', |
| 466 help=('Path to compiled Skia executable tools. ' | 466 help=('Path to compiled Skia executable tools. ' |
| 467 'render_pictures/render_pdfs/bench_pictures is run on the set ' | 467 'render_pictures/render_pdfs/bench_pictures is run on the set ' |
| 468 'after all SKPs are captured. If the script is run without ' | 468 'after all SKPs are captured. If the script is run without ' |
| 469 '--non-interactive then the debugger is also run at the end. Debug ' | 469 '--non-interactive then the debugger is also run at the end. Debug ' |
| 470 'builds are recommended because they seem to catch more failures ' | 470 'builds are recommended because they seem to catch more failures ' |
| 471 'than Release builds.'), | 471 'than Release builds.'), |
| 472 default=None) | 472 default=None) |
| 473 option_parser.add_option( | 473 option_parser.add_option( |
| (...skipping 15 matching lines...) Expand all Loading... |
| 489 default=None) | 489 default=None) |
| 490 option_parser.add_option( | 490 option_parser.add_option( |
| 491 '', '--non-interactive', action='store_true', | 491 '', '--non-interactive', action='store_true', |
| 492 help='Runs the script without any prompts. If this flag is specified and ' | 492 help='Runs the script without any prompts. If this flag is specified and ' |
| 493 '--skia_tools is specified then the debugger is not run.', | 493 '--skia_tools is specified then the debugger is not run.', |
| 494 default=False) | 494 default=False) |
| 495 options, unused_args = option_parser.parse_args() | 495 options, unused_args = option_parser.parse_args() |
| 496 | 496 |
| 497 playback = SkPicturePlayback(options) | 497 playback = SkPicturePlayback(options) |
| 498 sys.exit(playback.Run()) | 498 sys.exit(playback.Run()) |
| OLD | NEW |