OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Archives or replays webpages and creates SKPs in a Google Storage location. | 6 """Archives or replays webpages and creates SKPs in a Google Storage location. |
7 | 7 |
8 To archive webpages and store SKP files (archives should be rarely updated): | 8 To archive webpages and store SKP files (archives should be rarely updated): |
9 | 9 |
10 cd ../buildbot/slave/skia_slave_scripts | 10 cd ../buildbot/slave/skia_slave_scripts |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
62 # and site_config. | 62 # and site_config. |
63 BUILDBOT_PATH = os.path.realpath(os.path.join( | 63 BUILDBOT_PATH = os.path.realpath(os.path.join( |
64 os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) | 64 os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) |
65 sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) | 65 sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) |
66 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', | 66 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', |
67 'scripts')) | 67 'scripts')) |
68 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', | 68 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', |
69 'site_config')) | 69 'site_config')) |
70 | 70 |
71 from utils import file_utils | 71 from utils import file_utils |
72 from utils import old_gs_utils as gs_utils | 72 from utils import gs_utils |
| 73 from utils import old_gs_utils |
73 from py.utils import misc | 74 from py.utils import misc |
74 from py.utils import shell_utils | 75 from py.utils import shell_utils |
75 | 76 |
76 from slave import slave_utils | 77 from slave import slave_utils |
77 | 78 |
78 from build_step import PLAYBACK_CANNED_ACL | |
79 from playback_dirs import ROOT_PLAYBACK_DIR_NAME | 79 from playback_dirs import ROOT_PLAYBACK_DIR_NAME |
80 from playback_dirs import SKPICTURES_DIR_NAME | 80 from playback_dirs import SKPICTURES_DIR_NAME |
81 | 81 |
82 | 82 |
83 # Local archive and SKP directories. | 83 # Local archive and SKP directories. |
84 LOCAL_PLAYBACK_ROOT_DIR = os.path.join( | 84 LOCAL_PLAYBACK_ROOT_DIR = os.path.join( |
85 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME) | 85 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME) |
86 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join( | 86 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join( |
87 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data') | 87 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data') |
88 TMP_SKP_DIR = tempfile.mkdtemp() | 88 TMP_SKP_DIR = tempfile.mkdtemp() |
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
167 elif '*' in page_sets: | 167 elif '*' in page_sets: |
168 # Explode and return the glob. | 168 # Explode and return the glob. |
169 return glob.glob(page_sets) | 169 return glob.glob(page_sets) |
170 else: | 170 else: |
171 return page_sets.split(',') | 171 return page_sets.split(',') |
172 | 172 |
173 def Run(self): | 173 def Run(self): |
174 """Run the SkPicturePlayback BuildStep.""" | 174 """Run the SkPicturePlayback BuildStep.""" |
175 | 175 |
176 # Ensure the right .boto file is used by gsutil. | 176 # Ensure the right .boto file is used by gsutil. |
177 if not self._skip_all_gs_access and gs_utils.read_timestamp_file( | 177 if not self._skip_all_gs_access and old_gs_utils.read_timestamp_file( |
178 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 178 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
179 gs_base=self._dest_gsbase, | 179 gs_base=self._dest_gsbase, |
180 gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME, | 180 gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME, |
181 SKPICTURES_DIR_NAME)) == "0": | 181 SKPICTURES_DIR_NAME)) == "0": |
182 raise Exception( | 182 raise Exception( |
183 'Missing .boto file or .boto does not have the right credentials.' | 183 'Missing .boto file or .boto does not have the right credentials.' |
184 'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q' | 184 'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q' |
185 'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit ' | 185 'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit ' |
186 '(may have to request access). The .boto file will need to be placed ' | 186 '(may have to request access). The .boto file will need to be placed ' |
187 'under third_party/chromium_buildbot/site_config/') | 187 'under third_party/chromium_buildbot/site_config/') |
188 | 188 |
(...skipping 150 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
339 if not self._skip_all_gs_access and self._upload_to_gs: | 339 if not self._skip_all_gs_access and self._upload_to_gs: |
340 print '\n\n=======Uploading to Google Storage=======\n\n' | 340 print '\n\n=======Uploading to Google Storage=======\n\n' |
341 # Copy the directory structure in the root directory into Google Storage. | 341 # Copy the directory structure in the root directory into Google Storage. |
342 dest_dir_name = ROOT_PLAYBACK_DIR_NAME | 342 dest_dir_name = ROOT_PLAYBACK_DIR_NAME |
343 if self._alternate_upload_dir: | 343 if self._alternate_upload_dir: |
344 dest_dir_name = self._alternate_upload_dir | 344 dest_dir_name = self._alternate_upload_dir |
345 gs_status = slave_utils.GSUtilCopyDir( | 345 gs_status = slave_utils.GSUtilCopyDir( |
346 src_dir=LOCAL_PLAYBACK_ROOT_DIR, | 346 src_dir=LOCAL_PLAYBACK_ROOT_DIR, |
347 gs_base=self._dest_gsbase, | 347 gs_base=self._dest_gsbase, |
348 dest_dir=dest_dir_name, | 348 dest_dir=dest_dir_name, |
349 gs_acl=PLAYBACK_CANNED_ACL) | 349 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL) |
350 if gs_status != 0: | 350 if gs_status != 0: |
351 raise Exception( | 351 raise Exception( |
352 'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % ( | 352 'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % ( |
353 gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase, | 353 gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase, |
354 ROOT_PLAYBACK_DIR_NAME)) | 354 ROOT_PLAYBACK_DIR_NAME)) |
355 self._SetGoogleReadACLs( | 355 self._SetGoogleReadACLs( |
356 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) | 356 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) |
357 | 357 |
358 # Add a timestamp file to the SKP directory in Google Storage so we can | 358 # Add a timestamp file to the SKP directory in Google Storage so we can |
359 # use directory level rsync like functionality. | 359 # use directory level rsync like functionality. |
360 gs_utils.write_timestamp_file( | 360 old_gs_utils.write_timestamp_file( |
361 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 361 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
362 timestamp_value=time.time(), | 362 timestamp_value=time.time(), |
363 gs_base=self._dest_gsbase, | 363 gs_base=self._dest_gsbase, |
364 gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME), | 364 gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME), |
365 gs_acl=PLAYBACK_CANNED_ACL, | 365 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL, |
366 local_dir=LOCAL_PLAYBACK_ROOT_DIR) | 366 local_dir=LOCAL_PLAYBACK_ROOT_DIR) |
367 | 367 |
368 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( | 368 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( |
369 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) | 369 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) |
370 else: | 370 else: |
371 print '\n\n=======Not Uploading to Google Storage=======\n\n' | 371 print '\n\n=======Not Uploading to Google Storage=======\n\n' |
372 print 'Generated resources are available in %s\n\n' % ( | 372 print 'Generated resources are available in %s\n\n' % ( |
373 LOCAL_PLAYBACK_ROOT_DIR) | 373 LOCAL_PLAYBACK_ROOT_DIR) |
374 | 374 |
375 return 0 | 375 return 0 |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
422 file_utils.create_clean_local_dir(self._local_skp_dir) | 422 file_utils.create_clean_local_dir(self._local_skp_dir) |
423 | 423 |
424 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename): | 424 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename): |
425 """Downloads the webpages archive and its required page set from GS.""" | 425 """Downloads the webpages archive and its required page set from GS.""" |
426 wpr_source = posixpath.join( | 426 wpr_source = posixpath.join( |
427 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', | 427 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', |
428 wpr_data_file) | 428 wpr_data_file) |
429 page_set_source = posixpath.join( | 429 page_set_source = posixpath.join( |
430 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', | 430 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', |
431 page_set_basename) | 431 page_set_basename) |
432 if (gs_utils.does_storage_object_exist(wpr_source) and | 432 if (old_gs_utils.does_storage_object_exist(wpr_source) and |
433 gs_utils.does_storage_object_exist(page_set_source)): | 433 old_gs_utils.does_storage_object_exist(page_set_source)): |
434 slave_utils.GSUtilDownloadFile( | 434 slave_utils.GSUtilDownloadFile( |
435 src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) | 435 src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) |
436 slave_utils.GSUtilDownloadFile( | 436 slave_utils.GSUtilDownloadFile( |
437 src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) | 437 src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) |
438 else: | 438 else: |
439 raise Exception('%s and %s do not exist in Google Storage!' % ( | 439 raise Exception('%s and %s do not exist in Google Storage!' % ( |
440 wpr_source, page_set_source)) | 440 wpr_source, page_set_source)) |
441 | 441 |
442 | 442 |
443 if '__main__' == __name__: | 443 if '__main__' == __name__: |
444 option_parser = optparse.OptionParser() | 444 option_parser = optparse.OptionParser() |
445 option_parser.add_option( | 445 option_parser.add_option( |
446 '', '--page_sets', | 446 '', '--page_sets', |
447 help='Specifies the page sets to use to archive. Supports globs.', | 447 help='Specifies the page sets to use to archive. Supports globs.', |
448 default='all') | 448 default='all') |
449 option_parser.add_option( | 449 option_parser.add_option( |
450 '', '--skip_all_gs_access', action='store_true', | 450 '', '--skip_all_gs_access', action='store_true', |
451 help='All Google Storage interactions will be skipped if this flag is ' | 451 help='All Google Storage interactions will be skipped if this flag is ' |
452 'specified. This is useful for cases where the user does not have ' | 452 'specified. This is useful for cases where the user does not have ' |
453 'the required .boto file but would like to generate webpage ' | 453 'the required .boto file but would like to generate webpage ' |
454 'archives and SKPs from the Skia page sets.', | 454 'archives and SKPs from the Skia page sets.', |
455 default=False) | 455 default=False) |
456 option_parser.add_option( | 456 option_parser.add_option( |
457 '', '--record', action='store_true', | 457 '', '--record', action='store_true', |
458 help='Specifies whether a new website archive should be created.', | 458 help='Specifies whether a new website archive should be created.', |
459 default=False) | 459 default=False) |
460 option_parser.add_option( | 460 option_parser.add_option( |
461 '', '--dest_gsbase', | 461 '', '--dest_gsbase', |
462 help='gs:// bucket_name, the bucket to upload the file to.', | 462 help='gs:// bucket_name, the bucket to upload the file to.', |
463 default=gs_utils.DEFAULT_DEST_GSBASE) | 463 default=old_gs_utils.DEFAULT_DEST_GSBASE) |
464 option_parser.add_option( | 464 option_parser.add_option( |
465 '', '--skia_tools', | 465 '', '--skia_tools', |
466 help=('Path to compiled Skia executable tools. ' | 466 help=('Path to compiled Skia executable tools. ' |
467 'render_pictures/render_pdfs/bench_pictures is run on the set ' | 467 'render_pictures/render_pdfs/bench_pictures is run on the set ' |
468 'after all SKPs are captured. If the script is run without ' | 468 'after all SKPs are captured. If the script is run without ' |
469 '--non-interactive then the debugger is also run at the end. Debug ' | 469 '--non-interactive then the debugger is also run at the end. Debug ' |
470 'builds are recommended because they seem to catch more failures ' | 470 'builds are recommended because they seem to catch more failures ' |
471 'than Release builds.'), | 471 'than Release builds.'), |
472 default=None) | 472 default=None) |
473 option_parser.add_option( | 473 option_parser.add_option( |
(...skipping 15 matching lines...) Expand all Loading... |
489 default=None) | 489 default=None) |
490 option_parser.add_option( | 490 option_parser.add_option( |
491 '', '--non-interactive', action='store_true', | 491 '', '--non-interactive', action='store_true', |
492 help='Runs the script without any prompts. If this flag is specified and ' | 492 help='Runs the script without any prompts. If this flag is specified and ' |
493 '--skia_tools is specified then the debugger is not run.', | 493 '--skia_tools is specified then the debugger is not run.', |
494 default=False) | 494 default=False) |
495 options, unused_args = option_parser.parse_args() | 495 options, unused_args = option_parser.parse_args() |
496 | 496 |
497 playback = SkPicturePlayback(options) | 497 playback = SkPicturePlayback(options) |
498 sys.exit(playback.Run()) | 498 sys.exit(playback.Run()) |
OLD | NEW |