Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(325)

Side by Side Diff: slave/skia_slave_scripts/webpages_playback.py

Issue 405653004: add ACL-setting code to upload_rendered_skps.py (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: created local gs_utils wrapper module Created 6 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Archives or replays webpages and creates SKPs in a Google Storage location. 6 """Archives or replays webpages and creates SKPs in a Google Storage location.
7 7
8 To archive webpages and store SKP files (archives should be rarely updated): 8 To archive webpages and store SKP files (archives should be rarely updated):
9 9
10 cd ../buildbot/slave/skia_slave_scripts 10 cd ../buildbot/slave/skia_slave_scripts
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after
62 # and site_config. 62 # and site_config.
63 BUILDBOT_PATH = os.path.realpath(os.path.join( 63 BUILDBOT_PATH = os.path.realpath(os.path.join(
64 os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) 64 os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
65 sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) 65 sys.path.append(os.path.join(BUILDBOT_PATH, 'common'))
66 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', 66 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot',
67 'scripts')) 67 'scripts'))
68 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', 68 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot',
69 'site_config')) 69 'site_config'))
70 70
71 from utils import file_utils 71 from utils import file_utils
72 from utils import old_gs_utils as gs_utils 72 from utils import gs_utils
73 from utils import old_gs_utils
73 from py.utils import misc 74 from py.utils import misc
74 from py.utils import shell_utils 75 from py.utils import shell_utils
75 76
76 from slave import slave_utils 77 from slave import slave_utils
77 78
78 from build_step import PLAYBACK_CANNED_ACL
79 from playback_dirs import ROOT_PLAYBACK_DIR_NAME 79 from playback_dirs import ROOT_PLAYBACK_DIR_NAME
80 from playback_dirs import SKPICTURES_DIR_NAME 80 from playback_dirs import SKPICTURES_DIR_NAME
81 81
82 82
83 # Local archive and SKP directories. 83 # Local archive and SKP directories.
84 LOCAL_PLAYBACK_ROOT_DIR = os.path.join( 84 LOCAL_PLAYBACK_ROOT_DIR = os.path.join(
85 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME) 85 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME)
86 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join( 86 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join(
87 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data') 87 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data')
88 TMP_SKP_DIR = tempfile.mkdtemp() 88 TMP_SKP_DIR = tempfile.mkdtemp()
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
167 elif '*' in page_sets: 167 elif '*' in page_sets:
168 # Explode and return the glob. 168 # Explode and return the glob.
169 return glob.glob(page_sets) 169 return glob.glob(page_sets)
170 else: 170 else:
171 return page_sets.split(',') 171 return page_sets.split(',')
172 172
173 def Run(self): 173 def Run(self):
174 """Run the SkPicturePlayback BuildStep.""" 174 """Run the SkPicturePlayback BuildStep."""
175 175
176 # Ensure the right .boto file is used by gsutil. 176 # Ensure the right .boto file is used by gsutil.
177 if not self._skip_all_gs_access and gs_utils.read_timestamp_file( 177 if not self._skip_all_gs_access and old_gs_utils.read_timestamp_file(
178 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, 178 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
179 gs_base=self._dest_gsbase, 179 gs_base=self._dest_gsbase,
180 gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME, 180 gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME,
181 SKPICTURES_DIR_NAME)) == "0": 181 SKPICTURES_DIR_NAME)) == "0":
182 raise Exception( 182 raise Exception(
183 'Missing .boto file or .boto does not have the right credentials.' 183 'Missing .boto file or .boto does not have the right credentials.'
184 'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q' 184 'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q'
185 'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit ' 185 'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit '
186 '(may have to request access). The .boto file will need to be placed ' 186 '(may have to request access). The .boto file will need to be placed '
187 'under third_party/chromium_buildbot/site_config/') 187 'under third_party/chromium_buildbot/site_config/')
188 188
(...skipping 150 matching lines...) Expand 10 before | Expand all | Expand 10 after
339 if not self._skip_all_gs_access and self._upload_to_gs: 339 if not self._skip_all_gs_access and self._upload_to_gs:
340 print '\n\n=======Uploading to Google Storage=======\n\n' 340 print '\n\n=======Uploading to Google Storage=======\n\n'
341 # Copy the directory structure in the root directory into Google Storage. 341 # Copy the directory structure in the root directory into Google Storage.
342 dest_dir_name = ROOT_PLAYBACK_DIR_NAME 342 dest_dir_name = ROOT_PLAYBACK_DIR_NAME
343 if self._alternate_upload_dir: 343 if self._alternate_upload_dir:
344 dest_dir_name = self._alternate_upload_dir 344 dest_dir_name = self._alternate_upload_dir
345 gs_status = slave_utils.GSUtilCopyDir( 345 gs_status = slave_utils.GSUtilCopyDir(
346 src_dir=LOCAL_PLAYBACK_ROOT_DIR, 346 src_dir=LOCAL_PLAYBACK_ROOT_DIR,
347 gs_base=self._dest_gsbase, 347 gs_base=self._dest_gsbase,
348 dest_dir=dest_dir_name, 348 dest_dir=dest_dir_name,
349 gs_acl=PLAYBACK_CANNED_ACL) 349 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL)
350 if gs_status != 0: 350 if gs_status != 0:
351 raise Exception( 351 raise Exception(
352 'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % ( 352 'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % (
353 gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase, 353 gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase,
354 ROOT_PLAYBACK_DIR_NAME)) 354 ROOT_PLAYBACK_DIR_NAME))
355 self._SetGoogleReadACLs( 355 self._SetGoogleReadACLs(
356 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) 356 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
357 357
358 # Add a timestamp file to the SKP directory in Google Storage so we can 358 # Add a timestamp file to the SKP directory in Google Storage so we can
359 # use directory level rsync like functionality. 359 # use directory level rsync like functionality.
360 gs_utils.write_timestamp_file( 360 old_gs_utils.write_timestamp_file(
361 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, 361 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
362 timestamp_value=time.time(), 362 timestamp_value=time.time(),
363 gs_base=self._dest_gsbase, 363 gs_base=self._dest_gsbase,
364 gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME), 364 gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME),
365 gs_acl=PLAYBACK_CANNED_ACL, 365 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL,
366 local_dir=LOCAL_PLAYBACK_ROOT_DIR) 366 local_dir=LOCAL_PLAYBACK_ROOT_DIR)
367 367
368 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( 368 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
369 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) 369 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
370 else: 370 else:
371 print '\n\n=======Not Uploading to Google Storage=======\n\n' 371 print '\n\n=======Not Uploading to Google Storage=======\n\n'
372 print 'Generated resources are available in %s\n\n' % ( 372 print 'Generated resources are available in %s\n\n' % (
373 LOCAL_PLAYBACK_ROOT_DIR) 373 LOCAL_PLAYBACK_ROOT_DIR)
374 374
375 return 0 375 return 0
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after
422 file_utils.create_clean_local_dir(self._local_skp_dir) 422 file_utils.create_clean_local_dir(self._local_skp_dir)
423 423
424 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename): 424 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename):
425 """Downloads the webpages archive and its required page set from GS.""" 425 """Downloads the webpages archive and its required page set from GS."""
426 wpr_source = posixpath.join( 426 wpr_source = posixpath.join(
427 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', 427 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
428 wpr_data_file) 428 wpr_data_file)
429 page_set_source = posixpath.join( 429 page_set_source = posixpath.join(
430 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', 430 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
431 page_set_basename) 431 page_set_basename)
432 if (gs_utils.does_storage_object_exist(wpr_source) and 432 if (old_gs_utils.does_storage_object_exist(wpr_source) and
433 gs_utils.does_storage_object_exist(page_set_source)): 433 old_gs_utils.does_storage_object_exist(page_set_source)):
434 slave_utils.GSUtilDownloadFile( 434 slave_utils.GSUtilDownloadFile(
435 src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) 435 src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
436 slave_utils.GSUtilDownloadFile( 436 slave_utils.GSUtilDownloadFile(
437 src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR) 437 src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
438 else: 438 else:
439 raise Exception('%s and %s do not exist in Google Storage!' % ( 439 raise Exception('%s and %s do not exist in Google Storage!' % (
440 wpr_source, page_set_source)) 440 wpr_source, page_set_source))
441 441
442 442
443 if '__main__' == __name__: 443 if '__main__' == __name__:
444 option_parser = optparse.OptionParser() 444 option_parser = optparse.OptionParser()
445 option_parser.add_option( 445 option_parser.add_option(
446 '', '--page_sets', 446 '', '--page_sets',
447 help='Specifies the page sets to use to archive. Supports globs.', 447 help='Specifies the page sets to use to archive. Supports globs.',
448 default='all') 448 default='all')
449 option_parser.add_option( 449 option_parser.add_option(
450 '', '--skip_all_gs_access', action='store_true', 450 '', '--skip_all_gs_access', action='store_true',
451 help='All Google Storage interactions will be skipped if this flag is ' 451 help='All Google Storage interactions will be skipped if this flag is '
452 'specified. This is useful for cases where the user does not have ' 452 'specified. This is useful for cases where the user does not have '
453 'the required .boto file but would like to generate webpage ' 453 'the required .boto file but would like to generate webpage '
454 'archives and SKPs from the Skia page sets.', 454 'archives and SKPs from the Skia page sets.',
455 default=False) 455 default=False)
456 option_parser.add_option( 456 option_parser.add_option(
457 '', '--record', action='store_true', 457 '', '--record', action='store_true',
458 help='Specifies whether a new website archive should be created.', 458 help='Specifies whether a new website archive should be created.',
459 default=False) 459 default=False)
460 option_parser.add_option( 460 option_parser.add_option(
461 '', '--dest_gsbase', 461 '', '--dest_gsbase',
462 help='gs:// bucket_name, the bucket to upload the file to.', 462 help='gs:// bucket_name, the bucket to upload the file to.',
463 default=gs_utils.DEFAULT_DEST_GSBASE) 463 default=old_gs_utils.DEFAULT_DEST_GSBASE)
464 option_parser.add_option( 464 option_parser.add_option(
465 '', '--skia_tools', 465 '', '--skia_tools',
466 help=('Path to compiled Skia executable tools. ' 466 help=('Path to compiled Skia executable tools. '
467 'render_pictures/render_pdfs/bench_pictures is run on the set ' 467 'render_pictures/render_pdfs/bench_pictures is run on the set '
468 'after all SKPs are captured. If the script is run without ' 468 'after all SKPs are captured. If the script is run without '
469 '--non-interactive then the debugger is also run at the end. Debug ' 469 '--non-interactive then the debugger is also run at the end. Debug '
470 'builds are recommended because they seem to catch more failures ' 470 'builds are recommended because they seem to catch more failures '
471 'than Release builds.'), 471 'than Release builds.'),
472 default=None) 472 default=None)
473 option_parser.add_option( 473 option_parser.add_option(
(...skipping 15 matching lines...) Expand all
489 default=None) 489 default=None)
490 option_parser.add_option( 490 option_parser.add_option(
491 '', '--non-interactive', action='store_true', 491 '', '--non-interactive', action='store_true',
492 help='Runs the script without any prompts. If this flag is specified and ' 492 help='Runs the script without any prompts. If this flag is specified and '
493 '--skia_tools is specified then the debugger is not run.', 493 '--skia_tools is specified then the debugger is not run.',
494 default=False) 494 default=False)
495 options, unused_args = option_parser.parse_args() 495 options, unused_args = option_parser.parse_args()
496 496
497 playback = SkPicturePlayback(options) 497 playback = SkPicturePlayback(options)
498 sys.exit(playback.Run()) 498 sys.exit(playback.Run())
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698