Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(11)

Side by Side Diff: slave/skia_slave_scripts/upload_rendered_skps.py

Issue 405653004: add ACL-setting code to upload_rendered_skps.py (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: created local gs_utils wrapper module Created 6 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2014 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Uploads the results of render_skps.py.""" 6 """Uploads the results of render_skps.py."""
7 7
8 import os 8 import os
9 import posixpath 9 import posixpath
10 import sys 10 import sys
11 11
12 from build_step import BuildStep, PLAYBACK_CANNED_ACL 12 from build_step import BuildStep
13 from utils import old_gs_utils as gs_utils 13 from utils import gs_utils
14 from utils import old_gs_utils
14 import skia_vars 15 import skia_vars
15 16
16 GS_SUMMARIES_BUCKET = 'gs://chromium-skia-skp-summaries' 17 GS_SUMMARIES_BUCKET = 'gs://chromium-skia-skp-summaries'
17 SUBDIR_NAME = 'rendered-skps' 18 SUBDIR_NAME = 'rendered-skps'
18 19
19 20
20 class UploadRenderedSKPs(BuildStep): 21 class UploadRenderedSKPs(BuildStep):
21 22
22 def __init__(self, attempts=3, **kwargs): 23 def __init__(self, attempts=3, **kwargs):
23 super(UploadRenderedSKPs, self).__init__( 24 super(UploadRenderedSKPs, self).__init__(
24 attempts=attempts, **kwargs) 25 attempts=attempts, **kwargs)
25 26
26 def _Run(self): 27 def _Run(self):
27 # Upload individual image files to Google Storage. 28 # Upload individual image files to Google Storage.
28 # 29 #
29 # TODO(epoger): Add a "noclobber" mode to gs_utils.upload_dir_contents() 30 # TODO(epoger): Add a "noclobber" mode to gs_utils.upload_dir_contents()
30 # and use it here so we don't re-upload image files we already have 31 # and use it here so we don't re-upload image files we already have
31 # in Google Storage. 32 # in Google Storage.
32 # 33 #
33 # TODO(epoger): Change ACLs of files uploaded to Google Storage to 34 gs = gs_utils.GSUtils()
34 # google.com:READ . See _SetGoogleReadACLs() in 35 gs_bucket = gs.without_gs_prefix(
35 # https://skia.googlesource.com/buildbot/+/master/slave/skia_slave_scripts/ 36 skia_vars.GetGlobalVariable('googlestorage_bucket'))
36 # webpages_playback.py 37
37 src_dir = os.path.abspath(self.playback_actual_images_dir) 38 src_dir = os.path.abspath(self.playback_actual_images_dir)
38 if os.listdir(src_dir): 39 if os.listdir(src_dir):
39 dest_dir = posixpath.join( 40 print 'Uploading image files from %s to bucket=%s, dir=%s' % (
40 skia_vars.GetGlobalVariable('googlestorage_bucket'), SUBDIR_NAME) 41 src_dir, gs_bucket, SUBDIR_NAME)
41 print 'Uploading image files from %s to %s.' % (src_dir, dest_dir) 42 gs.upload_dir_contents(
42 gs_utils.upload_dir_contents( 43 source_dir=src_dir, dest_bucket=gs_bucket, dest_dir=SUBDIR_NAME,
43 local_src_dir=src_dir, remote_dest_dir=dest_dir, 44 predefined_acl=gs.PLAYBACK_CANNED_ACL,
44 gs_acl=PLAYBACK_CANNED_ACL) 45 fine_grained_acl_list=gs.PLAYBACK_FINEGRAINED_ACL_LIST)
45 else: 46 else:
46 print ('No image files in %s, so skipping upload to Google Storage.' % 47 print ('Skipping upload to Google Storage, because no image files in %s' %
47 src_dir) 48 src_dir)
48 49
49 # Upload image summaries (checksums) to Google Storage. 50 # Upload image summaries (checksums) to Google Storage.
50 src_dir = os.path.abspath(self.playback_actual_summaries_dir) 51 src_dir = os.path.abspath(self.playback_actual_summaries_dir)
51 filenames = os.listdir(src_dir) 52 filenames = os.listdir(src_dir)
52 if filenames: 53 if filenames:
53 dest_dir = posixpath.join(GS_SUMMARIES_BUCKET, self._args['builder_name']) 54 dest_dir = posixpath.join(GS_SUMMARIES_BUCKET, self._args['builder_name'])
54 print ('Uploading %d image summaries from %s to %s: %s' % ( 55 print ('Uploading %d image summaries from %s to %s: %s' % (
55 len(filenames), src_dir, dest_dir, filenames)) 56 len(filenames), src_dir, dest_dir, filenames))
56 for filename in filenames: 57 for filename in filenames:
57 src_path = os.path.join(src_dir, filename) 58 src_path = os.path.join(src_dir, filename)
58 dest_path = posixpath.join(dest_dir, filename) 59 dest_path = posixpath.join(dest_dir, filename)
59 gs_utils.upload_file( 60 # It's important to only upload the summary file when it has changed,
61 # because we use the history of the file in Google Storage to tell us
62 # when any of the results changed.
63 #
64 # TODO(epoger): Once gs_utils.upload_file() supports only_if_modified
65 # parameter, start using it, so we can set fine_grained_acl_list like
66 # we do above... we'll need that for google.com users to be able to
67 # download the summary files.
68 old_gs_utils.upload_file(
60 local_src_path=src_path, remote_dest_path=dest_path, 69 local_src_path=src_path, remote_dest_path=dest_path,
61 gs_acl=PLAYBACK_CANNED_ACL, only_if_modified=True) 70 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL, only_if_modified=True)
62 else: 71 else:
63 print ('No image summaries in %s, so skipping upload to Google Storage.' % 72 print ('Skipping upload to Google Storage, because no image summaries '
64 src_dir) 73 'in %s' % src_dir)
65 74
66 if '__main__' == __name__: 75 if '__main__' == __name__:
67 sys.exit(BuildStep.RunBuildStep(UploadRenderedSKPs)) 76 sys.exit(BuildStep.RunBuildStep(UploadRenderedSKPs))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698