Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(121)

Side by Side Diff: tools/rebaseline_webpage_images.py

Issue 405653004: add ACL-setting code to upload_rendered_skps.py (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: created local gs_utils wrapper module Created 6 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Rebaselines the specified gm_image_subdir or all gm_image_subdirs. 6 """Rebaselines the specified gm_image_subdir or all gm_image_subdirs.
7 7
8 Note: 8 Note:
9 Please make sure the gm-actuals of the builders of interest are not being 9 Please make sure the gm-actuals of the builders of interest are not being
10 updated at the time this script is run, or else this script will wait for 10 updated at the time this script is run, or else this script will wait for
(...skipping 19 matching lines...) Expand all
30 import time 30 import time
31 31
32 # Set the PYTHONPATH for this script to include chromium_buildbot scripts and 32 # Set the PYTHONPATH for this script to include chromium_buildbot scripts and
33 # site_config and skia_slave_scripts. 33 # site_config and skia_slave_scripts.
34 sys.path.append( 34 sys.path.append(
35 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'scripts')) 35 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'scripts'))
36 sys.path.append( 36 sys.path.append(
37 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'site_config')) 37 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'site_config'))
38 sys.path.append(os.path.join(os.pardir, 'slave', 'skia_slave_scripts')) 38 sys.path.append(os.path.join(os.pardir, 'slave', 'skia_slave_scripts'))
39 39
40 from build_step import PLAYBACK_CANNED_ACL
41 from common import chromium_utils 40 from common import chromium_utils
42 from slave import slave_utils 41 from slave import slave_utils
43 from slave import svn 42 from slave import svn
44 from utils import gs_utils 43 from utils import gs_utils
44 from utils import old_gs_utils
45 from utils import sync_bucket_subdir 45 from utils import sync_bucket_subdir
46 46
47 import compare_and_upload_webpage_gms 47 import compare_and_upload_webpage_gms
48 import playback_dirs 48 import playback_dirs
49 49
50 50
51 GM_IMAGE_TO_BASELINE_BUILDER = { 51 GM_IMAGE_TO_BASELINE_BUILDER = {
52 'base-shuttle-win7-intel-float': 52 'base-shuttle-win7-intel-float':
53 'Skia_Shuttle_Win7_Intel_Float_Release_32', 53 'Skia_Shuttle_Win7_Intel_Float_Release_32',
54 'base-shuttle-win7-intel-angle': 54 'base-shuttle-win7-intel-angle':
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
109 if not gm_image_subdir in GM_IMAGE_TO_BASELINE_BUILDER: 109 if not gm_image_subdir in GM_IMAGE_TO_BASELINE_BUILDER:
110 raise ValueError( 110 raise ValueError(
111 'Unknown specified gm_image_subdir (%s). Please use one from:\n%s' % ( 111 'Unknown specified gm_image_subdir (%s). Please use one from:\n%s' % (
112 gm_image_subdir, str(GM_IMAGE_TO_BASELINE_BUILDER.keys()))) 112 gm_image_subdir, str(GM_IMAGE_TO_BASELINE_BUILDER.keys())))
113 113
114 114
115 dest_gsbase = sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE 115 dest_gsbase = sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE
116 116
117 117
118 # Ensure the right .boto file is used by gsutil. 118 # Ensure the right .boto file is used by gsutil.
119 if not gs_utils.DoesStorageObjectExist(dest_gsbase): 119 if not old_gs_utils.DoesStorageObjectExist(dest_gsbase):
120 raise Exception( 120 raise Exception(
121 'Missing .boto file or .boto does not have the right credentials. Please ' 121 'Missing .boto file or .boto does not have the right credentials. Please '
122 'see https://docs.google.com/a/google.com/document/d/1ZzHP6M5qACA9nJnLq' 122 'see https://docs.google.com/a/google.com/document/d/1ZzHP6M5qACA9nJnLq'
123 'OZr2Hl0rjYqE4yQsQWAfVjKCzs/edit (may have to request access)') 123 'OZr2Hl0rjYqE4yQsQWAfVjKCzs/edit (may have to request access)')
124 124
125 125
126 for gm_image_subdir in gm_images_seq: 126 for gm_image_subdir in gm_images_seq:
127 print '\n\n' 127 print '\n\n'
128 print '======================================================================' 128 print '======================================================================'
129 print 'Rebaselining %s images' % gm_image_subdir 129 print 'Rebaselining %s images' % gm_image_subdir
130 print '======================================================================' 130 print '======================================================================'
131 131
132 builder_name = GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir] 132 builder_name = GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir]
133 133
134 storage_playback_dirs = playback_dirs.StorageSkpPlaybackDirs( 134 storage_playback_dirs = playback_dirs.StorageSkpPlaybackDirs(
135 builder_name=builder_name, 135 builder_name=builder_name,
136 gm_image_subdir=gm_image_subdir, 136 gm_image_subdir=gm_image_subdir,
137 perf_output_basedir=None) 137 perf_output_basedir=None)
138 138
139 gm_actual_dir = posixpath.join( 139 gm_actual_dir = posixpath.join(
140 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) 140 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir())
141 gm_expected_dir = posixpath.join( 141 gm_expected_dir = posixpath.join(
142 dest_gsbase, storage_playback_dirs.PlaybackGmExpectedDir()) 142 dest_gsbase, storage_playback_dirs.PlaybackGmExpectedDir())
143 143
144 print '\n\n=======Throw an Exception if gm_actual_dir does not exist=======' 144 print '\n\n=======Throw an Exception if gm_actual_dir does not exist======='
145 if not gs_utils.DoesStorageObjectExist(gm_actual_dir): 145 if not old_gs_utils.DoesStorageObjectExist(gm_actual_dir):
146 raise Exception("%s does not exist in Google Storage!" % gm_actual_dir) 146 raise Exception("%s does not exist in Google Storage!" % gm_actual_dir)
147 147
148 print '\n\n=======Wait if gm_actual_dir is being updated=======' 148 print '\n\n=======Wait if gm_actual_dir is being updated======='
149 keep_waiting = True 149 keep_waiting = True
150 while(keep_waiting): 150 while(keep_waiting):
151 gm_actual_started_timestamp = gs_utils.ReadTimeStampFile( 151 gm_actual_started_timestamp = old_gs_utils.ReadTimeStampFile(
152 timestamp_file_name=gs_utils.TIMESTAMP_STARTED_FILENAME, 152 timestamp_file_name=old_gs_utils.TIMESTAMP_STARTED_FILENAME,
153 gs_base=dest_gsbase, 153 gs_base=dest_gsbase,
154 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) 154 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir())
155 gm_actual_completed_timestamp = gs_utils.ReadTimeStampFile( 155 gm_actual_completed_timestamp = old_gs_utils.ReadTimeStampFile(
156 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, 156 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
157 gs_base=dest_gsbase, 157 gs_base=dest_gsbase,
158 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) 158 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir())
159 if gm_actual_started_timestamp != gm_actual_completed_timestamp: 159 if gm_actual_started_timestamp != gm_actual_completed_timestamp:
160 print '\n\nSleep for a minute since gm_actual_dir is being updated.' 160 print '\n\nSleep for a minute since gm_actual_dir is being updated.'
161 print 'Please manually exit the script if you want to rerun it later.\n' 161 print 'Please manually exit the script if you want to rerun it later.\n'
162 time.sleep(60) 162 time.sleep(60)
163 else: 163 else:
164 keep_waiting = False 164 keep_waiting = False
165 165
166 print '\n\n=======Add the REBASELINE_IN_PROGRESS lock file=======' 166 print '\n\n=======Add the REBASELINE_IN_PROGRESS lock file======='
167 gs_utils.WriteTimeStampFile( 167 old_gs_utils.WriteTimeStampFile(
168 timestamp_file_name=( 168 timestamp_file_name=(
169 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME), 169 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME),
170 timestamp_value=time.time(), 170 timestamp_value=time.time(),
171 gs_base=dest_gsbase, 171 gs_base=dest_gsbase,
172 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir(), 172 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir(),
173 gs_acl=PLAYBACK_CANNED_ACL) 173 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL)
174 174
175 print '\n\n=======Delete contents of gm_expected_dir=======' 175 print '\n\n=======Delete contents of gm_expected_dir======='
176 gs_utils.DeleteStorageObject(gm_expected_dir) 176 old_gs_utils.DeleteStorageObject(gm_expected_dir)
177 177
178 print '\n\n=====Copy all contents from gm_actual_dir to gm_expected_dir======' 178 print '\n\n=====Copy all contents from gm_actual_dir to gm_expected_dir======'
179 179
180 # Gather list of all files. 180 # Gather list of all files.
181 gm_actual_contents = gs_utils.ListStorageDirectory( 181 gm_actual_contents = old_gs_utils.ListStorageDirectory(
182 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) 182 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir())
183 183
184 # Remove REBASELINE, TIMESTAMP_* and COMPARISON files from the list. 184 # Remove REBASELINE, TIMESTAMP_* and COMPARISON files from the list.
185 for file_to_remove in ( 185 for file_to_remove in (
186 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME, 186 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME,
187 gs_utils.TIMESTAMP_STARTED_FILENAME, 187 old_gs_utils.TIMESTAMP_STARTED_FILENAME,
188 gs_utils.TIMESTAMP_COMPLETED_FILENAME, 188 old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
189 compare_and_upload_webpage_gms.LAST_COMPARISON_FILENAME): 189 compare_and_upload_webpage_gms.LAST_COMPARISON_FILENAME):
190 gs_file_to_remove = posixpath.join(gm_actual_dir, file_to_remove) 190 gs_file_to_remove = posixpath.join(gm_actual_dir, file_to_remove)
191 if gs_file_to_remove in gm_actual_contents: 191 if gs_file_to_remove in gm_actual_contents:
192 gm_actual_contents.remove(gs_file_to_remove) 192 gm_actual_contents.remove(gs_file_to_remove)
193 193
194 # Copy over files in chunks. 194 # Copy over files in chunks.
195 # pylint: disable=W0212 195 # pylint: disable=W0212
196 for files_chunk in gs_utils._GetChunks(gm_actual_contents, 196 for files_chunk in old_gs_utils._GetChunks(gm_actual_contents,
197 gs_utils.FILES_CHUNK): 197 old_gs_utils.FILES_CHUNK):
198 gsutil = slave_utils.GSUtilSetup() 198 gsutil = slave_utils.GSUtilSetup()
199 command = ([gsutil, 'cp'] + files_chunk + 199 command = ([gsutil, 'cp'] + files_chunk +
200 [posixpath.join(gm_expected_dir, '')]) 200 [posixpath.join(gm_expected_dir, '')])
201 if chromium_utils.RunCommand(command) != 0: 201 if chromium_utils.RunCommand(command) != 0:
202 raise Exception( 202 raise Exception(
203 'Could not upload the chunk to Google Storage! The chunk: %s' 203 'Could not upload the chunk to Google Storage! The chunk: %s'
204 % files_chunk) 204 % files_chunk)
205 205
206 print '\n\n=======Delete the REBASELINE_IN_PROGRESS lock file=======' 206 print '\n\n=======Delete the REBASELINE_IN_PROGRESS lock file======='
207 gs_utils.DeleteStorageObject( 207 old_gs_utils.DeleteStorageObject(
208 posixpath.join( 208 posixpath.join(
209 gm_actual_dir, 209 gm_actual_dir,
210 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME)) 210 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME))
211 211
212 print '\n\n=======Update gm_expected_dir timestamp=======' 212 print '\n\n=======Update gm_expected_dir timestamp======='
213 gs_utils.WriteTimeStampFile( 213 old_gs_utils.WriteTimeStampFile(
214 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, 214 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
215 timestamp_value=time.time(), 215 timestamp_value=time.time(),
216 gs_base=dest_gsbase, 216 gs_base=dest_gsbase,
217 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), 217 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(),
218 gs_acl=PLAYBACK_CANNED_ACL, 218 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL,
219 local_dir=None) 219 local_dir=None)
220 220
221 print '\n\n=======Add LAST_REBASELINED_BY file=======' 221 print '\n\n=======Add LAST_REBASELINED_BY file======='
222 gs_utils.WriteTimeStampFile( 222 old_gs_utils.WriteTimeStampFile(
223 timestamp_file_name=gs_utils.LAST_REBASELINED_BY_FILENAME, 223 timestamp_file_name=old_gs_utils.LAST_REBASELINED_BY_FILENAME,
224 timestamp_value=getpass.getuser(), 224 timestamp_value=getpass.getuser(),
225 gs_base=dest_gsbase, 225 gs_base=dest_gsbase,
226 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), 226 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(),
227 gs_acl=PLAYBACK_CANNED_ACL, 227 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL,
228 local_dir=None) 228 local_dir=None)
229 229
230 230
231 # Submit whitespace change to trigger rebuilds if skia trunk location, svn 231 # Submit whitespace change to trigger rebuilds if skia trunk location, svn
232 # username and password have been provided. 232 # username and password have been provided.
233 if commit_whitespace_change: 233 if commit_whitespace_change:
234 skia_trunk_dir = sys.argv[2] 234 skia_trunk_dir = sys.argv[2]
235 svn_username = sys.argv[3] 235 svn_username = sys.argv[3]
236 svn_password = sys.argv[4] 236 svn_password = sys.argv[4]
237 repo = svn.Svn(skia_trunk_dir, svn_username, svn_password, 237 repo = svn.Svn(skia_trunk_dir, svn_username, svn_password,
238 additional_svn_flags=['--trust-server-cert', '--no-auth-cache', 238 additional_svn_flags=['--trust-server-cert', '--no-auth-cache',
239 '--non-interactive']) 239 '--non-interactive'])
240 whitespace_file = open(os.path.join(skia_trunk_dir, 'whitespace.txt'), 'a') 240 whitespace_file = open(os.path.join(skia_trunk_dir, 'whitespace.txt'), 'a')
241 try: 241 try:
242 whitespace_file.write('\n') 242 whitespace_file.write('\n')
243 finally: 243 finally:
244 whitespace_file.close() 244 whitespace_file.close()
245 245
246 print '\n\n=======Submit whitespace change to trigger rebuilds=======' 246 print '\n\n=======Submit whitespace change to trigger rebuilds======='
247 builders_to_run = [] 247 builders_to_run = []
248 for gm_image_subdir in gm_images_seq: 248 for gm_image_subdir in gm_images_seq:
249 builders_to_run.append(GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir]) 249 builders_to_run.append(GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir])
250 run_builders_keyword = '(RunBuilders:%s)' % ','.join(builders_to_run) 250 run_builders_keyword = '(RunBuilders:%s)' % ','.join(builders_to_run)
251 # pylint: disable=W0212 251 # pylint: disable=W0212
252 repo._RunSvnCommand( 252 repo._RunSvnCommand(
253 ['commit', '--message', 253 ['commit', '--message',
254 'Rebaselined webpage image GMs for %s on Google Storage.\n%s' 254 'Rebaselined webpage image GMs for %s on Google Storage.\n%s'
255 % (gm_images_seq, run_builders_keyword), 255 % (gm_images_seq, run_builders_keyword),
256 'whitespace.txt']) 256 'whitespace.txt'])
OLDNEW
« slave/skia_slave_scripts/utils/gs_utils.py ('K') | « slave/skia_slave_scripts/webpages_playback.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698