OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Rebaselines the specified gm_image_subdir or all gm_image_subdirs. | 6 """Rebaselines the specified gm_image_subdir or all gm_image_subdirs. |
7 | 7 |
8 Note: | 8 Note: |
9 Please make sure the gm-actuals of the builders of interest are not being | 9 Please make sure the gm-actuals of the builders of interest are not being |
10 updated at the time this script is run, or else this script will wait for | 10 updated at the time this script is run, or else this script will wait for |
(...skipping 19 matching lines...) Expand all Loading... |
30 import time | 30 import time |
31 | 31 |
32 # Set the PYTHONPATH for this script to include chromium_buildbot scripts and | 32 # Set the PYTHONPATH for this script to include chromium_buildbot scripts and |
33 # site_config and skia_slave_scripts. | 33 # site_config and skia_slave_scripts. |
34 sys.path.append( | 34 sys.path.append( |
35 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'scripts')) | 35 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'scripts')) |
36 sys.path.append( | 36 sys.path.append( |
37 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'site_config')) | 37 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'site_config')) |
38 sys.path.append(os.path.join(os.pardir, 'slave', 'skia_slave_scripts')) | 38 sys.path.append(os.path.join(os.pardir, 'slave', 'skia_slave_scripts')) |
39 | 39 |
40 from build_step import PLAYBACK_CANNED_ACL | |
41 from common import chromium_utils | 40 from common import chromium_utils |
42 from slave import slave_utils | 41 from slave import slave_utils |
43 from slave import svn | 42 from slave import svn |
44 from utils import gs_utils | 43 from utils import gs_utils |
| 44 from utils import old_gs_utils |
45 from utils import sync_bucket_subdir | 45 from utils import sync_bucket_subdir |
46 | 46 |
47 import compare_and_upload_webpage_gms | 47 import compare_and_upload_webpage_gms |
48 import playback_dirs | 48 import playback_dirs |
49 | 49 |
50 | 50 |
51 GM_IMAGE_TO_BASELINE_BUILDER = { | 51 GM_IMAGE_TO_BASELINE_BUILDER = { |
52 'base-shuttle-win7-intel-float': | 52 'base-shuttle-win7-intel-float': |
53 'Skia_Shuttle_Win7_Intel_Float_Release_32', | 53 'Skia_Shuttle_Win7_Intel_Float_Release_32', |
54 'base-shuttle-win7-intel-angle': | 54 'base-shuttle-win7-intel-angle': |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
109 if not gm_image_subdir in GM_IMAGE_TO_BASELINE_BUILDER: | 109 if not gm_image_subdir in GM_IMAGE_TO_BASELINE_BUILDER: |
110 raise ValueError( | 110 raise ValueError( |
111 'Unknown specified gm_image_subdir (%s). Please use one from:\n%s' % ( | 111 'Unknown specified gm_image_subdir (%s). Please use one from:\n%s' % ( |
112 gm_image_subdir, str(GM_IMAGE_TO_BASELINE_BUILDER.keys()))) | 112 gm_image_subdir, str(GM_IMAGE_TO_BASELINE_BUILDER.keys()))) |
113 | 113 |
114 | 114 |
115 dest_gsbase = sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE | 115 dest_gsbase = sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE |
116 | 116 |
117 | 117 |
118 # Ensure the right .boto file is used by gsutil. | 118 # Ensure the right .boto file is used by gsutil. |
119 if not gs_utils.DoesStorageObjectExist(dest_gsbase): | 119 if not old_gs_utils.DoesStorageObjectExist(dest_gsbase): |
120 raise Exception( | 120 raise Exception( |
121 'Missing .boto file or .boto does not have the right credentials. Please ' | 121 'Missing .boto file or .boto does not have the right credentials. Please ' |
122 'see https://docs.google.com/a/google.com/document/d/1ZzHP6M5qACA9nJnLq' | 122 'see https://docs.google.com/a/google.com/document/d/1ZzHP6M5qACA9nJnLq' |
123 'OZr2Hl0rjYqE4yQsQWAfVjKCzs/edit (may have to request access)') | 123 'OZr2Hl0rjYqE4yQsQWAfVjKCzs/edit (may have to request access)') |
124 | 124 |
125 | 125 |
126 for gm_image_subdir in gm_images_seq: | 126 for gm_image_subdir in gm_images_seq: |
127 print '\n\n' | 127 print '\n\n' |
128 print '======================================================================' | 128 print '======================================================================' |
129 print 'Rebaselining %s images' % gm_image_subdir | 129 print 'Rebaselining %s images' % gm_image_subdir |
130 print '======================================================================' | 130 print '======================================================================' |
131 | 131 |
132 builder_name = GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir] | 132 builder_name = GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir] |
133 | 133 |
134 storage_playback_dirs = playback_dirs.StorageSkpPlaybackDirs( | 134 storage_playback_dirs = playback_dirs.StorageSkpPlaybackDirs( |
135 builder_name=builder_name, | 135 builder_name=builder_name, |
136 gm_image_subdir=gm_image_subdir, | 136 gm_image_subdir=gm_image_subdir, |
137 perf_output_basedir=None) | 137 perf_output_basedir=None) |
138 | 138 |
139 gm_actual_dir = posixpath.join( | 139 gm_actual_dir = posixpath.join( |
140 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) | 140 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) |
141 gm_expected_dir = posixpath.join( | 141 gm_expected_dir = posixpath.join( |
142 dest_gsbase, storage_playback_dirs.PlaybackGmExpectedDir()) | 142 dest_gsbase, storage_playback_dirs.PlaybackGmExpectedDir()) |
143 | 143 |
144 print '\n\n=======Throw an Exception if gm_actual_dir does not exist=======' | 144 print '\n\n=======Throw an Exception if gm_actual_dir does not exist=======' |
145 if not gs_utils.DoesStorageObjectExist(gm_actual_dir): | 145 if not old_gs_utils.DoesStorageObjectExist(gm_actual_dir): |
146 raise Exception("%s does not exist in Google Storage!" % gm_actual_dir) | 146 raise Exception("%s does not exist in Google Storage!" % gm_actual_dir) |
147 | 147 |
148 print '\n\n=======Wait if gm_actual_dir is being updated=======' | 148 print '\n\n=======Wait if gm_actual_dir is being updated=======' |
149 keep_waiting = True | 149 keep_waiting = True |
150 while(keep_waiting): | 150 while(keep_waiting): |
151 gm_actual_started_timestamp = gs_utils.ReadTimeStampFile( | 151 gm_actual_started_timestamp = old_gs_utils.ReadTimeStampFile( |
152 timestamp_file_name=gs_utils.TIMESTAMP_STARTED_FILENAME, | 152 timestamp_file_name=old_gs_utils.TIMESTAMP_STARTED_FILENAME, |
153 gs_base=dest_gsbase, | 153 gs_base=dest_gsbase, |
154 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) | 154 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) |
155 gm_actual_completed_timestamp = gs_utils.ReadTimeStampFile( | 155 gm_actual_completed_timestamp = old_gs_utils.ReadTimeStampFile( |
156 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 156 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
157 gs_base=dest_gsbase, | 157 gs_base=dest_gsbase, |
158 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) | 158 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) |
159 if gm_actual_started_timestamp != gm_actual_completed_timestamp: | 159 if gm_actual_started_timestamp != gm_actual_completed_timestamp: |
160 print '\n\nSleep for a minute since gm_actual_dir is being updated.' | 160 print '\n\nSleep for a minute since gm_actual_dir is being updated.' |
161 print 'Please manually exit the script if you want to rerun it later.\n' | 161 print 'Please manually exit the script if you want to rerun it later.\n' |
162 time.sleep(60) | 162 time.sleep(60) |
163 else: | 163 else: |
164 keep_waiting = False | 164 keep_waiting = False |
165 | 165 |
166 print '\n\n=======Add the REBASELINE_IN_PROGRESS lock file=======' | 166 print '\n\n=======Add the REBASELINE_IN_PROGRESS lock file=======' |
167 gs_utils.WriteTimeStampFile( | 167 old_gs_utils.WriteTimeStampFile( |
168 timestamp_file_name=( | 168 timestamp_file_name=( |
169 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME), | 169 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME), |
170 timestamp_value=time.time(), | 170 timestamp_value=time.time(), |
171 gs_base=dest_gsbase, | 171 gs_base=dest_gsbase, |
172 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir(), | 172 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir(), |
173 gs_acl=PLAYBACK_CANNED_ACL) | 173 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL) |
174 | 174 |
175 print '\n\n=======Delete contents of gm_expected_dir=======' | 175 print '\n\n=======Delete contents of gm_expected_dir=======' |
176 gs_utils.DeleteStorageObject(gm_expected_dir) | 176 old_gs_utils.DeleteStorageObject(gm_expected_dir) |
177 | 177 |
178 print '\n\n=====Copy all contents from gm_actual_dir to gm_expected_dir======' | 178 print '\n\n=====Copy all contents from gm_actual_dir to gm_expected_dir======' |
179 | 179 |
180 # Gather list of all files. | 180 # Gather list of all files. |
181 gm_actual_contents = gs_utils.ListStorageDirectory( | 181 gm_actual_contents = old_gs_utils.ListStorageDirectory( |
182 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) | 182 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) |
183 | 183 |
184 # Remove REBASELINE, TIMESTAMP_* and COMPARISON files from the list. | 184 # Remove REBASELINE, TIMESTAMP_* and COMPARISON files from the list. |
185 for file_to_remove in ( | 185 for file_to_remove in ( |
186 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME, | 186 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME, |
187 gs_utils.TIMESTAMP_STARTED_FILENAME, | 187 old_gs_utils.TIMESTAMP_STARTED_FILENAME, |
188 gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 188 old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
189 compare_and_upload_webpage_gms.LAST_COMPARISON_FILENAME): | 189 compare_and_upload_webpage_gms.LAST_COMPARISON_FILENAME): |
190 gs_file_to_remove = posixpath.join(gm_actual_dir, file_to_remove) | 190 gs_file_to_remove = posixpath.join(gm_actual_dir, file_to_remove) |
191 if gs_file_to_remove in gm_actual_contents: | 191 if gs_file_to_remove in gm_actual_contents: |
192 gm_actual_contents.remove(gs_file_to_remove) | 192 gm_actual_contents.remove(gs_file_to_remove) |
193 | 193 |
194 # Copy over files in chunks. | 194 # Copy over files in chunks. |
195 # pylint: disable=W0212 | 195 # pylint: disable=W0212 |
196 for files_chunk in gs_utils._GetChunks(gm_actual_contents, | 196 for files_chunk in old_gs_utils._GetChunks(gm_actual_contents, |
197 gs_utils.FILES_CHUNK): | 197 old_gs_utils.FILES_CHUNK): |
198 gsutil = slave_utils.GSUtilSetup() | 198 gsutil = slave_utils.GSUtilSetup() |
199 command = ([gsutil, 'cp'] + files_chunk + | 199 command = ([gsutil, 'cp'] + files_chunk + |
200 [posixpath.join(gm_expected_dir, '')]) | 200 [posixpath.join(gm_expected_dir, '')]) |
201 if chromium_utils.RunCommand(command) != 0: | 201 if chromium_utils.RunCommand(command) != 0: |
202 raise Exception( | 202 raise Exception( |
203 'Could not upload the chunk to Google Storage! The chunk: %s' | 203 'Could not upload the chunk to Google Storage! The chunk: %s' |
204 % files_chunk) | 204 % files_chunk) |
205 | 205 |
206 print '\n\n=======Delete the REBASELINE_IN_PROGRESS lock file=======' | 206 print '\n\n=======Delete the REBASELINE_IN_PROGRESS lock file=======' |
207 gs_utils.DeleteStorageObject( | 207 old_gs_utils.DeleteStorageObject( |
208 posixpath.join( | 208 posixpath.join( |
209 gm_actual_dir, | 209 gm_actual_dir, |
210 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME)) | 210 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME)) |
211 | 211 |
212 print '\n\n=======Update gm_expected_dir timestamp=======' | 212 print '\n\n=======Update gm_expected_dir timestamp=======' |
213 gs_utils.WriteTimeStampFile( | 213 old_gs_utils.WriteTimeStampFile( |
214 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 214 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
215 timestamp_value=time.time(), | 215 timestamp_value=time.time(), |
216 gs_base=dest_gsbase, | 216 gs_base=dest_gsbase, |
217 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), | 217 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), |
218 gs_acl=PLAYBACK_CANNED_ACL, | 218 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL, |
219 local_dir=None) | 219 local_dir=None) |
220 | 220 |
221 print '\n\n=======Add LAST_REBASELINED_BY file=======' | 221 print '\n\n=======Add LAST_REBASELINED_BY file=======' |
222 gs_utils.WriteTimeStampFile( | 222 old_gs_utils.WriteTimeStampFile( |
223 timestamp_file_name=gs_utils.LAST_REBASELINED_BY_FILENAME, | 223 timestamp_file_name=old_gs_utils.LAST_REBASELINED_BY_FILENAME, |
224 timestamp_value=getpass.getuser(), | 224 timestamp_value=getpass.getuser(), |
225 gs_base=dest_gsbase, | 225 gs_base=dest_gsbase, |
226 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), | 226 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), |
227 gs_acl=PLAYBACK_CANNED_ACL, | 227 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL, |
228 local_dir=None) | 228 local_dir=None) |
229 | 229 |
230 | 230 |
231 # Submit whitespace change to trigger rebuilds if skia trunk location, svn | 231 # Submit whitespace change to trigger rebuilds if skia trunk location, svn |
232 # username and password have been provided. | 232 # username and password have been provided. |
233 if commit_whitespace_change: | 233 if commit_whitespace_change: |
234 skia_trunk_dir = sys.argv[2] | 234 skia_trunk_dir = sys.argv[2] |
235 svn_username = sys.argv[3] | 235 svn_username = sys.argv[3] |
236 svn_password = sys.argv[4] | 236 svn_password = sys.argv[4] |
237 repo = svn.Svn(skia_trunk_dir, svn_username, svn_password, | 237 repo = svn.Svn(skia_trunk_dir, svn_username, svn_password, |
238 additional_svn_flags=['--trust-server-cert', '--no-auth-cache', | 238 additional_svn_flags=['--trust-server-cert', '--no-auth-cache', |
239 '--non-interactive']) | 239 '--non-interactive']) |
240 whitespace_file = open(os.path.join(skia_trunk_dir, 'whitespace.txt'), 'a') | 240 whitespace_file = open(os.path.join(skia_trunk_dir, 'whitespace.txt'), 'a') |
241 try: | 241 try: |
242 whitespace_file.write('\n') | 242 whitespace_file.write('\n') |
243 finally: | 243 finally: |
244 whitespace_file.close() | 244 whitespace_file.close() |
245 | 245 |
246 print '\n\n=======Submit whitespace change to trigger rebuilds=======' | 246 print '\n\n=======Submit whitespace change to trigger rebuilds=======' |
247 builders_to_run = [] | 247 builders_to_run = [] |
248 for gm_image_subdir in gm_images_seq: | 248 for gm_image_subdir in gm_images_seq: |
249 builders_to_run.append(GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir]) | 249 builders_to_run.append(GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir]) |
250 run_builders_keyword = '(RunBuilders:%s)' % ','.join(builders_to_run) | 250 run_builders_keyword = '(RunBuilders:%s)' % ','.join(builders_to_run) |
251 # pylint: disable=W0212 | 251 # pylint: disable=W0212 |
252 repo._RunSvnCommand( | 252 repo._RunSvnCommand( |
253 ['commit', '--message', | 253 ['commit', '--message', |
254 'Rebaselined webpage image GMs for %s on Google Storage.\n%s' | 254 'Rebaselined webpage image GMs for %s on Google Storage.\n%s' |
255 % (gm_images_seq, run_builders_keyword), | 255 % (gm_images_seq, run_builders_keyword), |
256 'whitespace.txt']) | 256 'whitespace.txt']) |
OLD | NEW |