| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Rebaselines the specified gm_image_subdir or all gm_image_subdirs. | 6 """Rebaselines the specified gm_image_subdir or all gm_image_subdirs. |
| 7 | 7 |
| 8 Note: | 8 Note: |
| 9 Please make sure the gm-actuals of the builders of interest are not being | 9 Please make sure the gm-actuals of the builders of interest are not being |
| 10 updated at the time this script is run, or else this script will wait for | 10 updated at the time this script is run, or else this script will wait for |
| (...skipping 19 matching lines...) Expand all Loading... |
| 30 import time | 30 import time |
| 31 | 31 |
| 32 # Set the PYTHONPATH for this script to include chromium_buildbot scripts and | 32 # Set the PYTHONPATH for this script to include chromium_buildbot scripts and |
| 33 # site_config and skia_slave_scripts. | 33 # site_config and skia_slave_scripts. |
| 34 sys.path.append( | 34 sys.path.append( |
| 35 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'scripts')) | 35 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'scripts')) |
| 36 sys.path.append( | 36 sys.path.append( |
| 37 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'site_config')) | 37 os.path.join(os.pardir, 'third_party', 'chromium_buildbot', 'site_config')) |
| 38 sys.path.append(os.path.join(os.pardir, 'slave', 'skia_slave_scripts')) | 38 sys.path.append(os.path.join(os.pardir, 'slave', 'skia_slave_scripts')) |
| 39 | 39 |
| 40 from build_step import PLAYBACK_CANNED_ACL | |
| 41 from common import chromium_utils | 40 from common import chromium_utils |
| 42 from slave import slave_utils | 41 from slave import slave_utils |
| 43 from slave import svn | 42 from slave import svn |
| 44 from utils import gs_utils | 43 from utils import gs_utils |
| 44 from utils import old_gs_utils |
| 45 from utils import sync_bucket_subdir | 45 from utils import sync_bucket_subdir |
| 46 | 46 |
| 47 import compare_and_upload_webpage_gms | 47 import compare_and_upload_webpage_gms |
| 48 import playback_dirs | 48 import playback_dirs |
| 49 | 49 |
| 50 | 50 |
| 51 GM_IMAGE_TO_BASELINE_BUILDER = { | 51 GM_IMAGE_TO_BASELINE_BUILDER = { |
| 52 'base-shuttle-win7-intel-float': | 52 'base-shuttle-win7-intel-float': |
| 53 'Skia_Shuttle_Win7_Intel_Float_Release_32', | 53 'Skia_Shuttle_Win7_Intel_Float_Release_32', |
| 54 'base-shuttle-win7-intel-angle': | 54 'base-shuttle-win7-intel-angle': |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 109 if not gm_image_subdir in GM_IMAGE_TO_BASELINE_BUILDER: | 109 if not gm_image_subdir in GM_IMAGE_TO_BASELINE_BUILDER: |
| 110 raise ValueError( | 110 raise ValueError( |
| 111 'Unknown specified gm_image_subdir (%s). Please use one from:\n%s' % ( | 111 'Unknown specified gm_image_subdir (%s). Please use one from:\n%s' % ( |
| 112 gm_image_subdir, str(GM_IMAGE_TO_BASELINE_BUILDER.keys()))) | 112 gm_image_subdir, str(GM_IMAGE_TO_BASELINE_BUILDER.keys()))) |
| 113 | 113 |
| 114 | 114 |
| 115 dest_gsbase = sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE | 115 dest_gsbase = sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE |
| 116 | 116 |
| 117 | 117 |
| 118 # Ensure the right .boto file is used by gsutil. | 118 # Ensure the right .boto file is used by gsutil. |
| 119 if not gs_utils.DoesStorageObjectExist(dest_gsbase): | 119 if not old_gs_utils.DoesStorageObjectExist(dest_gsbase): |
| 120 raise Exception( | 120 raise Exception( |
| 121 'Missing .boto file or .boto does not have the right credentials. Please ' | 121 'Missing .boto file or .boto does not have the right credentials. Please ' |
| 122 'see https://docs.google.com/a/google.com/document/d/1ZzHP6M5qACA9nJnLq' | 122 'see https://docs.google.com/a/google.com/document/d/1ZzHP6M5qACA9nJnLq' |
| 123 'OZr2Hl0rjYqE4yQsQWAfVjKCzs/edit (may have to request access)') | 123 'OZr2Hl0rjYqE4yQsQWAfVjKCzs/edit (may have to request access)') |
| 124 | 124 |
| 125 | 125 |
| 126 for gm_image_subdir in gm_images_seq: | 126 for gm_image_subdir in gm_images_seq: |
| 127 print '\n\n' | 127 print '\n\n' |
| 128 print '======================================================================' | 128 print '======================================================================' |
| 129 print 'Rebaselining %s images' % gm_image_subdir | 129 print 'Rebaselining %s images' % gm_image_subdir |
| 130 print '======================================================================' | 130 print '======================================================================' |
| 131 | 131 |
| 132 builder_name = GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir] | 132 builder_name = GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir] |
| 133 | 133 |
| 134 storage_playback_dirs = playback_dirs.StorageSkpPlaybackDirs( | 134 storage_playback_dirs = playback_dirs.StorageSkpPlaybackDirs( |
| 135 builder_name=builder_name, | 135 builder_name=builder_name, |
| 136 gm_image_subdir=gm_image_subdir, | 136 gm_image_subdir=gm_image_subdir, |
| 137 perf_output_basedir=None) | 137 perf_output_basedir=None) |
| 138 | 138 |
| 139 gm_actual_dir = posixpath.join( | 139 gm_actual_dir = posixpath.join( |
| 140 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) | 140 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) |
| 141 gm_expected_dir = posixpath.join( | 141 gm_expected_dir = posixpath.join( |
| 142 dest_gsbase, storage_playback_dirs.PlaybackGmExpectedDir()) | 142 dest_gsbase, storage_playback_dirs.PlaybackGmExpectedDir()) |
| 143 | 143 |
| 144 print '\n\n=======Throw an Exception if gm_actual_dir does not exist=======' | 144 print '\n\n=======Throw an Exception if gm_actual_dir does not exist=======' |
| 145 if not gs_utils.DoesStorageObjectExist(gm_actual_dir): | 145 if not old_gs_utils.DoesStorageObjectExist(gm_actual_dir): |
| 146 raise Exception("%s does not exist in Google Storage!" % gm_actual_dir) | 146 raise Exception("%s does not exist in Google Storage!" % gm_actual_dir) |
| 147 | 147 |
| 148 print '\n\n=======Wait if gm_actual_dir is being updated=======' | 148 print '\n\n=======Wait if gm_actual_dir is being updated=======' |
| 149 keep_waiting = True | 149 keep_waiting = True |
| 150 while(keep_waiting): | 150 while(keep_waiting): |
| 151 gm_actual_started_timestamp = gs_utils.ReadTimeStampFile( | 151 gm_actual_started_timestamp = old_gs_utils.ReadTimeStampFile( |
| 152 timestamp_file_name=gs_utils.TIMESTAMP_STARTED_FILENAME, | 152 timestamp_file_name=old_gs_utils.TIMESTAMP_STARTED_FILENAME, |
| 153 gs_base=dest_gsbase, | 153 gs_base=dest_gsbase, |
| 154 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) | 154 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) |
| 155 gm_actual_completed_timestamp = gs_utils.ReadTimeStampFile( | 155 gm_actual_completed_timestamp = old_gs_utils.ReadTimeStampFile( |
| 156 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 156 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
| 157 gs_base=dest_gsbase, | 157 gs_base=dest_gsbase, |
| 158 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) | 158 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir()) |
| 159 if gm_actual_started_timestamp != gm_actual_completed_timestamp: | 159 if gm_actual_started_timestamp != gm_actual_completed_timestamp: |
| 160 print '\n\nSleep for a minute since gm_actual_dir is being updated.' | 160 print '\n\nSleep for a minute since gm_actual_dir is being updated.' |
| 161 print 'Please manually exit the script if you want to rerun it later.\n' | 161 print 'Please manually exit the script if you want to rerun it later.\n' |
| 162 time.sleep(60) | 162 time.sleep(60) |
| 163 else: | 163 else: |
| 164 keep_waiting = False | 164 keep_waiting = False |
| 165 | 165 |
| 166 print '\n\n=======Add the REBASELINE_IN_PROGRESS lock file=======' | 166 print '\n\n=======Add the REBASELINE_IN_PROGRESS lock file=======' |
| 167 gs_utils.WriteTimeStampFile( | 167 old_gs_utils.WriteTimeStampFile( |
| 168 timestamp_file_name=( | 168 timestamp_file_name=( |
| 169 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME), | 169 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME), |
| 170 timestamp_value=time.time(), | 170 timestamp_value=time.time(), |
| 171 gs_base=dest_gsbase, | 171 gs_base=dest_gsbase, |
| 172 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir(), | 172 gs_relative_dir=storage_playback_dirs.PlaybackGmActualDir(), |
| 173 gs_acl=PLAYBACK_CANNED_ACL) | 173 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL) |
| 174 | 174 |
| 175 print '\n\n=======Delete contents of gm_expected_dir=======' | 175 print '\n\n=======Delete contents of gm_expected_dir=======' |
| 176 gs_utils.DeleteStorageObject(gm_expected_dir) | 176 old_gs_utils.DeleteStorageObject(gm_expected_dir) |
| 177 | 177 |
| 178 print '\n\n=====Copy all contents from gm_actual_dir to gm_expected_dir======' | 178 print '\n\n=====Copy all contents from gm_actual_dir to gm_expected_dir======' |
| 179 | 179 |
| 180 # Gather list of all files. | 180 # Gather list of all files. |
| 181 gm_actual_contents = gs_utils.ListStorageDirectory( | 181 gm_actual_contents = old_gs_utils.ListStorageDirectory( |
| 182 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) | 182 dest_gsbase, storage_playback_dirs.PlaybackGmActualDir()) |
| 183 | 183 |
| 184 # Remove REBASELINE, TIMESTAMP_* and COMPARISON files from the list. | 184 # Remove REBASELINE, TIMESTAMP_* and COMPARISON files from the list. |
| 185 for file_to_remove in ( | 185 for file_to_remove in ( |
| 186 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME, | 186 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME, |
| 187 gs_utils.TIMESTAMP_STARTED_FILENAME, | 187 old_gs_utils.TIMESTAMP_STARTED_FILENAME, |
| 188 gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 188 old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
| 189 compare_and_upload_webpage_gms.LAST_COMPARISON_FILENAME): | 189 compare_and_upload_webpage_gms.LAST_COMPARISON_FILENAME): |
| 190 gs_file_to_remove = posixpath.join(gm_actual_dir, file_to_remove) | 190 gs_file_to_remove = posixpath.join(gm_actual_dir, file_to_remove) |
| 191 if gs_file_to_remove in gm_actual_contents: | 191 if gs_file_to_remove in gm_actual_contents: |
| 192 gm_actual_contents.remove(gs_file_to_remove) | 192 gm_actual_contents.remove(gs_file_to_remove) |
| 193 | 193 |
| 194 # Copy over files in chunks. | 194 # Copy over files in chunks. |
| 195 # pylint: disable=W0212 | 195 # pylint: disable=W0212 |
| 196 for files_chunk in gs_utils._GetChunks(gm_actual_contents, | 196 for files_chunk in old_gs_utils._GetChunks(gm_actual_contents, |
| 197 gs_utils.FILES_CHUNK): | 197 old_gs_utils.FILES_CHUNK): |
| 198 gsutil = slave_utils.GSUtilSetup() | 198 gsutil = slave_utils.GSUtilSetup() |
| 199 command = ([gsutil, 'cp'] + files_chunk + | 199 command = ([gsutil, 'cp'] + files_chunk + |
| 200 [posixpath.join(gm_expected_dir, '')]) | 200 [posixpath.join(gm_expected_dir, '')]) |
| 201 if chromium_utils.RunCommand(command) != 0: | 201 if chromium_utils.RunCommand(command) != 0: |
| 202 raise Exception( | 202 raise Exception( |
| 203 'Could not upload the chunk to Google Storage! The chunk: %s' | 203 'Could not upload the chunk to Google Storage! The chunk: %s' |
| 204 % files_chunk) | 204 % files_chunk) |
| 205 | 205 |
| 206 print '\n\n=======Delete the REBASELINE_IN_PROGRESS lock file=======' | 206 print '\n\n=======Delete the REBASELINE_IN_PROGRESS lock file=======' |
| 207 gs_utils.DeleteStorageObject( | 207 old_gs_utils.DeleteStorageObject( |
| 208 posixpath.join( | 208 posixpath.join( |
| 209 gm_actual_dir, | 209 gm_actual_dir, |
| 210 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME)) | 210 compare_and_upload_webpage_gms.REBASELINE_IN_PROGRESS_FILENAME)) |
| 211 | 211 |
| 212 print '\n\n=======Update gm_expected_dir timestamp=======' | 212 print '\n\n=======Update gm_expected_dir timestamp=======' |
| 213 gs_utils.WriteTimeStampFile( | 213 old_gs_utils.WriteTimeStampFile( |
| 214 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | 214 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME, |
| 215 timestamp_value=time.time(), | 215 timestamp_value=time.time(), |
| 216 gs_base=dest_gsbase, | 216 gs_base=dest_gsbase, |
| 217 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), | 217 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), |
| 218 gs_acl=PLAYBACK_CANNED_ACL, | 218 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL, |
| 219 local_dir=None) | 219 local_dir=None) |
| 220 | 220 |
| 221 print '\n\n=======Add LAST_REBASELINED_BY file=======' | 221 print '\n\n=======Add LAST_REBASELINED_BY file=======' |
| 222 gs_utils.WriteTimeStampFile( | 222 old_gs_utils.WriteTimeStampFile( |
| 223 timestamp_file_name=gs_utils.LAST_REBASELINED_BY_FILENAME, | 223 timestamp_file_name=old_gs_utils.LAST_REBASELINED_BY_FILENAME, |
| 224 timestamp_value=getpass.getuser(), | 224 timestamp_value=getpass.getuser(), |
| 225 gs_base=dest_gsbase, | 225 gs_base=dest_gsbase, |
| 226 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), | 226 gs_relative_dir=storage_playback_dirs.PlaybackGmExpectedDir(), |
| 227 gs_acl=PLAYBACK_CANNED_ACL, | 227 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL, |
| 228 local_dir=None) | 228 local_dir=None) |
| 229 | 229 |
| 230 | 230 |
| 231 # Submit whitespace change to trigger rebuilds if skia trunk location, svn | 231 # Submit whitespace change to trigger rebuilds if skia trunk location, svn |
| 232 # username and password have been provided. | 232 # username and password have been provided. |
| 233 if commit_whitespace_change: | 233 if commit_whitespace_change: |
| 234 skia_trunk_dir = sys.argv[2] | 234 skia_trunk_dir = sys.argv[2] |
| 235 svn_username = sys.argv[3] | 235 svn_username = sys.argv[3] |
| 236 svn_password = sys.argv[4] | 236 svn_password = sys.argv[4] |
| 237 repo = svn.Svn(skia_trunk_dir, svn_username, svn_password, | 237 repo = svn.Svn(skia_trunk_dir, svn_username, svn_password, |
| 238 additional_svn_flags=['--trust-server-cert', '--no-auth-cache', | 238 additional_svn_flags=['--trust-server-cert', '--no-auth-cache', |
| 239 '--non-interactive']) | 239 '--non-interactive']) |
| 240 whitespace_file = open(os.path.join(skia_trunk_dir, 'whitespace.txt'), 'a') | 240 whitespace_file = open(os.path.join(skia_trunk_dir, 'whitespace.txt'), 'a') |
| 241 try: | 241 try: |
| 242 whitespace_file.write('\n') | 242 whitespace_file.write('\n') |
| 243 finally: | 243 finally: |
| 244 whitespace_file.close() | 244 whitespace_file.close() |
| 245 | 245 |
| 246 print '\n\n=======Submit whitespace change to trigger rebuilds=======' | 246 print '\n\n=======Submit whitespace change to trigger rebuilds=======' |
| 247 builders_to_run = [] | 247 builders_to_run = [] |
| 248 for gm_image_subdir in gm_images_seq: | 248 for gm_image_subdir in gm_images_seq: |
| 249 builders_to_run.append(GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir]) | 249 builders_to_run.append(GM_IMAGE_TO_BASELINE_BUILDER[gm_image_subdir]) |
| 250 run_builders_keyword = '(RunBuilders:%s)' % ','.join(builders_to_run) | 250 run_builders_keyword = '(RunBuilders:%s)' % ','.join(builders_to_run) |
| 251 # pylint: disable=W0212 | 251 # pylint: disable=W0212 |
| 252 repo._RunSvnCommand( | 252 repo._RunSvnCommand( |
| 253 ['commit', '--message', | 253 ['commit', '--message', |
| 254 'Rebaselined webpage image GMs for %s on Google Storage.\n%s' | 254 'Rebaselined webpage image GMs for %s on Google Storage.\n%s' |
| 255 % (gm_images_seq, run_builders_keyword), | 255 % (gm_images_seq, run_builders_keyword), |
| 256 'whitespace.txt']) | 256 'whitespace.txt']) |
| OLD | NEW |