| OLD | NEW |
|---|
| (Empty) |
| 1 ''' | |
| 2 Downloads the actual gm results most recently generated by the Skia buildbots, | |
| 3 and adds any new ones to SVN control. | |
| 4 | |
| 5 Launch with --help to see more information. | |
| 6 | |
| 7 | |
| 8 Copyright 2011 Google Inc. | |
| 9 | |
| 10 Use of this source code is governed by a BSD-style license that can be | |
| 11 found in the LICENSE file. | |
| 12 ''' | |
| 13 | |
| 14 # common Python modules | |
| 15 import fnmatch | |
| 16 import optparse | |
| 17 import os | |
| 18 import re | |
| 19 import shutil | |
| 20 import sys | |
| 21 import tempfile | |
| 22 | |
| 23 # modules declared within this same directory | |
| 24 import compare_baselines | |
| 25 import svn | |
| 26 | |
| 27 USAGE_STRING = 'Usage: %s [options] [baseline_subdir]...' | |
| 28 HELP_STRING = ''' | |
| 29 | |
| 30 Downloads the actual gm results most recently generated by the Skia buildbots, | |
| 31 and adds any new ones to SVN control. | |
| 32 | |
| 33 If no baseline_subdir is given, then this tool will download the most-recently | |
| 34 generated actual gm results for ALL platforms. | |
| 35 | |
| 36 ''' + compare_baselines.HOWTO_STRING | |
| 37 | |
| 38 # Base URL of SVN repository where buildbots store actual gm image results. | |
| 39 GM_ACTUAL_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual' | |
| 40 | |
| 41 # GM baseline image URL in regular Skia SVN repository | |
| 42 GM_BASELINE_URL = 'https://skia.googlecode.com/svn/gm-expected' | |
| 43 | |
| 44 GM_EXPECTED_DIR = 'gm-expected' | |
| 45 | |
| 46 OPTION_ADD_NEW_FILES = '--add-new-files' | |
| 47 OPTION_BUILDER_SUFFIX = '--builder-suffix' | |
| 48 DEFAULT_BUILDER_SUFFIX = '32' | |
| 49 OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods' | |
| 50 | |
| 51 def GetLatestResultsSvnUrl(svn, baseline_subdir, builder_suffix): | |
| 52 """Return SVN URL from which we can check out the MOST RECENTLY generated im
ages for this | |
| 53 baseline type. | |
| 54 | |
| 55 @param svn an Svn object we can use to call ListSubdirs() | |
| 56 @param baseline_subdir indicates which platform we want images for | |
| 57 @param builder_suffix if multiple builders uploaded actual GM images for thi
s baseline type, | |
| 58 choose the one whose builder_name matches this suffix | |
| 59 """ | |
| 60 root_url = '%s/%s' % (GM_ACTUAL_URL, baseline_subdir) | |
| 61 subdirs = sorted(svn.ListSubdirs(root_url)) | |
| 62 num_subdirs = len(subdirs) | |
| 63 print('Within actual-results root URL %s, found these %d subdirs (presumably
builder_names): %s' | |
| 64 % (root_url, num_subdirs, subdirs)) | |
| 65 | |
| 66 selected_subdir = None | |
| 67 if num_subdirs == 0: | |
| 68 print 'Found no builder_name subdirs, so reading actual images from the
root_url itself.' | |
| 69 return root_url | |
| 70 elif num_subdirs == 1: | |
| 71 selected_subdir = subdirs[0] | |
| 72 print 'Found exactly one subdir in actual-results root_url: %s' % select
ed_subdir | |
| 73 else: | |
| 74 for possible_subdir in subdirs: | |
| 75 if possible_subdir.endswith(builder_suffix): | |
| 76 selected_subdir = possible_subdir | |
| 77 print 'Selected the first subdir ending in "%s": %s' % ( | |
| 78 builder_suffix, selected_subdir) | |
| 79 break | |
| 80 | |
| 81 if selected_subdir: | |
| 82 return '%s/%s/%s' % (root_url, selected_subdir, baseline_subdir) | |
| 83 else: | |
| 84 raise Exception('none of these subdirs of %s ended in "%s": %s' % ( | |
| 85 root_url, builder_suffix, subdirs)) | |
| 86 | |
| 87 def GetBaselineSvnUrl(baseline_subdir): | |
| 88 """Return SVN URL from which we can check out the baseline images for this | |
| 89 baseline type. | |
| 90 | |
| 91 @param baseline_subdir indicates which platform we want baselines for | |
| 92 """ | |
| 93 return '%s/%s' % (GM_BASELINE_URL, baseline_subdir) | |
| 94 | |
| 95 def CopyMatchingFiles(source_dir, dest_dir, filename_pattern, only_copy_updates=
False): | |
| 96 """Copy all files from source_dir that match filename_pattern, and save them
(with their | |
| 97 original filenames) in dest_dir. | |
| 98 | |
| 99 @param source_dir | |
| 100 @param dest_dir where to save the copied files | |
| 101 @param filename_pattern only copy files that match this Unix-style filename | |
| 102 pattern (e.g., '*.jpg') | |
| 103 @param only_copy_updates if True, only copy files that are already present i
n dest_dir | |
| 104 """ | |
| 105 all_filenames = os.listdir(source_dir) | |
| 106 matching_filenames = fnmatch.filter(all_filenames, filename_pattern) | |
| 107 for filename in matching_filenames: | |
| 108 source_path = os.path.join(source_dir, filename) | |
| 109 dest_path = os.path.join(dest_dir, filename) | |
| 110 if only_copy_updates and not os.path.isfile(dest_path): | |
| 111 continue | |
| 112 shutil.copyfile(source_path, dest_path) | |
| 113 | |
| 114 def DownloadBaselinesForOnePlatform(baseline_subdir): | |
| 115 """Download most recently generated baseline images for a single platform, | |
| 116 and add any new ones to SVN control. | |
| 117 | |
| 118 @param baseline_subdir | |
| 119 """ | |
| 120 # Create repo_to_modify to handle the SVN repository we will add files to. | |
| 121 gm_dir = os.path.join(os.pardir, GM_EXPECTED_DIR) # Shouldn't assume we're i
n trunk... | |
| 122 try: | |
| 123 os.makedirs(gm_dir) | |
| 124 except: | |
| 125 pass | |
| 126 repo_to_modify = svn.Svn(gm_dir) | |
| 127 repo_to_modify.Checkout(GetBaselineSvnUrl(baseline_subdir), baseline_subdir) | |
| 128 | |
| 129 # If there are any locally modified files in that directory, exit | |
| 130 # (so that we don't risk overwriting the user's previous work). | |
| 131 new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles() | |
| 132 if not options.ignore_local_mods: | |
| 133 if new_and_modified_files: | |
| 134 raise Exception('Exiting because there are already new and/or ' | |
| 135 'modified files in %s. To continue in spite of ' | |
| 136 'that, run with %s option.' % ( | |
| 137 baseline_subdir, OPTION_IGNORE_LOCAL_MODS)) | |
| 138 | |
| 139 # Download actual gm images into a separate repo in a temporary directory. | |
| 140 actual_dir = tempfile.mkdtemp() | |
| 141 actual_repo = svn.Svn(actual_dir) | |
| 142 print 'Using %s as a temp dir' % actual_dir | |
| 143 actual_url = GetLatestResultsSvnUrl(svn=actual_repo, baseline_subdir=baselin
e_subdir, | |
| 144 builder_suffix=options.builder_suffix) | |
| 145 print 'Reading actual buildbot GM results from %s' % actual_url | |
| 146 actual_repo.Checkout(actual_url, '.') | |
| 147 | |
| 148 # Copy any of those files we are interested in into repo_to_modify, | |
| 149 # and then delete the temporary directory. | |
| 150 CopyMatchingFiles(source_dir=actual_dir, | |
| 151 dest_dir=os.path.join(gm_dir, baseline_subdir), | |
| 152 filename_pattern='*.png', | |
| 153 only_copy_updates=(not options.add_new_files)) | |
| 154 shutil.rmtree(actual_dir) | |
| 155 actual_repo = None | |
| 156 | |
| 157 # Add any new files to SVN control (if we are running with add_new_files). | |
| 158 if options.add_new_files: | |
| 159 new_files = repo_to_modify.GetNewFiles() | |
| 160 if new_files: | |
| 161 repo_to_modify.AddFiles(sorted(new_files)) | |
| 162 | |
| 163 # Set the mimetype property on any new/modified image files in | |
| 164 # baseline_subdir. (We used to set the mimetype property on *all* image | |
| 165 # files in the directory, even those whose content wasn't changing, | |
| 166 # but that caused confusion. See | |
| 167 # http://code.google.com/p/skia/issues/detail?id=618 .) | |
| 168 modified_files = repo_to_modify.GetNewAndModifiedFiles() | |
| 169 repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')), | |
| 170 svn.PROPERTY_MIMETYPE, 'image/png') | |
| 171 repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')), | |
| 172 svn.PROPERTY_MIMETYPE, 'application/pdf') | |
| 173 | |
| 174 def RaiseUsageException(): | |
| 175 raise Exception('%s\nRun with --help for more detail.' % ( | |
| 176 USAGE_STRING % __file__)) | |
| 177 | |
| 178 def Main(options, args): | |
| 179 """Allow other scripts to call this script with fake command-line args. | |
| 180 """ | |
| 181 # If no platforms are specified, do 'em all. | |
| 182 num_args = len(args) | |
| 183 if num_args == 0: | |
| 184 # TODO(epoger): automate the default set of platforms. We want to ensure | |
| 185 # that the user gets all of the platforms that the bots are running, | |
| 186 # not just whatever subdirectories he happens to have checked out... | |
| 187 # See http://code.google.com/p/skia/issues/detail?id=678 | |
| 188 # Now that we have added Svn.ListSubdirs(), we should be able to do this | |
| 189 # pretty easily... | |
| 190 # | |
| 191 # For now, I generate this list using these Unix commands: | |
| 192 # svn ls http://skia.googlecode.com/svn/gm-expected | grep ^base | sort
>/tmp/baselines | |
| 193 # svn ls http://skia-autogen.googlecode.com/svn/gm-actual | grep ^base |
sort >/tmp/actual | |
| 194 # comm -1 -2 /tmp/baselines /tmp/actual | |
| 195 args = [ | |
| 196 'base-android-galaxy-nexus', | |
| 197 'base-android-nexus-7', | |
| 198 'base-android-nexus-s', | |
| 199 'base-android-xoom', | |
| 200 'base-macmini', | |
| 201 'base-macmini-lion-float', | |
| 202 'base-shuttle-win7-intel-float', | |
| 203 'base-shuttle_ubuntu12_ati5770', | |
| 204 'base-shuttle-win7-intel-angle', | |
| 205 'base-shuttle-win7-intel-directwrite', | |
| 206 ] | |
| 207 | |
| 208 # Trim all subdir names. | |
| 209 baseline_subdirs = [] | |
| 210 for arg in args: | |
| 211 baseline_subdirs.append(arg.rstrip(os.sep)) | |
| 212 | |
| 213 # Process the subdirs, one at a time. | |
| 214 for baseline_subdir in baseline_subdirs: | |
| 215 DownloadBaselinesForOnePlatform(baseline_subdir=baseline_subdir) | |
| 216 | |
| 217 if __name__ == '__main__': | |
| 218 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) | |
| 219 parser.add_option(OPTION_ADD_NEW_FILES, | |
| 220 action='store_true', default=False, | |
| 221 help='in addition to downloading new versions of ' | |
| 222 'existing baselines, also download baselines that are ' | |
| 223 'not under SVN control yet') | |
| 224 parser.add_option(OPTION_BUILDER_SUFFIX, | |
| 225 action='store', type='string', default=DEFAULT_BUILDER_SUF
FIX, | |
| 226 help='if multiple builders have uploaded actual GM images
' | |
| 227 'for this platform, download the images uploaded by the ' | |
| 228 'builder whose name ends in this suffix; defaults to ' | |
| 229 '"%s".' % DEFAULT_BUILDER_SUFFIX) | |
| 230 parser.add_option(OPTION_IGNORE_LOCAL_MODS, | |
| 231 action='store_true', default=False, | |
| 232 help='allow tool to run even if there are already ' | |
| 233 'local modifications in the baseline_subdir') | |
| 234 (options, args) = parser.parse_args() | |
| 235 Main(options, args) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 23082002:
Delete compare_baselines.py and download_baselines.py ; we use rebaseline.py now (Closed)
Base URL: http://skia.googlecode.com/svn/trunk/