| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 | |
| 7 """rebase.py: standalone script to batch update bench expectations. | |
| 8 | |
| 9 Requires gsutil to access gs://chromium-skia-gm and Rietveld credentials. | |
| 10 | |
| 11 Usage: | |
| 12 Copy script to a separate dir outside Skia repo. The script will create a | |
| 13 skia dir on the first run to host the repo, and will create/delete | |
| 14 temp dirs as needed. | |
| 15 ./rebase.py --githash <githash prefix to use for getting bench data> | |
| 16 """ | |
| 17 | |
| 18 | |
| 19 import argparse | |
| 20 import filecmp | |
| 21 import os | |
| 22 import re | |
| 23 import shutil | |
| 24 import subprocess | |
| 25 import time | |
| 26 import urllib2 | |
| 27 | |
| 28 | |
| 29 # googlesource url that has most recent Skia git hash info. | |
| 30 SKIA_GIT_HEAD_URL = 'https://skia.googlesource.com/skia/+log/HEAD' | |
| 31 | |
| 32 # Google Storage bench file prefix. | |
| 33 GS_PREFIX = 'gs://chromium-skia-gm/perfdata' | |
| 34 | |
| 35 # Regular expression for matching githash data. | |
| 36 HA_RE = '<a href="/skia/\+/([0-9a-f]+)">' | |
| 37 HA_RE_COMPILED = re.compile(HA_RE) | |
| 38 | |
| 39 | |
| 40 def get_git_hashes(): | |
| 41 print 'Getting recent git hashes...' | |
| 42 hashes = HA_RE_COMPILED.findall( | |
| 43 urllib2.urlopen(SKIA_GIT_HEAD_URL).read()) | |
| 44 | |
| 45 return hashes | |
| 46 | |
| 47 def filter_file(f): | |
| 48 if f.find('_msaa') > 0 or f.find('_record') > 0: | |
| 49 return True | |
| 50 | |
| 51 return False | |
| 52 | |
| 53 def clean_dir(d): | |
| 54 if os.path.exists(d): | |
| 55 shutil.rmtree(d) | |
| 56 os.makedirs(d) | |
| 57 | |
| 58 def get_gs_filelist(p, h): | |
| 59 print 'Looking up for the closest bench files in Google Storage...' | |
| 60 proc = subprocess.Popen(['gsutil', 'ls', | |
| 61 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*'])], | |
| 62 stdout=subprocess.PIPE) | |
| 63 out, err = proc.communicate() | |
| 64 if err or not out: | |
| 65 return [] | |
| 66 return [i for i in out.strip().split('\n') if not filter_file(i)] | |
| 67 | |
| 68 def download_gs_files(p, h, gs_dir): | |
| 69 print 'Downloading raw bench files from Google Storage...' | |
| 70 proc = subprocess.Popen(['gsutil', 'cp', | |
| 71 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*']), | |
| 72 '%s/%s' % (gs_dir, p)], | |
| 73 stdout=subprocess.PIPE) | |
| 74 out, err = proc.communicate() | |
| 75 if err: | |
| 76 clean_dir(gs_dir) | |
| 77 return False | |
| 78 files = 0 | |
| 79 for f in os.listdir(os.path.join(gs_dir, p)): | |
| 80 if filter_file(f): | |
| 81 os.remove(os.path.join(gs_dir, p, f)) | |
| 82 else: | |
| 83 files += 1 | |
| 84 if files: | |
| 85 return True | |
| 86 return False | |
| 87 | |
| 88 def get_expectations_dict(f): | |
| 89 """Given an expectations file f, returns a dictionary of data.""" | |
| 90 # maps row_key to (expected, lower_bound, upper_bound) float tuple. | |
| 91 dic = {} | |
| 92 for l in open(f).readlines(): | |
| 93 line_parts = l.strip().split(',') | |
| 94 if line_parts[0].startswith('#') or len(line_parts) != 5: | |
| 95 continue | |
| 96 dic[','.join(line_parts[:2])] = (float(line_parts[2]), float(line_parts[3]), | |
| 97 float(line_parts[4])) | |
| 98 | |
| 99 return dic | |
| 100 | |
| 101 def calc_expectations(p, h, gs_dir, exp_dir, repo_dir, extra_dir, extra_hash): | |
| 102 exp_filename = 'bench_expectations_%s.txt' % p | |
| 103 exp_fullname = os.path.join(exp_dir, exp_filename) | |
| 104 proc = subprocess.Popen(['python', 'skia/bench/gen_bench_expectations.py', | |
| 105 '-r', h, '-b', p, '-d', os.path.join(gs_dir, p), '-o', exp_fullname], | |
| 106 stdout=subprocess.PIPE) | |
| 107 out, err = proc.communicate() | |
| 108 if err: | |
| 109 print 'ERR_CALCULATING_EXPECTATIONS: ' + err | |
| 110 return False | |
| 111 print 'CALCULATED_EXPECTATIONS: ' + out | |
| 112 if extra_dir: # Adjust data with the ones in extra_dir | |
| 113 print 'USE_EXTRA_DATA_FOR_ADJUSTMENT.' | |
| 114 proc = subprocess.Popen(['python', 'skia/bench/gen_bench_expectations.py', | |
| 115 '-r', extra_hash, '-b', p, '-d', os.path.join(extra_dir, p), '-o', | |
| 116 os.path.join(extra_dir, exp_filename)], | |
| 117 stdout=subprocess.PIPE) | |
| 118 out, err = proc.communicate() | |
| 119 if err: | |
| 120 print 'ERR_CALCULATING_EXTRA_EXPECTATIONS: ' + err | |
| 121 return False | |
| 122 extra_dic = get_expectations_dict(os.path.join(extra_dir, exp_filename)) | |
| 123 output_lines = [] | |
| 124 for l in open(exp_fullname).readlines(): | |
| 125 parts = l.strip().split(',') | |
| 126 if parts[0].startswith('#') or len(parts) != 5: | |
| 127 output_lines.append(l.strip()) | |
| 128 continue | |
| 129 key = ','.join(parts[:2]) | |
| 130 if key in extra_dic: | |
| 131 exp, lb, ub = (float(parts[2]), float(parts[3]), float(parts[4])) | |
| 132 alt, _, _ = extra_dic[key] | |
| 133 avg = (exp + alt) / 2 | |
| 134 # Keeps the extra range in lower/upper bounds from two actual values. | |
| 135 new_lb = min(exp, alt) - (exp - lb) | |
| 136 new_ub = max(exp, alt) + (ub - exp) | |
| 137 output_lines.append('%s,%.2f,%.2f,%.2f' % (key, avg, new_lb, new_ub)) | |
| 138 else: | |
| 139 output_lines.append(l.strip()) | |
| 140 with open(exp_fullname, 'w') as f: | |
| 141 f.write('\n'.join(output_lines)) | |
| 142 | |
| 143 repo_file = os.path.join(repo_dir, 'expectations', 'bench', exp_filename) | |
| 144 if (os.path.isfile(repo_file) and | |
| 145 filecmp.cmp(repo_file, os.path.join(exp_dir, exp_filename))): | |
| 146 print 'NO CHANGE ON %s' % repo_file | |
| 147 return False | |
| 148 return True | |
| 149 | |
| 150 def checkout_or_update_skia(repo_dir): | |
| 151 status = True | |
| 152 old_cwd = os.getcwd() | |
| 153 os.chdir(repo_dir) | |
| 154 print 'CHECK SKIA REPO...' | |
| 155 if subprocess.call(['git', 'pull'], | |
| 156 stderr=subprocess.PIPE): | |
| 157 print 'Checking out Skia from git, please be patient...' | |
| 158 os.chdir(old_cwd) | |
| 159 clean_dir(repo_dir) | |
| 160 os.chdir(repo_dir) | |
| 161 if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch', | |
| 162 'https://skia.googlesource.com/skia.git', '.']): | |
| 163 status = False | |
| 164 subprocess.call(['git', 'checkout', 'master']) | |
| 165 subprocess.call(['git', 'pull']) | |
| 166 os.chdir(old_cwd) | |
| 167 return status | |
| 168 | |
| 169 def git_commit_expectations(repo_dir, exp_dir, update_li, h, commit, | |
| 170 extra_hash): | |
| 171 if extra_hash: | |
| 172 extra_hash = ', adjusted with ' + extra_hash | |
| 173 commit_msg = """manual bench rebase after %s%s | |
| 174 | |
| 175 TBR=robertphillips@google.com | |
| 176 | |
| 177 Bypassing trybots: | |
| 178 NOTRY=true""" % (h, extra_hash) | |
| 179 old_cwd = os.getcwd() | |
| 180 os.chdir(repo_dir) | |
| 181 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', | |
| 182 '--bypass-watchlists', '-m', commit_msg] | |
| 183 branch = exp_dir.split('/')[-1] | |
| 184 if commit: | |
| 185 upload.append('--use-commit-queue') | |
| 186 cmds = ([['git', 'checkout', 'master'], | |
| 187 ['git', 'pull'], | |
| 188 ['git', 'checkout', '-b', branch, '-t', 'origin/master']] + | |
| 189 [['cp', '%s/%s' % (exp_dir, f), 'expectations/bench'] for f in | |
| 190 update_li] + | |
| 191 [['git', 'add'] + ['expectations/bench/%s' % i for i in update_li], | |
| 192 ['git', 'commit', '-m', commit_msg], | |
| 193 upload, | |
| 194 ['git', 'checkout', 'master'], | |
| 195 ['git', 'branch', '-D', branch], | |
| 196 ]) | |
| 197 status = True | |
| 198 for cmd in cmds: | |
| 199 print 'Running ' + ' '.join(cmd) | |
| 200 if subprocess.call(cmd): | |
| 201 print 'FAILED. Please check if skia git repo is present.' | |
| 202 subprocess.call(['git', 'checkout', 'master']) | |
| 203 status = False | |
| 204 break | |
| 205 os.chdir(old_cwd) | |
| 206 return status | |
| 207 | |
| 208 def delete_dirs(li): | |
| 209 for d in li: | |
| 210 print 'Deleting directory %s' % d | |
| 211 shutil.rmtree(d) | |
| 212 | |
| 213 | |
| 214 def main(): | |
| 215 d = os.path.dirname(os.path.abspath(__file__)) | |
| 216 os.chdir(d) | |
| 217 if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE): | |
| 218 print 'Please copy script to a separate dir outside git repos to use.' | |
| 219 return | |
| 220 parser = argparse.ArgumentParser() | |
| 221 parser.add_argument('--githash', | |
| 222 help=('Githash prefix (7+ chars) to rebaseline to. If ' | |
| 223 'a second one is supplied after comma, and it has ' | |
| 224 'corresponding bench data, will shift the range ' | |
| 225 'center to the average of two expected values.')) | |
| 226 parser.add_argument('--bots', | |
| 227 help=('Comma-separated list of bots to work on. If no ' | |
| 228 'matching bots are found in the list, will default ' | |
| 229 'to processing all bots.')) | |
| 230 parser.add_argument('--commit', action='store_true', | |
| 231 help='Whether to commit changes automatically.') | |
| 232 args = parser.parse_args() | |
| 233 | |
| 234 repo_dir = os.path.join(d, 'skia') | |
| 235 if not os.path.exists(repo_dir): | |
| 236 os.makedirs(repo_dir) | |
| 237 if not checkout_or_update_skia(repo_dir): | |
| 238 print 'ERROR setting up Skia repo at %s' % repo_dir | |
| 239 return 1 | |
| 240 | |
| 241 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/rebase.py') | |
| 242 if not filecmp.cmp(__file__, file_in_repo): | |
| 243 shutil.copy(file_in_repo, __file__) | |
| 244 print 'Updated this script from repo; please run again.' | |
| 245 return | |
| 246 | |
| 247 all_platforms = [] # Find existing list of platforms with expectations. | |
| 248 for item in os.listdir(os.path.join(d, 'skia/expectations/bench')): | |
| 249 all_platforms.append( | |
| 250 item.replace('bench_expectations_', '').replace('.txt', '')) | |
| 251 | |
| 252 platforms = [] | |
| 253 # If at least one given bot is in all_platforms, use list of valid args.bots. | |
| 254 if args.bots: | |
| 255 bots = args.bots.strip().split(',') | |
| 256 for bot in bots: | |
| 257 if bot in all_platforms: # Filters platforms with given bot list. | |
| 258 platforms.append(bot) | |
| 259 if not platforms: # Include all existing platforms with expectations. | |
| 260 platforms = all_platforms | |
| 261 | |
| 262 if not args.githash or len(args.githash) < 7: | |
| 263 raise Exception('Please provide --githash with a longer prefix (7+).') | |
| 264 githashes = args.githash.strip().split(',') | |
| 265 if len(githashes[0]) < 7: | |
| 266 raise Exception('Please provide --githash with longer prefixes (7+).') | |
| 267 commit = False | |
| 268 if args.commit: | |
| 269 commit = True | |
| 270 rebase_hash = githashes[0][:7] | |
| 271 extra_hash = '' | |
| 272 if len(githashes) == 2: | |
| 273 extra_hash = githashes[1][:7] | |
| 274 hashes = get_git_hashes() | |
| 275 short_hashes = [h[:7] for h in hashes] | |
| 276 if (rebase_hash not in short_hashes or | |
| 277 (extra_hash and extra_hash not in short_hashes) or | |
| 278 rebase_hash == extra_hash): | |
| 279 raise Exception('Provided --githashes not found, or identical!') | |
| 280 if extra_hash: | |
| 281 extra_hash = hashes[short_hashes.index(extra_hash)] | |
| 282 hashes = hashes[:short_hashes.index(rebase_hash) + 1] | |
| 283 update_li = [] | |
| 284 | |
| 285 ts_str = '%s' % time.time() | |
| 286 gs_dir = os.path.join(d, 'gs' + ts_str) | |
| 287 exp_dir = os.path.join(d, 'exp' + ts_str) | |
| 288 extra_dir = os.path.join(d, 'extra' + ts_str) | |
| 289 clean_dir(gs_dir) | |
| 290 clean_dir(exp_dir) | |
| 291 clean_dir(extra_dir) | |
| 292 for p in platforms: | |
| 293 clean_dir(os.path.join(gs_dir, p)) | |
| 294 clean_dir(os.path.join(extra_dir, p)) | |
| 295 hash_to_use = '' | |
| 296 for h in reversed(hashes): | |
| 297 li = get_gs_filelist(p, h) | |
| 298 if not len(li): # no data | |
| 299 continue | |
| 300 if download_gs_files(p, h, gs_dir): | |
| 301 print 'Copied %s/%s' % (p, h) | |
| 302 hash_to_use = h | |
| 303 break | |
| 304 else: | |
| 305 print 'DOWNLOAD BENCH FAILED %s/%s' % (p, h) | |
| 306 break | |
| 307 if hash_to_use: | |
| 308 if extra_hash and download_gs_files(p, extra_hash, extra_dir): | |
| 309 print 'Copied extra data %s/%s' % (p, extra_hash) | |
| 310 if calc_expectations(p, h, gs_dir, exp_dir, repo_dir, extra_dir, | |
| 311 extra_hash): | |
| 312 update_li.append('bench_expectations_%s.txt' % p) | |
| 313 elif calc_expectations(p, h, gs_dir, exp_dir, repo_dir, '', ''): | |
| 314 update_li.append('bench_expectations_%s.txt' % p) | |
| 315 if not update_li: | |
| 316 print 'No bench data to update after %s!' % args.githash | |
| 317 elif not git_commit_expectations( | |
| 318 repo_dir, exp_dir, update_li, rebase_hash, commit, extra_hash): | |
| 319 print 'ERROR uploading expectations using git.' | |
| 320 elif not commit: | |
| 321 print 'CL created. Please take a look at the link above.' | |
| 322 else: | |
| 323 print 'New bench baselines should be in CQ now.' | |
| 324 delete_dirs([gs_dir, exp_dir, extra_dir]) | |
| 325 | |
| 326 | |
| 327 if __name__ == "__main__": | |
| 328 main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 801373003:
Delete old bench expectation code. (Closed)
Base URL: https://skia.googlesource.com/skia.git@master