| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 | |
| 3 | |
| 4 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | |
| 5 # Use of this source code is governed by a BSD-style license that can be | |
| 6 # found in the LICENSE file. | |
| 7 | |
| 8 | |
| 9 """greenify.py: standalone script to correct flaky bench expectations. | |
| 10 | |
| 11 Requires Rietveld credentials on the running machine. | |
| 12 | |
| 13 Usage: | |
| 14 Copy script to a separate dir outside Skia repo. The script will create a | |
| 15 skia dir on the first run to host the repo, and will create/delete | |
| 16 temp dirs as needed. | |
| 17 ./greenify.py --url <the stdio url from failed CheckForRegressions step> | |
| 18 """ | |
| 19 | |
| 20 import argparse | |
| 21 import filecmp | |
| 22 import os | |
| 23 import re | |
| 24 import shutil | |
| 25 import subprocess | |
| 26 import time | |
| 27 import urllib2 | |
| 28 | |
| 29 | |
| 30 # Regular expression for matching exception data. | |
| 31 EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) ' | |
| 32 'vs (\d+.\d+), ') | |
| 33 EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE) | |
| 34 | |
| 35 | |
| 36 def clean_dir(d): | |
| 37 if os.path.exists(d): | |
| 38 shutil.rmtree(d) | |
| 39 os.makedirs(d) | |
| 40 | |
| 41 def checkout_or_update_skia(repo_dir): | |
| 42 status = True | |
| 43 old_cwd = os.getcwd() | |
| 44 os.chdir(repo_dir) | |
| 45 print 'CHECK SKIA REPO...' | |
| 46 if subprocess.call(['git', 'pull'], | |
| 47 stderr=subprocess.PIPE): | |
| 48 print 'Checking out Skia from git, please be patient...' | |
| 49 os.chdir(old_cwd) | |
| 50 clean_dir(repo_dir) | |
| 51 os.chdir(repo_dir) | |
| 52 if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch', | |
| 53 'https://skia.googlesource.com/skia.git', '.']): | |
| 54 status = False | |
| 55 subprocess.call(['git', 'checkout', 'master']) | |
| 56 subprocess.call(['git', 'pull']) | |
| 57 os.chdir(old_cwd) | |
| 58 return status | |
| 59 | |
| 60 def git_commit_expectations(repo_dir, exp_dir, bot, build, commit): | |
| 61 commit_msg = """Greenify bench bot %s at build %s | |
| 62 | |
| 63 TBR=bsalomon@google.com | |
| 64 | |
| 65 Bypassing trybots: | |
| 66 NOTRY=true""" % (bot, build) | |
| 67 old_cwd = os.getcwd() | |
| 68 os.chdir(repo_dir) | |
| 69 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', | |
| 70 '--bypass-watchlists', '-m', commit_msg] | |
| 71 if commit: | |
| 72 upload.append('--use-commit-queue') | |
| 73 branch = exp_dir[exp_dir.rfind('/') + 1:] | |
| 74 filename = 'bench_expectations_%s.txt' % bot | |
| 75 cmds = ([['git', 'checkout', 'master'], | |
| 76 ['git', 'pull'], | |
| 77 ['git', 'checkout', '-b', branch, '-t', 'origin/master'], | |
| 78 ['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'], | |
| 79 ['git', 'add', 'expectations/bench/' + filename], | |
| 80 ['git', 'commit', '-m', commit_msg], | |
| 81 upload, | |
| 82 ['git', 'checkout', 'master'], | |
| 83 ['git', 'branch', '-D', branch], | |
| 84 ]) | |
| 85 status = True | |
| 86 for cmd in cmds: | |
| 87 print 'Running ' + ' '.join(cmd) | |
| 88 if subprocess.call(cmd): | |
| 89 print 'FAILED. Please check if skia git repo is present.' | |
| 90 subprocess.call(['git', 'checkout', 'master']) | |
| 91 status = False | |
| 92 break | |
| 93 os.chdir(old_cwd) | |
| 94 return status | |
| 95 | |
| 96 def delete_dirs(li): | |
| 97 for d in li: | |
| 98 print 'Deleting directory %s' % d | |
| 99 shutil.rmtree(d) | |
| 100 | |
| 101 def widen_bench_ranges(url, bot, repo_dir, exp_dir): | |
| 102 fname = 'bench_expectations_%s.txt' % bot | |
| 103 src = os.path.join(repo_dir, 'expectations', 'bench', fname) | |
| 104 if not os.path.isfile(src): | |
| 105 print 'This bot has no expectations! %s' % bot | |
| 106 return False | |
| 107 row_dic = {} | |
| 108 for l in urllib2.urlopen(url).read().split('\n'): | |
| 109 data = EXCEPTION_RE_COMPILED.search(l) | |
| 110 if data: | |
| 111 row = data.group(1) | |
| 112 lb = float(data.group(2)) | |
| 113 ub = float(data.group(3)) | |
| 114 actual = float(data.group(4)) | |
| 115 exp = float(data.group(5)) | |
| 116 avg = (actual + exp) / 2 | |
| 117 shift = avg - exp | |
| 118 lb = lb + shift | |
| 119 ub = ub + shift | |
| 120 # In case outlier really fluctuates a lot | |
| 121 if actual < lb: | |
| 122 lb = actual - abs(shift) * 0.1 + 0.5 | |
| 123 elif actual > ub: | |
| 124 ub = actual + abs(shift) * 0.1 + 0.5 | |
| 125 row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub) | |
| 126 if not row_dic: | |
| 127 print 'NO out-of-range benches found at %s' % url | |
| 128 return False | |
| 129 | |
| 130 changed = 0 | |
| 131 li = [] | |
| 132 for l in open(src).readlines(): | |
| 133 parts = l.strip().split(',') | |
| 134 if parts[0].startswith('#') or len(parts) != 5: | |
| 135 li.append(l.strip()) | |
| 136 continue | |
| 137 if ','.join(parts[:2]) in row_dic: | |
| 138 li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])]) | |
| 139 changed += 1 | |
| 140 else: | |
| 141 li.append(l.strip()) | |
| 142 if not changed: | |
| 143 print 'Not in source file:\n' + '\n'.join(row_dic.keys()) | |
| 144 return False | |
| 145 | |
| 146 dst = os.path.join(exp_dir, fname) | |
| 147 with open(dst, 'w+') as f: | |
| 148 f.write('\n'.join(li)) | |
| 149 return True | |
| 150 | |
| 151 | |
| 152 def main(): | |
| 153 d = os.path.dirname(os.path.abspath(__file__)) | |
| 154 os.chdir(d) | |
| 155 if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE): | |
| 156 print 'Please copy script to a separate dir outside git repos to use.' | |
| 157 return | |
| 158 ts_str = '%s' % time.time() | |
| 159 | |
| 160 parser = argparse.ArgumentParser() | |
| 161 parser.add_argument('--url', | |
| 162 help='Broken bench build CheckForRegressions page url.') | |
| 163 parser.add_argument('--commit', action='store_true', | |
| 164 help='Whether to commit changes automatically.') | |
| 165 args = parser.parse_args() | |
| 166 repo_dir = os.path.join(d, 'skia') | |
| 167 if not os.path.exists(repo_dir): | |
| 168 os.makedirs(repo_dir) | |
| 169 if not checkout_or_update_skia(repo_dir): | |
| 170 print 'ERROR setting up Skia repo at %s' % repo_dir | |
| 171 return 1 | |
| 172 | |
| 173 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py') | |
| 174 if not filecmp.cmp(__file__, file_in_repo): | |
| 175 shutil.copy(file_in_repo, __file__) | |
| 176 print 'Updated this script from repo; please run again.' | |
| 177 return | |
| 178 | |
| 179 if not args.url: | |
| 180 raise Exception('Please provide a url with broken CheckForRegressions.') | |
| 181 path = args.url.split('/') | |
| 182 if len(path) != 11 or not path[6].isdigit(): | |
| 183 raise Exception('Unexpected url format: %s' % args.url) | |
| 184 bot = path[4] | |
| 185 build = path[6] | |
| 186 commit = False | |
| 187 if args.commit: | |
| 188 commit = True | |
| 189 | |
| 190 exp_dir = os.path.join(d, 'exp' + ts_str) | |
| 191 clean_dir(exp_dir) | |
| 192 if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir): | |
| 193 print 'NO bench exceptions found! %s' % args.url | |
| 194 elif not git_commit_expectations( | |
| 195 repo_dir, exp_dir, bot, build, commit): | |
| 196 print 'ERROR uploading expectations using git.' | |
| 197 elif not commit: | |
| 198 print 'CL created. Please take a look at the link above.' | |
| 199 else: | |
| 200 print 'New bench baselines should be in CQ now.' | |
| 201 delete_dirs([exp_dir]) | |
| 202 | |
| 203 | |
| 204 if __name__ == "__main__": | |
| 205 main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 801373003:
Delete old bench expectation code. (Closed)
Base URL: https://skia.googlesource.com/skia.git@master