OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python2.7 |
| 2 |
| 3 """greenify.py: standalone script to correct flaky bench expectations. |
| 4 Usage: |
| 5 Copy script to a separate dir outside Skia repo. The script will create a |
| 6 skia dir on the first run to host the repo, and will create/delete temp |
| 7 dirs as needed. |
| 8 ./greenify.py --url <the stdio url from failed CheckForRegressions step> |
| 9 """ |
| 10 |
| 11 import argparse |
| 12 import filecmp |
| 13 import os |
| 14 import re |
| 15 import shutil |
| 16 import subprocess |
| 17 import time |
| 18 import urllib2 |
| 19 |
| 20 |
| 21 # Regular expression for matching exception data. |
| 22 EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) ' |
| 23 'vs (\d+.\d+), ') |
| 24 EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE) |
| 25 |
| 26 |
| 27 def clean_dir(d): |
| 28 if os.path.exists(d): |
| 29 shutil.rmtree(d) |
| 30 os.makedirs(d) |
| 31 |
| 32 def checkout_or_update_skia(repo_dir): |
| 33 status = True |
| 34 old_cwd = os.getcwd() |
| 35 os.chdir(repo_dir) |
| 36 print 'CHECK SKIA REPO...' |
| 37 if subprocess.call(['git', 'pull'], |
| 38 stderr=subprocess.PIPE): |
| 39 print 'Checking out Skia from git, please be patient...' |
| 40 os.chdir(old_cwd) |
| 41 clean_dir(repo_dir) |
| 42 os.chdir(repo_dir) |
| 43 if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch', |
| 44 'https://skia.googlesource.com/skia.git', '.']): |
| 45 status = False |
| 46 subprocess.call(['git', 'checkout', 'master']) |
| 47 subprocess.call(['git', 'pull']) |
| 48 os.chdir(old_cwd) |
| 49 return status |
| 50 |
| 51 def git_commit_expectations(repo_dir, exp_dir, bot, build, commit): |
| 52 commit_msg = """Greenify bench bot %s at build %s |
| 53 |
| 54 TBR=bsalomon@google.com |
| 55 |
| 56 Bypassing trybots: |
| 57 NOTRY=true""" % (bot, build) |
| 58 old_cwd = os.getcwd() |
| 59 os.chdir(repo_dir) |
| 60 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', |
| 61 '--bypass-watchlists', '-m', commit_msg] |
| 62 if commit: |
| 63 upload.append('--use-commit-queue') |
| 64 branch = exp_dir[exp_dir.rfind('/') + 1:] |
| 65 filename = 'bench_expectations_%s.txt' % bot |
| 66 cmds = ([['git', 'checkout', 'master'], |
| 67 ['git', 'pull'], |
| 68 ['git', 'checkout', '-b', branch, '-t', 'origin/master'], |
| 69 ['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'], |
| 70 ['git', 'add', 'expectations/bench/' + filename], |
| 71 ['git', 'commit', '-m', commit_msg], |
| 72 upload, |
| 73 ['git', 'checkout', 'master'], |
| 74 ['git', 'branch', '-D', branch], |
| 75 ]) |
| 76 status = True |
| 77 for cmd in cmds: |
| 78 print 'Running ' + ' '.join(cmd) |
| 79 if subprocess.call(cmd): |
| 80 print 'FAILED. Please check if skia git repo is present.' |
| 81 subprocess.call(['git', 'checkout', 'master']) |
| 82 status = False |
| 83 break |
| 84 os.chdir(old_cwd) |
| 85 return status |
| 86 |
| 87 def delete_dirs(li): |
| 88 for d in li: |
| 89 print 'Deleting directory %s' % d |
| 90 shutil.rmtree(d) |
| 91 |
| 92 def widen_bench_ranges(url, bot, repo_dir, exp_dir): |
| 93 fname = 'bench_expectations_%s.txt' % bot |
| 94 src = os.path.join(repo_dir, 'expectations', 'bench', fname) |
| 95 if not os.path.isfile(src): |
| 96 print 'This bot has no expectations! %s' % bot |
| 97 return False |
| 98 row_dic = {} |
| 99 for l in urllib2.urlopen(url).read().split('\n'): |
| 100 data = EXCEPTION_RE_COMPILED.search(l) |
| 101 if data: |
| 102 row = data.group(1) |
| 103 lb = float(data.group(2)) |
| 104 ub = float(data.group(3)) |
| 105 actual = float(data.group(4)) |
| 106 exp = float(data.group(5)) |
| 107 avg = (actual + exp) / 2 |
| 108 shift = avg - exp |
| 109 lb = lb + shift |
| 110 ub = ub + shift |
| 111 # In case outlier really fluctuates a lot |
| 112 if actual < lb: |
| 113 lb = actual - abs(shift) * 0.1 + 0.5 |
| 114 elif actual > ub: |
| 115 ub = actual + abs(shift) * 0.1 + 0.5 |
| 116 row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub) |
| 117 if not row_dic: |
| 118 print 'NO out-of-range benches found at %s' % url |
| 119 return False |
| 120 |
| 121 changed = 0 |
| 122 li = [] |
| 123 for l in open(src).readlines(): |
| 124 parts = l.strip().split(',') |
| 125 if parts[0].startswith('#') or len(parts) != 5: |
| 126 li.append(l.strip()) |
| 127 continue |
| 128 if ','.join(parts[:2]) in row_dic: |
| 129 li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])]) |
| 130 changed += 1 |
| 131 else: |
| 132 li.append(l.strip()) |
| 133 if not changed: |
| 134 print 'Not in source file:\n' + '\n'.join(row_dic.keys()) |
| 135 return False |
| 136 |
| 137 dst = os.path.join(exp_dir, fname) |
| 138 with open(dst, 'w+') as f: |
| 139 f.write('\n'.join(li)) |
| 140 return True |
| 141 |
| 142 |
| 143 def main(): |
| 144 d = os.path.dirname(os.path.abspath(__file__)) |
| 145 os.chdir(d) |
| 146 if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE): |
| 147 print 'Please copy script to a separate dir outside git repos to use.' |
| 148 return |
| 149 ts_str = '%s' % time.time() |
| 150 exp_dir = os.path.join(d, 'exp' + ts_str) |
| 151 clean_dir(exp_dir) |
| 152 |
| 153 parser = argparse.ArgumentParser() |
| 154 parser.add_argument('--url', |
| 155 help='Broken bench build CheckForRegressions page url.') |
| 156 parser.add_argument('--commit', action='store_true', |
| 157 help='Whether to commit changes automatically.') |
| 158 args = parser.parse_args() |
| 159 repo_dir = os.path.join(d, 'skia') |
| 160 if not os.path.exists(repo_dir): |
| 161 os.makedirs(repo_dir) |
| 162 if not checkout_or_update_skia(repo_dir): |
| 163 print 'ERROR setting up Skia repo at %s' % repo_dir |
| 164 return 1 |
| 165 |
| 166 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py') |
| 167 if not filecmp.cmp(__file__, file_in_repo): |
| 168 shutil.copy(file_in_repo, __file__) |
| 169 print 'Updated this script from repo; please run again.' |
| 170 return |
| 171 |
| 172 if not args.url: |
| 173 raise Exception('Please provide a url with broken CheckForRegressions.') |
| 174 path = args.url.split('/') |
| 175 if len(path) != 11 or not path[6].isdigit(): |
| 176 raise Exception('Unexpected url format: %s' % args.url) |
| 177 bot = path[4] |
| 178 build = path[6] |
| 179 commit = False |
| 180 if args.commit: |
| 181 commit = True |
| 182 |
| 183 if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir): |
| 184 print 'NO bench exceptions found! %s' % args.url |
| 185 elif not git_commit_expectations( |
| 186 repo_dir, exp_dir, bot, build, commit): |
| 187 print 'ERROR uploading expectations using git.' |
| 188 elif not commit: |
| 189 print 'CL created. Please take a look at the link above.' |
| 190 else: |
| 191 print 'New bench baselines should be in CQ now.' |
| 192 delete_dirs([exp_dir]) |
| 193 |
| 194 |
| 195 if __name__ == "__main__": |
| 196 main() |
OLD | NEW |