OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 | |
3 | |
4 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | |
5 # Use of this source code is governed by a BSD-style license that can be | |
6 # found in the LICENSE file. | |
7 | |
8 | |
9 """greenify.py: standalone script to correct flaky bench expectations. | |
10 | |
11 Requires Rietveld credentials on the running machine. | |
12 | |
13 Usage: | |
14 Copy script to a separate dir outside Skia repo. The script will create a | |
15 skia dir on the first run to host the repo, and will create/delete | |
16 temp dirs as needed. | |
17 ./greenify.py --url <the stdio url from failed CheckForRegressions step> | |
18 """ | |
19 | |
20 import argparse | |
21 import filecmp | |
22 import os | |
23 import re | |
24 import shutil | |
25 import subprocess | |
26 import time | |
27 import urllib2 | |
28 | |
29 | |
30 # Regular expression for matching exception data. | |
31 EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) ' | |
32 'vs (\d+.\d+), ') | |
33 EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE) | |
34 | |
35 | |
36 def clean_dir(d): | |
37 if os.path.exists(d): | |
38 shutil.rmtree(d) | |
39 os.makedirs(d) | |
40 | |
41 def checkout_or_update_skia(repo_dir): | |
42 status = True | |
43 old_cwd = os.getcwd() | |
44 os.chdir(repo_dir) | |
45 print 'CHECK SKIA REPO...' | |
46 if subprocess.call(['git', 'pull'], | |
47 stderr=subprocess.PIPE): | |
48 print 'Checking out Skia from git, please be patient...' | |
49 os.chdir(old_cwd) | |
50 clean_dir(repo_dir) | |
51 os.chdir(repo_dir) | |
52 if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch', | |
53 'https://skia.googlesource.com/skia.git', '.']): | |
54 status = False | |
55 subprocess.call(['git', 'checkout', 'master']) | |
56 subprocess.call(['git', 'pull']) | |
57 os.chdir(old_cwd) | |
58 return status | |
59 | |
60 def git_commit_expectations(repo_dir, exp_dir, bot, build, commit): | |
61 commit_msg = """Greenify bench bot %s at build %s | |
62 | |
63 TBR=bsalomon@google.com | |
64 | |
65 Bypassing trybots: | |
66 NOTRY=true""" % (bot, build) | |
67 old_cwd = os.getcwd() | |
68 os.chdir(repo_dir) | |
69 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', | |
70 '--bypass-watchlists', '-m', commit_msg] | |
71 if commit: | |
72 upload.append('--use-commit-queue') | |
73 branch = exp_dir[exp_dir.rfind('/') + 1:] | |
74 filename = 'bench_expectations_%s.txt' % bot | |
75 cmds = ([['git', 'checkout', 'master'], | |
76 ['git', 'pull'], | |
77 ['git', 'checkout', '-b', branch, '-t', 'origin/master'], | |
78 ['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'], | |
79 ['git', 'add', 'expectations/bench/' + filename], | |
80 ['git', 'commit', '-m', commit_msg], | |
81 upload, | |
82 ['git', 'checkout', 'master'], | |
83 ['git', 'branch', '-D', branch], | |
84 ]) | |
85 status = True | |
86 for cmd in cmds: | |
87 print 'Running ' + ' '.join(cmd) | |
88 if subprocess.call(cmd): | |
89 print 'FAILED. Please check if skia git repo is present.' | |
90 subprocess.call(['git', 'checkout', 'master']) | |
91 status = False | |
92 break | |
93 os.chdir(old_cwd) | |
94 return status | |
95 | |
96 def delete_dirs(li): | |
97 for d in li: | |
98 print 'Deleting directory %s' % d | |
99 shutil.rmtree(d) | |
100 | |
101 def widen_bench_ranges(url, bot, repo_dir, exp_dir): | |
102 fname = 'bench_expectations_%s.txt' % bot | |
103 src = os.path.join(repo_dir, 'expectations', 'bench', fname) | |
104 if not os.path.isfile(src): | |
105 print 'This bot has no expectations! %s' % bot | |
106 return False | |
107 row_dic = {} | |
108 for l in urllib2.urlopen(url).read().split('\n'): | |
109 data = EXCEPTION_RE_COMPILED.search(l) | |
110 if data: | |
111 row = data.group(1) | |
112 lb = float(data.group(2)) | |
113 ub = float(data.group(3)) | |
114 actual = float(data.group(4)) | |
115 exp = float(data.group(5)) | |
116 avg = (actual + exp) / 2 | |
117 shift = avg - exp | |
118 lb = lb + shift | |
119 ub = ub + shift | |
120 # In case outlier really fluctuates a lot | |
121 if actual < lb: | |
122 lb = actual - abs(shift) * 0.1 + 0.5 | |
123 elif actual > ub: | |
124 ub = actual + abs(shift) * 0.1 + 0.5 | |
125 row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub) | |
126 if not row_dic: | |
127 print 'NO out-of-range benches found at %s' % url | |
128 return False | |
129 | |
130 changed = 0 | |
131 li = [] | |
132 for l in open(src).readlines(): | |
133 parts = l.strip().split(',') | |
134 if parts[0].startswith('#') or len(parts) != 5: | |
135 li.append(l.strip()) | |
136 continue | |
137 if ','.join(parts[:2]) in row_dic: | |
138 li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])]) | |
139 changed += 1 | |
140 else: | |
141 li.append(l.strip()) | |
142 if not changed: | |
143 print 'Not in source file:\n' + '\n'.join(row_dic.keys()) | |
144 return False | |
145 | |
146 dst = os.path.join(exp_dir, fname) | |
147 with open(dst, 'w+') as f: | |
148 f.write('\n'.join(li)) | |
149 return True | |
150 | |
151 | |
152 def main(): | |
153 d = os.path.dirname(os.path.abspath(__file__)) | |
154 os.chdir(d) | |
155 if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE): | |
156 print 'Please copy script to a separate dir outside git repos to use.' | |
157 return | |
158 ts_str = '%s' % time.time() | |
159 | |
160 parser = argparse.ArgumentParser() | |
161 parser.add_argument('--url', | |
162 help='Broken bench build CheckForRegressions page url.') | |
163 parser.add_argument('--commit', action='store_true', | |
164 help='Whether to commit changes automatically.') | |
165 args = parser.parse_args() | |
166 repo_dir = os.path.join(d, 'skia') | |
167 if not os.path.exists(repo_dir): | |
168 os.makedirs(repo_dir) | |
169 if not checkout_or_update_skia(repo_dir): | |
170 print 'ERROR setting up Skia repo at %s' % repo_dir | |
171 return 1 | |
172 | |
173 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py') | |
174 if not filecmp.cmp(__file__, file_in_repo): | |
175 shutil.copy(file_in_repo, __file__) | |
176 print 'Updated this script from repo; please run again.' | |
177 return | |
178 | |
179 if not args.url: | |
180 raise Exception('Please provide a url with broken CheckForRegressions.') | |
181 path = args.url.split('/') | |
182 if len(path) != 11 or not path[6].isdigit(): | |
183 raise Exception('Unexpected url format: %s' % args.url) | |
184 bot = path[4] | |
185 build = path[6] | |
186 commit = False | |
187 if args.commit: | |
188 commit = True | |
189 | |
190 exp_dir = os.path.join(d, 'exp' + ts_str) | |
191 clean_dir(exp_dir) | |
192 if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir): | |
193 print 'NO bench exceptions found! %s' % args.url | |
194 elif not git_commit_expectations( | |
195 repo_dir, exp_dir, bot, build, commit): | |
196 print 'ERROR uploading expectations using git.' | |
197 elif not commit: | |
198 print 'CL created. Please take a look at the link above.' | |
199 else: | |
200 print 'New bench baselines should be in CQ now.' | |
201 delete_dirs([exp_dir]) | |
202 | |
203 | |
204 if __name__ == "__main__": | |
205 main() | |
OLD | NEW |