OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 | |
7 """rebase.py: standalone script to batch update bench expectations. | |
8 | |
9 Requires gsutil to access gs://chromium-skia-gm and Rietveld credentials. | |
10 | |
11 Usage: | |
12 Copy script to a separate dir outside Skia repo. The script will create a | |
13 skia dir on the first run to host the repo, and will create/delete | |
14 temp dirs as needed. | |
15 ./rebase.py --githash <githash prefix to use for getting bench data> | |
16 """ | |
17 | |
18 | |
19 import argparse | |
20 import filecmp | |
21 import os | |
22 import re | |
23 import shutil | |
24 import subprocess | |
25 import time | |
26 import urllib2 | |
27 | |
28 | |
29 # googlesource url that has most recent Skia git hash info. | |
30 SKIA_GIT_HEAD_URL = 'https://skia.googlesource.com/skia/+log/HEAD' | |
31 | |
32 # Google Storage bench file prefix. | |
33 GS_PREFIX = 'gs://chromium-skia-gm/perfdata' | |
34 | |
35 # Regular expression for matching githash data. | |
36 HA_RE = '<a href="/skia/\+/([0-9a-f]+)">' | |
37 HA_RE_COMPILED = re.compile(HA_RE) | |
38 | |
39 | |
40 def get_git_hashes(): | |
41 print 'Getting recent git hashes...' | |
42 hashes = HA_RE_COMPILED.findall( | |
43 urllib2.urlopen(SKIA_GIT_HEAD_URL).read()) | |
44 | |
45 return hashes | |
46 | |
47 def filter_file(f): | |
48 if f.find('_msaa') > 0 or f.find('_record') > 0: | |
49 return True | |
50 | |
51 return False | |
52 | |
53 def clean_dir(d): | |
54 if os.path.exists(d): | |
55 shutil.rmtree(d) | |
56 os.makedirs(d) | |
57 | |
58 def get_gs_filelist(p, h): | |
59 print 'Looking up for the closest bench files in Google Storage...' | |
60 proc = subprocess.Popen(['gsutil', 'ls', | |
61 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*'])], | |
62 stdout=subprocess.PIPE) | |
63 out, err = proc.communicate() | |
64 if err or not out: | |
65 return [] | |
66 return [i for i in out.strip().split('\n') if not filter_file(i)] | |
67 | |
68 def download_gs_files(p, h, gs_dir): | |
69 print 'Downloading raw bench files from Google Storage...' | |
70 proc = subprocess.Popen(['gsutil', 'cp', | |
71 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*']), | |
72 '%s/%s' % (gs_dir, p)], | |
73 stdout=subprocess.PIPE) | |
74 out, err = proc.communicate() | |
75 if err: | |
76 clean_dir(gs_dir) | |
77 return False | |
78 files = 0 | |
79 for f in os.listdir(os.path.join(gs_dir, p)): | |
80 if filter_file(f): | |
81 os.remove(os.path.join(gs_dir, p, f)) | |
82 else: | |
83 files += 1 | |
84 if files: | |
85 return True | |
86 return False | |
87 | |
88 def get_expectations_dict(f): | |
89 """Given an expectations file f, returns a dictionary of data.""" | |
90 # maps row_key to (expected, lower_bound, upper_bound) float tuple. | |
91 dic = {} | |
92 for l in open(f).readlines(): | |
93 line_parts = l.strip().split(',') | |
94 if line_parts[0].startswith('#') or len(line_parts) != 5: | |
95 continue | |
96 dic[','.join(line_parts[:2])] = (float(line_parts[2]), float(line_parts[3]), | |
97 float(line_parts[4])) | |
98 | |
99 return dic | |
100 | |
101 def calc_expectations(p, h, gs_dir, exp_dir, repo_dir, extra_dir, extra_hash): | |
102 exp_filename = 'bench_expectations_%s.txt' % p | |
103 exp_fullname = os.path.join(exp_dir, exp_filename) | |
104 proc = subprocess.Popen(['python', 'skia/bench/gen_bench_expectations.py', | |
105 '-r', h, '-b', p, '-d', os.path.join(gs_dir, p), '-o', exp_fullname], | |
106 stdout=subprocess.PIPE) | |
107 out, err = proc.communicate() | |
108 if err: | |
109 print 'ERR_CALCULATING_EXPECTATIONS: ' + err | |
110 return False | |
111 print 'CALCULATED_EXPECTATIONS: ' + out | |
112 if extra_dir: # Adjust data with the ones in extra_dir | |
113 print 'USE_EXTRA_DATA_FOR_ADJUSTMENT.' | |
114 proc = subprocess.Popen(['python', 'skia/bench/gen_bench_expectations.py', | |
115 '-r', extra_hash, '-b', p, '-d', os.path.join(extra_dir, p), '-o', | |
116 os.path.join(extra_dir, exp_filename)], | |
117 stdout=subprocess.PIPE) | |
118 out, err = proc.communicate() | |
119 if err: | |
120 print 'ERR_CALCULATING_EXTRA_EXPECTATIONS: ' + err | |
121 return False | |
122 extra_dic = get_expectations_dict(os.path.join(extra_dir, exp_filename)) | |
123 output_lines = [] | |
124 for l in open(exp_fullname).readlines(): | |
125 parts = l.strip().split(',') | |
126 if parts[0].startswith('#') or len(parts) != 5: | |
127 output_lines.append(l.strip()) | |
128 continue | |
129 key = ','.join(parts[:2]) | |
130 if key in extra_dic: | |
131 exp, lb, ub = (float(parts[2]), float(parts[3]), float(parts[4])) | |
132 alt, _, _ = extra_dic[key] | |
133 avg = (exp + alt) / 2 | |
134 # Keeps the extra range in lower/upper bounds from two actual values. | |
135 new_lb = min(exp, alt) - (exp - lb) | |
136 new_ub = max(exp, alt) + (ub - exp) | |
137 output_lines.append('%s,%.2f,%.2f,%.2f' % (key, avg, new_lb, new_ub)) | |
138 else: | |
139 output_lines.append(l.strip()) | |
140 with open(exp_fullname, 'w') as f: | |
141 f.write('\n'.join(output_lines)) | |
142 | |
143 repo_file = os.path.join(repo_dir, 'expectations', 'bench', exp_filename) | |
144 if (os.path.isfile(repo_file) and | |
145 filecmp.cmp(repo_file, os.path.join(exp_dir, exp_filename))): | |
146 print 'NO CHANGE ON %s' % repo_file | |
147 return False | |
148 return True | |
149 | |
150 def checkout_or_update_skia(repo_dir): | |
151 status = True | |
152 old_cwd = os.getcwd() | |
153 os.chdir(repo_dir) | |
154 print 'CHECK SKIA REPO...' | |
155 if subprocess.call(['git', 'pull'], | |
156 stderr=subprocess.PIPE): | |
157 print 'Checking out Skia from git, please be patient...' | |
158 os.chdir(old_cwd) | |
159 clean_dir(repo_dir) | |
160 os.chdir(repo_dir) | |
161 if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch', | |
162 'https://skia.googlesource.com/skia.git', '.']): | |
163 status = False | |
164 subprocess.call(['git', 'checkout', 'master']) | |
165 subprocess.call(['git', 'pull']) | |
166 os.chdir(old_cwd) | |
167 return status | |
168 | |
169 def git_commit_expectations(repo_dir, exp_dir, update_li, h, commit, | |
170 extra_hash): | |
171 if extra_hash: | |
172 extra_hash = ', adjusted with ' + extra_hash | |
173 commit_msg = """manual bench rebase after %s%s | |
174 | |
175 TBR=robertphillips@google.com | |
176 | |
177 Bypassing trybots: | |
178 NOTRY=true""" % (h, extra_hash) | |
179 old_cwd = os.getcwd() | |
180 os.chdir(repo_dir) | |
181 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', | |
182 '--bypass-watchlists', '-m', commit_msg] | |
183 branch = exp_dir.split('/')[-1] | |
184 if commit: | |
185 upload.append('--use-commit-queue') | |
186 cmds = ([['git', 'checkout', 'master'], | |
187 ['git', 'pull'], | |
188 ['git', 'checkout', '-b', branch, '-t', 'origin/master']] + | |
189 [['cp', '%s/%s' % (exp_dir, f), 'expectations/bench'] for f in | |
190 update_li] + | |
191 [['git', 'add'] + ['expectations/bench/%s' % i for i in update_li], | |
192 ['git', 'commit', '-m', commit_msg], | |
193 upload, | |
194 ['git', 'checkout', 'master'], | |
195 ['git', 'branch', '-D', branch], | |
196 ]) | |
197 status = True | |
198 for cmd in cmds: | |
199 print 'Running ' + ' '.join(cmd) | |
200 if subprocess.call(cmd): | |
201 print 'FAILED. Please check if skia git repo is present.' | |
202 subprocess.call(['git', 'checkout', 'master']) | |
203 status = False | |
204 break | |
205 os.chdir(old_cwd) | |
206 return status | |
207 | |
208 def delete_dirs(li): | |
209 for d in li: | |
210 print 'Deleting directory %s' % d | |
211 shutil.rmtree(d) | |
212 | |
213 | |
214 def main(): | |
215 d = os.path.dirname(os.path.abspath(__file__)) | |
216 os.chdir(d) | |
217 if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE): | |
218 print 'Please copy script to a separate dir outside git repos to use.' | |
219 return | |
220 parser = argparse.ArgumentParser() | |
221 parser.add_argument('--githash', | |
222 help=('Githash prefix (7+ chars) to rebaseline to. If ' | |
223 'a second one is supplied after comma, and it has ' | |
224 'corresponding bench data, will shift the range ' | |
225 'center to the average of two expected values.')) | |
226 parser.add_argument('--bots', | |
227 help=('Comma-separated list of bots to work on. If no ' | |
228 'matching bots are found in the list, will default ' | |
229 'to processing all bots.')) | |
230 parser.add_argument('--commit', action='store_true', | |
231 help='Whether to commit changes automatically.') | |
232 args = parser.parse_args() | |
233 | |
234 repo_dir = os.path.join(d, 'skia') | |
235 if not os.path.exists(repo_dir): | |
236 os.makedirs(repo_dir) | |
237 if not checkout_or_update_skia(repo_dir): | |
238 print 'ERROR setting up Skia repo at %s' % repo_dir | |
239 return 1 | |
240 | |
241 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/rebase.py') | |
242 if not filecmp.cmp(__file__, file_in_repo): | |
243 shutil.copy(file_in_repo, __file__) | |
244 print 'Updated this script from repo; please run again.' | |
245 return | |
246 | |
247 all_platforms = [] # Find existing list of platforms with expectations. | |
248 for item in os.listdir(os.path.join(d, 'skia/expectations/bench')): | |
249 all_platforms.append( | |
250 item.replace('bench_expectations_', '').replace('.txt', '')) | |
251 | |
252 platforms = [] | |
253 # If at least one given bot is in all_platforms, use list of valid args.bots. | |
254 if args.bots: | |
255 bots = args.bots.strip().split(',') | |
256 for bot in bots: | |
257 if bot in all_platforms: # Filters platforms with given bot list. | |
258 platforms.append(bot) | |
259 if not platforms: # Include all existing platforms with expectations. | |
260 platforms = all_platforms | |
261 | |
262 if not args.githash or len(args.githash) < 7: | |
263 raise Exception('Please provide --githash with a longer prefix (7+).') | |
264 githashes = args.githash.strip().split(',') | |
265 if len(githashes[0]) < 7: | |
266 raise Exception('Please provide --githash with longer prefixes (7+).') | |
267 commit = False | |
268 if args.commit: | |
269 commit = True | |
270 rebase_hash = githashes[0][:7] | |
271 extra_hash = '' | |
272 if len(githashes) == 2: | |
273 extra_hash = githashes[1][:7] | |
274 hashes = get_git_hashes() | |
275 short_hashes = [h[:7] for h in hashes] | |
276 if (rebase_hash not in short_hashes or | |
277 (extra_hash and extra_hash not in short_hashes) or | |
278 rebase_hash == extra_hash): | |
279 raise Exception('Provided --githashes not found, or identical!') | |
280 if extra_hash: | |
281 extra_hash = hashes[short_hashes.index(extra_hash)] | |
282 hashes = hashes[:short_hashes.index(rebase_hash) + 1] | |
283 update_li = [] | |
284 | |
285 ts_str = '%s' % time.time() | |
286 gs_dir = os.path.join(d, 'gs' + ts_str) | |
287 exp_dir = os.path.join(d, 'exp' + ts_str) | |
288 extra_dir = os.path.join(d, 'extra' + ts_str) | |
289 clean_dir(gs_dir) | |
290 clean_dir(exp_dir) | |
291 clean_dir(extra_dir) | |
292 for p in platforms: | |
293 clean_dir(os.path.join(gs_dir, p)) | |
294 clean_dir(os.path.join(extra_dir, p)) | |
295 hash_to_use = '' | |
296 for h in reversed(hashes): | |
297 li = get_gs_filelist(p, h) | |
298 if not len(li): # no data | |
299 continue | |
300 if download_gs_files(p, h, gs_dir): | |
301 print 'Copied %s/%s' % (p, h) | |
302 hash_to_use = h | |
303 break | |
304 else: | |
305 print 'DOWNLOAD BENCH FAILED %s/%s' % (p, h) | |
306 break | |
307 if hash_to_use: | |
308 if extra_hash and download_gs_files(p, extra_hash, extra_dir): | |
309 print 'Copied extra data %s/%s' % (p, extra_hash) | |
310 if calc_expectations(p, h, gs_dir, exp_dir, repo_dir, extra_dir, | |
311 extra_hash): | |
312 update_li.append('bench_expectations_%s.txt' % p) | |
313 elif calc_expectations(p, h, gs_dir, exp_dir, repo_dir, '', ''): | |
314 update_li.append('bench_expectations_%s.txt' % p) | |
315 if not update_li: | |
316 print 'No bench data to update after %s!' % args.githash | |
317 elif not git_commit_expectations( | |
318 repo_dir, exp_dir, update_li, rebase_hash, commit, extra_hash): | |
319 print 'ERROR uploading expectations using git.' | |
320 elif not commit: | |
321 print 'CL created. Please take a look at the link above.' | |
322 else: | |
323 print 'New bench baselines should be in CQ now.' | |
324 delete_dirs([gs_dir, exp_dir, extra_dir]) | |
325 | |
326 | |
327 if __name__ == "__main__": | |
328 main() | |
OLD | NEW |