OLD | NEW |
1 #!/usr/bin/python2.7 | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
2 | 6 |
3 """rebase.py: standalone script to batch update bench expectations. | 7 """rebase.py: standalone script to batch update bench expectations. |
4 Usage: | 8 |
5 Copy script to a separate dir outside Skia repo. The script will create a | 9 Requires gsutil to access gs://chromium-skia-gm and Rietveld credentials. |
6 skia dir on the first run to host the repo, and will create/delete temp | 10 |
7 dirs as needed. | 11 Usage: |
8 ./rebase.py --githash <githash prefix to use for getting bench data> | 12 Copy script to a separate dir outside Skia repo. The script will create a |
| 13 skia dir on the first run to host the repo, and will create/delete |
| 14 temp dirs as needed. |
| 15 ./rebase.py --githash <githash prefix to use for getting bench data> |
9 """ | 16 """ |
10 | 17 |
| 18 |
11 import argparse | 19 import argparse |
12 import filecmp | 20 import filecmp |
13 import os | 21 import os |
14 import re | 22 import re |
15 import shutil | 23 import shutil |
16 import subprocess | 24 import subprocess |
17 import time | 25 import time |
18 import urllib2 | 26 import urllib2 |
19 | 27 |
20 | 28 |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
55 proc = subprocess.Popen(['gsutil', 'ls', | 63 proc = subprocess.Popen(['gsutil', 'ls', |
56 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*'])], | 64 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*'])], |
57 stdout=subprocess.PIPE) | 65 stdout=subprocess.PIPE) |
58 out, err = proc.communicate() | 66 out, err = proc.communicate() |
59 if err or not out: | 67 if err or not out: |
60 return [] | 68 return [] |
61 return [i for i in out.strip().split('\n') if not filter_file(i)] | 69 return [i for i in out.strip().split('\n') if not filter_file(i)] |
62 | 70 |
63 def download_gs_files(p, h, gs_dir): | 71 def download_gs_files(p, h, gs_dir): |
64 print 'Downloading raw bench files from Google Storage...' | 72 print 'Downloading raw bench files from Google Storage...' |
65 proc = subprocess.Popen(['gsutil', '-q', 'cp', | 73 proc = subprocess.Popen(['gsutil', 'cp', '-q', |
66 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*']), | 74 '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*']), |
67 '%s/%s' % (gs_dir, p)], | 75 '%s/%s' % (gs_dir, p)], |
68 stdout=subprocess.PIPE) | 76 stdout=subprocess.PIPE) |
69 out, err = proc.communicate() | 77 out, err = proc.communicate() |
70 if err: | 78 if err: |
71 clean_dir(gs_dir) | 79 clean_dir(gs_dir) |
72 return False | 80 return False |
73 files = 0 | 81 files = 0 |
74 for f in os.listdir(os.path.join(gs_dir, p)): | 82 for f in os.listdir(os.path.join(gs_dir, p)): |
75 if filter_file(f): | 83 if filter_file(f): |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
121 commit_msg = """bench rebase after %s | 129 commit_msg = """bench rebase after %s |
122 | 130 |
123 TBR=robertphillips@google.com | 131 TBR=robertphillips@google.com |
124 | 132 |
125 Bypassing trybots: | 133 Bypassing trybots: |
126 NOTRY=true""" % h | 134 NOTRY=true""" % h |
127 old_cwd = os.getcwd() | 135 old_cwd = os.getcwd() |
128 os.chdir(repo_dir) | 136 os.chdir(repo_dir) |
129 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', | 137 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', |
130 '--bypass-watchlists', '-m', commit_msg] | 138 '--bypass-watchlists', '-m', commit_msg] |
| 139 branch = exp_dir.split('/')[-1] |
131 if commit: | 140 if commit: |
132 upload.append('--use-commit-queue') | 141 upload.append('--use-commit-queue') |
133 cmds = ([['git', 'checkout', 'master'], | 142 cmds = ([['git', 'checkout', 'master'], |
134 ['git', 'pull'], | 143 ['git', 'pull'], |
135 ['git', 'checkout', '-b', exp_dir, '-t', 'origin/master']] + | 144 ['git', 'checkout', '-b', branch, '-t', 'origin/master']] + |
136 [['cp', '../%s/%s' % (exp_dir, f), 'expectations/bench'] for f in | 145 [['cp', '%s/%s' % (exp_dir, f), 'expectations/bench'] for f in |
137 update_li] + | 146 update_li] + |
138 [['git', 'add'] + ['expectations/bench/%s' % i for i in update_li], | 147 [['git', 'add'] + ['expectations/bench/%s' % i for i in update_li], |
139 ['git', 'commit', '-m', commit_msg], | 148 ['git', 'commit', '-m', commit_msg], |
140 upload, | 149 upload, |
141 ['git', 'checkout', 'master'], | 150 ['git', 'checkout', 'master'], |
142 ['git', 'branch', '-D', exp_dir], | 151 ['git', 'branch', '-D', branch], |
143 ]) | 152 ]) |
144 status = True | 153 status = True |
145 for cmd in cmds: | 154 for cmd in cmds: |
146 print 'Running ' + ' '.join(cmd) | 155 print 'Running ' + ' '.join(cmd) |
147 if subprocess.call(cmd): | 156 if subprocess.call(cmd): |
148 print 'FAILED. Please check if skia git repo is present.' | 157 print 'FAILED. Please check if skia git repo is present.' |
149 subprocess.call(['git', 'checkout', 'master']) | 158 subprocess.call(['git', 'checkout', 'master']) |
150 status = False | 159 status = False |
151 break | 160 break |
152 os.chdir(old_cwd) | 161 os.chdir(old_cwd) |
(...skipping 18 matching lines...) Expand all Loading... |
171 help='Whether to commit changes automatically.') | 180 help='Whether to commit changes automatically.') |
172 args = parser.parse_args() | 181 args = parser.parse_args() |
173 | 182 |
174 repo_dir = os.path.join(d, 'skia') | 183 repo_dir = os.path.join(d, 'skia') |
175 if not os.path.exists(repo_dir): | 184 if not os.path.exists(repo_dir): |
176 os.makedirs(repo_dir) | 185 os.makedirs(repo_dir) |
177 if not checkout_or_update_skia(repo_dir): | 186 if not checkout_or_update_skia(repo_dir): |
178 print 'ERROR setting up Skia repo at %s' % repo_dir | 187 print 'ERROR setting up Skia repo at %s' % repo_dir |
179 return 1 | 188 return 1 |
180 | 189 |
181 for item in os.listdir(os.path.join(d, 'skia/expectations/bench')): | |
182 PLATFORMS.append( | |
183 item.replace('bench_expectations_', '').replace('.txt', '')) | |
184 | |
185 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/rebase.py') | 190 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/rebase.py') |
186 if not filecmp.cmp(__file__, file_in_repo): | 191 if not filecmp.cmp(__file__, file_in_repo): |
187 shutil.copy(file_in_repo, __file__) | 192 shutil.copy(file_in_repo, __file__) |
188 print 'Updated this script from repo; please run again.' | 193 print 'Updated this script from repo; please run again.' |
189 return | 194 return |
190 | 195 |
| 196 for item in os.listdir(os.path.join(d, 'skia/expectations/bench')): |
| 197 PLATFORMS.append( |
| 198 item.replace('bench_expectations_', '').replace('.txt', '')) |
| 199 |
191 if not args.githash or len(args.githash) < 7: | 200 if not args.githash or len(args.githash) < 7: |
192 raise Exception('Please provide --githash with a longer prefix (7+).') | 201 raise Exception('Please provide --githash with a longer prefix (7+).') |
193 commit = False | 202 commit = False |
194 if args.commit: | 203 if args.commit: |
195 commit = True | 204 commit = True |
196 rebase_hash = args.githash[:7] | 205 rebase_hash = args.githash[:7] |
197 hashes = get_git_hashes() | 206 hashes = get_git_hashes() |
198 short_hashes = [h[:7] for h in hashes] | 207 short_hashes = [h[:7] for h in hashes] |
199 if rebase_hash not in short_hashes: | 208 if rebase_hash not in short_hashes: |
200 raise Exception('Provided --githash not found in recent history!') | 209 raise Exception('Provided --githash not found in recent history!') |
(...skipping 29 matching lines...) Expand all Loading... |
230 print 'ERROR uploading expectations using git.' | 239 print 'ERROR uploading expectations using git.' |
231 elif not commit: | 240 elif not commit: |
232 print 'CL created. Please take a look at the link above.' | 241 print 'CL created. Please take a look at the link above.' |
233 else: | 242 else: |
234 print 'New bench baselines should be in CQ now.' | 243 print 'New bench baselines should be in CQ now.' |
235 delete_dirs([gs_dir, exp_dir]) | 244 delete_dirs([gs_dir, exp_dir]) |
236 | 245 |
237 | 246 |
238 if __name__ == "__main__": | 247 if __name__ == "__main__": |
239 main() | 248 main() |
OLD | NEW |