OLD | NEW |
| (Empty) |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 import hashlib | |
6 import json | |
7 import re | |
8 | |
9 from recipe_engine import recipe_api | |
10 | |
11 | |
12 def get_reviewers(commit_infos): | |
13 """Get a set of authors and reviewers from 'recipes.py autoroll' commit infos. | |
14 """ | |
15 reviewers = set() | |
16 for project, commits in commit_infos.iteritems(): | |
17 for commit in commits: | |
18 reviewers.add(commit['author']) | |
19 for field in ('R', 'TBR'): | |
20 for m in re.findall( | |
21 '^%s=(.*)' % field, commit['message'], re.MULTILINE): | |
22 for s in m.split(','): | |
23 # TODO(martiniss): infer domain for email address somehow? | |
24 parts = s.split('@') | |
25 if len(parts) != 2: | |
26 continue | |
27 # This mirrors a check in depot_tools/third_party/upload.py . | |
28 if '.' not in parts[1]: | |
29 continue | |
30 reviewers.add(s.strip()) | |
31 return reviewers | |
32 | |
33 | |
34 def get_bugs(commit_infos): | |
35 """Return a set of bug IDs from 'recipes.py autoroll' commit infos. | |
36 """ | |
37 bugs = set() | |
38 for project, commits in commit_infos.iteritems(): | |
39 for commit in commits: | |
40 for m in re.findall('^BUG=(.*)', commit['message'], re.MULTILINE): | |
41 for s in m.split(','): | |
42 if s: | |
43 bugs.add(s.strip()) | |
44 return bugs | |
45 | |
46 | |
47 def get_blame(commit_infos): | |
48 blame = [] | |
49 for project, commits in commit_infos.iteritems(): | |
50 blame.append('%s:' % project) | |
51 for commit in commits: | |
52 message = commit['message'].splitlines() | |
53 # TODO(phajdan.jr): truncate long messages. | |
54 message = message[0] if message else 'n/a' | |
55 blame.append(' https://crrev.com/%s %s (%s)' % ( | |
56 commit['revision'], message, commit['author'])) | |
57 return blame | |
58 | |
59 | |
60 COMMIT_MESSAGE_HEADER = ( | |
61 """ | |
62 This is an automated CL created by the recipe roller. This CL rolls recipe | |
63 changes from upstream projects (e.g. depot_tools) into downstream projects | |
64 (e.g. tools/build). | |
65 """) | |
66 | |
67 | |
68 NON_TRIVIAL_MESSAGE = ( | |
69 """ | |
70 | |
71 Please review the expectation changes, and LGTM as normal. The recipe roller | |
72 will *NOT* CQ the change itself, so you must commit the change manually. | |
73 """ | |
74 ) | |
75 | |
76 COMMIT_MESSAGE_FOOTER = ( | |
77 """ | |
78 | |
79 More info is at https://goo.gl/zkKdpD. Use https://goo.gl/noib3a to file a bug | |
80 (or complain) | |
81 | |
82 """) | |
83 | |
84 | |
85 TRIVIAL_ROLL_TBR_EMAILS = ( | |
86 'martiniss@chromium.org', | |
87 'phajdan.jr@chromium.org', | |
88 ) | |
89 | |
90 | |
91 # These are different results of a roll attempt: | |
92 # - success means we have a working non-empty roll | |
93 # - empty means the repo is using latest revision of its dependencies | |
94 # - failure means there are roll candidates but none of them are suitable | |
95 # for an automated roll | |
96 ROLL_SUCCESS, ROLL_EMPTY, ROLL_FAILURE = range(3) | |
97 | |
98 | |
99 def get_commit_message(roll_result): | |
100 """Construct a roll commit message from 'recipes.py autoroll' result. | |
101 """ | |
102 message = 'Roll recipe dependencies (%s).\n' % ( | |
103 'trivial' if roll_result['trivial'] else 'nontrivial') | |
104 message += COMMIT_MESSAGE_HEADER | |
105 if not roll_result['trivial']: | |
106 message += NON_TRIVIAL_MESSAGE | |
107 message += COMMIT_MESSAGE_FOOTER | |
108 | |
109 commit_infos = roll_result['picked_roll_details']['commit_infos'] | |
110 | |
111 message += '%s\n' % '\n'.join(get_blame(commit_infos)) | |
112 message += '\n' | |
113 message += 'R=%s\n' % ','.join(get_reviewers(commit_infos)) | |
114 message += 'BUG=%s\n' % ','.join(get_bugs(commit_infos)) | |
115 return message | |
116 | |
117 | |
118 class RecipeAutorollerApi(recipe_api.RecipeApi): | |
119 def prepare_checkout(self): #pragma: no cover | |
120 """Creates a default checkout for the recipe autoroller.""" | |
121 # Removed, but keep it here so roll succeeds | |
122 # TODO(martiniss): Delete once safe | |
123 pass | |
124 | |
125 | |
126 def roll_projects(self, projects): | |
127 """Attempts to roll each project from the provided list. | |
128 | |
129 If rolling any of the projects leads to failures, other | |
130 projects are not affected. | |
131 """ | |
132 project_data = self.m.luci_config.get_projects() | |
133 | |
134 self.m.cipd.install_client() | |
135 with self.m.tempfile.temp_dir('recipes') as recipes_dir: | |
136 self.m.cipd.ensure(recipes_dir, { | |
137 'infra/recipes-py': 'latest', | |
138 }) | |
139 | |
140 results = [] | |
141 with recipe_api.defer_results(): | |
142 for project in projects: | |
143 with self.m.step.nest(str(project)): | |
144 results.append(self._roll_project( | |
145 project_data[project], recipes_dir)) | |
146 | |
147 # We need to unwrap |DeferredResult|s. | |
148 results = [r.get_result() for r in results] | |
149 | |
150 # Failures to roll are OK as long as at least one of the repos is moving | |
151 # forward. For example, with repos with following dependencies: | |
152 # | |
153 # A <- B | |
154 # A, B <- C | |
155 # | |
156 # New commit in A repo will need to get rolled into B first. However, | |
157 # it'd also appear as a candidate for C roll, leading to a failure there. | |
158 if ROLL_FAILURE in results and ROLL_SUCCESS not in results: | |
159 self.m.python.failing_step( | |
160 'roll result', | |
161 'manual intervention needed: automated roll attempt failed') | |
162 | |
163 def _roll_project(self, project_data, recipes_dir): | |
164 with self.m.tempfile.temp_dir('roll_%s' % project_data['id']) as workdir: | |
165 self.m.git.checkout( | |
166 project_data['repo_url'], dir_path=workdir, submodules=False) | |
167 | |
168 # Introduce ourselves to git - also needed for git cl upload to work. | |
169 self.m.git( | |
170 'config', 'user.email', 'recipe-roller@chromium.org', cwd=workdir) | |
171 self.m.git('config', 'user.name', 'recipe-roller', cwd=workdir) | |
172 | |
173 # git cl upload cannot work with detached HEAD, it requires a branch. | |
174 self.m.git('checkout', '-t', '-b', 'roll', 'origin/master', cwd=workdir) | |
175 | |
176 recipes_cfg_path = workdir.join('infra', 'config', 'recipes.cfg') | |
177 | |
178 # Use the recipes bootstrap to checkout coverage. | |
179 roll_step = self.m.step( | |
180 'roll', | |
181 [recipes_dir.join('recipes.py'), '--use-bootstrap', '--package', | |
182 recipes_cfg_path, 'autoroll', '--output-json', self.m.json.output()]) | |
183 roll_result = roll_step.json.output | |
184 | |
185 if roll_result['success']: | |
186 self._process_successful_roll(roll_step, roll_result, workdir) | |
187 return ROLL_SUCCESS | |
188 else: | |
189 if (not roll_result['roll_details'] and | |
190 not roll_result['rejected_candidates_details']): | |
191 roll_step.presentation.step_text += ' (already at latest revisions)' | |
192 return ROLL_EMPTY | |
193 else: | |
194 return ROLL_FAILURE | |
195 | |
196 def _process_successful_roll(self, roll_step, roll_result, workdir): | |
197 roll_step.presentation.logs['blame'] = get_blame( | |
198 roll_result['picked_roll_details']['commit_infos']) | |
199 | |
200 if roll_result['trivial']: | |
201 roll_step.presentation.step_text += ' (trivial)' | |
202 else: | |
203 roll_step.presentation.status = self.m.step.FAILURE | |
204 | |
205 # We use recipes.cfg hashes to uniquely identify changes (which might be | |
206 # rebased). | |
207 cfg_contents = roll_result['picked_roll_details']['spec'] | |
208 cfg_digest = hashlib.md5(cfg_contents).hexdigest() | |
209 | |
210 # We use diff hashes to uniquely identify patchsets within a change. | |
211 self.m.git('commit', '-a', '-m', 'roll recipes.cfg', cwd=workdir) | |
212 diff_result = self.m.git( | |
213 'show', '--format=%b', | |
214 stdout=self.m.raw_io.output(), | |
215 cwd=workdir, | |
216 step_test_data=lambda: self.m.raw_io.test_api.stream_output( | |
217 '-some line\n+some other line\n')) | |
218 diff = diff_result.stdout | |
219 diff_result.presentation.logs['output'] = diff.splitlines() | |
220 diff_digest = hashlib.md5(diff).hexdigest() | |
221 | |
222 # Check if we have uploaded this before. | |
223 need_to_upload = False | |
224 rebase = False | |
225 cat_result = self.m.gsutil.cat( | |
226 'gs://recipe-roller-cl-uploads/%s' % cfg_digest, | |
227 stdout=self.m.raw_io.output(), | |
228 stderr=self.m.raw_io.output(), | |
229 ok_ret=(0,1)) | |
230 | |
231 if cat_result.retcode: | |
232 cat_result.presentation.logs['stderr'] = [ | |
233 self.m.step.active_result.stderr] | |
234 assert re.search('No URLs matched', cat_result.stderr), ( | |
235 'gsutil failed in an unexpected way; see stderr log') | |
236 # We have never uploaded this change before. | |
237 need_to_upload = True | |
238 | |
239 if not need_to_upload: | |
240 # We have uploaded before, now let's check the diff hash to see if we | |
241 # have uploaded this patchset before. | |
242 change_data = json.loads(cat_result.stdout) | |
243 if change_data['diff_digest'] != diff_digest: | |
244 self.m.git('cl', 'issue', change_data['issue'], cwd=workdir) | |
245 need_to_upload = True | |
246 rebase = True | |
247 cat_result.presentation.links['Issue %s' % change_data['issue']] = ( | |
248 change_data['issue_url']) | |
249 | |
250 if need_to_upload: | |
251 commit_message = ( | |
252 'Rebase' if rebase else get_commit_message(roll_result)) | |
253 if roll_result['trivial']: | |
254 # Land immediately. | |
255 upload_args = ['--use-commit-queue'] | |
256 if not rebase: | |
257 commit_message += '\nTBR=%s\n' % ','.join(TRIVIAL_ROLL_TBR_EMAILS) | |
258 else: | |
259 upload_args = ['--send-mail', '--cq-dry-run'] | |
260 upload_args.extend(['--bypass-hooks', '-f', '-m', commit_message]) | |
261 upload_args.extend([ | |
262 '--auth-refresh-token-json=/creds/refresh_tokens/recipe-roller']) | |
263 self.m.git('cl', 'upload', *upload_args, name='git cl upload', cwd=workdir
) | |
264 issue_result = self.m.git( | |
265 'cl', 'issue', | |
266 name='git cl issue', stdout=self.m.raw_io.output(), | |
267 cwd=workdir, | |
268 step_test_data=lambda: self.m.raw_io.test_api.stream_output( | |
269 'Issue number: ' | |
270 '123456789 (https://codereview.chromium.org/123456789)')) | |
271 | |
272 m = re.match('Issue number: (\d+) \((\S*)\)', issue_result.stdout.strip()) | |
273 if not m: | |
274 self.m.python.failing_step( | |
275 'git cl upload failed', 'git cl issue output "%s" is not valid' % | |
276 issue_result.stdout.strip()) | |
277 | |
278 change_data = { | |
279 'issue': m.group(1), | |
280 'issue_url': m.group(2), | |
281 'diff_digest': diff_digest, | |
282 } | |
283 issue_result.presentation.links['Issue %s' % change_data['issue']] = ( | |
284 change_data['issue_url']) | |
285 self.m.gsutil.upload( | |
286 self.m.json.input(change_data), | |
287 'recipe-roller-cl-uploads', | |
288 cfg_digest) | |
OLD | NEW |