OLD | NEW |
(Empty) | |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. |
| 4 |
| 5 import hashlib |
| 6 import json |
| 7 import re |
| 8 |
| 9 from recipe_engine import recipe_api |
| 10 |
| 11 |
| 12 def get_reviewers(commit_infos): |
| 13 """Get a set of authors and reviewers from 'recipes.py autoroll' commit infos. |
| 14 """ |
| 15 reviewers = set() |
| 16 for project, commits in commit_infos.iteritems(): |
| 17 for commit in commits: |
| 18 reviewers.add(commit['author']) |
| 19 for field in ('R', 'TBR'): |
| 20 for m in re.findall( |
| 21 '^%s=(.*)' % field, commit['message'], re.MULTILINE): |
| 22 for s in m.split(','): |
| 23 # TODO(martiniss): infer domain for email address somehow? |
| 24 parts = s.split('@') |
| 25 if len(parts) != 2: |
| 26 continue |
| 27 # This mirrors a check in depot_tools/third_party/upload.py . |
| 28 if '.' not in parts[1]: |
| 29 continue |
| 30 reviewers.add(s.strip()) |
| 31 return reviewers |
| 32 |
| 33 |
| 34 def get_bugs(commit_infos): |
| 35 """Return a set of bug IDs from 'recipes.py autoroll' commit infos. |
| 36 """ |
| 37 bugs = set() |
| 38 for project, commits in commit_infos.iteritems(): |
| 39 for commit in commits: |
| 40 for m in re.findall('^BUG=(.*)', commit['message'], re.MULTILINE): |
| 41 for s in m.split(','): |
| 42 if s: |
| 43 bugs.add(s.strip()) |
| 44 return bugs |
| 45 |
| 46 |
| 47 def get_blame(commit_infos): |
| 48 blame = [] |
| 49 for project, commits in commit_infos.iteritems(): |
| 50 blame.append('%s:' % project) |
| 51 for commit in commits: |
| 52 message = commit['message'].splitlines() |
| 53 # TODO(phajdan.jr): truncate long messages. |
| 54 message = message[0] if message else 'n/a' |
| 55 blame.append(' https://crrev.com/%s %s (%s)' % ( |
| 56 commit['revision'], message, commit['author'])) |
| 57 return blame |
| 58 |
| 59 |
| 60 COMMIT_MESSAGE_HEADER = ( |
| 61 """ |
| 62 This is an automated CL created by the recipe roller. This CL rolls recipe |
| 63 changes from upstream projects (e.g. depot_tools) into downstream projects |
| 64 (e.g. tools/build). |
| 65 """) |
| 66 |
| 67 |
| 68 NON_TRIVIAL_MESSAGE = ( |
| 69 """ |
| 70 |
| 71 Please review the expectation changes, and LGTM as normal. The recipe roller |
| 72 will *NOT* CQ the change itself, so you must commit the change manually. |
| 73 """ |
| 74 ) |
| 75 |
| 76 COMMIT_MESSAGE_FOOTER = ( |
| 77 """ |
| 78 |
| 79 More info is at https://goo.gl/zkKdpD. Use https://goo.gl/noib3a to file a bug |
| 80 (or complain) |
| 81 |
| 82 """) |
| 83 |
| 84 |
| 85 TRIVIAL_ROLL_TBR_EMAILS = ( |
| 86 'martiniss@chromium.org', |
| 87 'phajdan.jr@chromium.org', |
| 88 ) |
| 89 |
| 90 |
| 91 # These are different results of a roll attempt: |
| 92 # - success means we have a working non-empty roll |
| 93 # - empty means the repo is using latest revision of its dependencies |
| 94 # - failure means there are roll candidates but none of them are suitable |
| 95 # for an automated roll |
| 96 ROLL_SUCCESS, ROLL_EMPTY, ROLL_FAILURE = range(3) |
| 97 |
| 98 |
| 99 def get_commit_message(roll_result): |
| 100 """Construct a roll commit message from 'recipes.py autoroll' result. |
| 101 """ |
| 102 message = 'Roll recipe dependencies (%s).\n' % ( |
| 103 'trivial' if roll_result['trivial'] else 'nontrivial') |
| 104 message += COMMIT_MESSAGE_HEADER |
| 105 if not roll_result['trivial']: |
| 106 message += NON_TRIVIAL_MESSAGE |
| 107 message += COMMIT_MESSAGE_FOOTER |
| 108 |
| 109 commit_infos = roll_result['picked_roll_details']['commit_infos'] |
| 110 |
| 111 message += '%s\n' % '\n'.join(get_blame(commit_infos)) |
| 112 message += '\n' |
| 113 message += 'R=%s\n' % ','.join(get_reviewers(commit_infos)) |
| 114 message += 'BUG=%s\n' % ','.join(get_bugs(commit_infos)) |
| 115 return message |
| 116 |
| 117 |
| 118 class RecipeAutorollerApi(recipe_api.RecipeApi): |
| 119 def prepare_checkout(self): #pragma: no cover |
| 120 """Creates a default checkout for the recipe autoroller.""" |
| 121 # Removed, but keep it here so roll succeeds |
| 122 # TODO(martiniss): Delete once safe |
| 123 pass |
| 124 |
| 125 |
| 126 def roll_projects(self, projects): |
| 127 """Attempts to roll each project from the provided list. |
| 128 |
| 129 If rolling any of the projects leads to failures, other |
| 130 projects are not affected. |
| 131 """ |
| 132 project_data = self.m.luci_config.get_projects() |
| 133 |
| 134 self.m.cipd.install_client() |
| 135 with self.m.tempfile.temp_dir('recipes') as recipes_dir: |
| 136 self.m.cipd.ensure(recipes_dir, { |
| 137 'infra/recipes-py': 'latest', |
| 138 }) |
| 139 |
| 140 results = [] |
| 141 with recipe_api.defer_results(): |
| 142 for project in projects: |
| 143 with self.m.step.nest(str(project)): |
| 144 results.append(self._roll_project( |
| 145 project_data[project], recipes_dir)) |
| 146 |
| 147 # We need to unwrap |DeferredResult|s. |
| 148 results = [r.get_result() for r in results] |
| 149 |
| 150 # Failures to roll are OK as long as at least one of the repos is moving |
| 151 # forward. For example, with repos with following dependencies: |
| 152 # |
| 153 # A <- B |
| 154 # A, B <- C |
| 155 # |
| 156 # New commit in A repo will need to get rolled into B first. However, |
| 157 # it'd also appear as a candidate for C roll, leading to a failure there. |
| 158 if ROLL_FAILURE in results and ROLL_SUCCESS not in results: |
| 159 self.m.python.failing_step( |
| 160 'roll result', |
| 161 'manual intervention needed: automated roll attempt failed') |
| 162 |
| 163 def _roll_project(self, project_data, recipes_dir): |
| 164 with self.m.tempfile.temp_dir('roll_%s' % project_data['id']) as workdir: |
| 165 self.m.git.checkout( |
| 166 project_data['repo_url'], dir_path=workdir, submodules=False) |
| 167 |
| 168 # Introduce ourselves to git - also needed for git cl upload to work. |
| 169 self.m.git( |
| 170 'config', 'user.email', 'recipe-roller@chromium.org', cwd=workdir) |
| 171 self.m.git('config', 'user.name', 'recipe-roller', cwd=workdir) |
| 172 |
| 173 # git cl upload cannot work with detached HEAD, it requires a branch. |
| 174 self.m.git('checkout', '-t', '-b', 'roll', 'origin/master', cwd=workdir) |
| 175 |
| 176 recipes_cfg_path = workdir.join('infra', 'config', 'recipes.cfg') |
| 177 |
| 178 # Use the recipes bootstrap to checkout coverage. |
| 179 roll_step = self.m.step( |
| 180 'roll', |
| 181 [recipes_dir.join('recipes.py'), '--use-bootstrap', '--package', |
| 182 recipes_cfg_path, 'autoroll', '--output-json', self.m.json.output()]) |
| 183 roll_result = roll_step.json.output |
| 184 |
| 185 if roll_result['success']: |
| 186 self._process_successful_roll(roll_step, roll_result, workdir) |
| 187 return ROLL_SUCCESS |
| 188 else: |
| 189 if (not roll_result['roll_details'] and |
| 190 not roll_result['rejected_candidates_details']): |
| 191 roll_step.presentation.step_text += ' (already at latest revisions)' |
| 192 return ROLL_EMPTY |
| 193 else: |
| 194 return ROLL_FAILURE |
| 195 |
| 196 def _process_successful_roll(self, roll_step, roll_result, workdir): |
| 197 roll_step.presentation.logs['blame'] = get_blame( |
| 198 roll_result['picked_roll_details']['commit_infos']) |
| 199 |
| 200 if roll_result['trivial']: |
| 201 roll_step.presentation.step_text += ' (trivial)' |
| 202 else: |
| 203 roll_step.presentation.status = self.m.step.FAILURE |
| 204 |
| 205 # We use recipes.cfg hashes to uniquely identify changes (which might be |
| 206 # rebased). |
| 207 cfg_contents = roll_result['picked_roll_details']['spec'] |
| 208 cfg_digest = hashlib.md5(cfg_contents).hexdigest() |
| 209 |
| 210 # We use diff hashes to uniquely identify patchsets within a change. |
| 211 self.m.git('commit', '-a', '-m', 'roll recipes.cfg', cwd=workdir) |
| 212 diff_result = self.m.git( |
| 213 'show', '--format=%b', |
| 214 stdout=self.m.raw_io.output(), |
| 215 cwd=workdir, |
| 216 step_test_data=lambda: self.m.raw_io.test_api.stream_output( |
| 217 '-some line\n+some other line\n')) |
| 218 diff = diff_result.stdout |
| 219 diff_result.presentation.logs['output'] = diff.splitlines() |
| 220 diff_digest = hashlib.md5(diff).hexdigest() |
| 221 |
| 222 # Check if we have uploaded this before. |
| 223 need_to_upload = False |
| 224 rebase = False |
| 225 cat_result = self.m.gsutil.cat( |
| 226 'gs://recipe-roller-cl-uploads/%s' % cfg_digest, |
| 227 stdout=self.m.raw_io.output(), |
| 228 stderr=self.m.raw_io.output(), |
| 229 ok_ret=(0,1)) |
| 230 |
| 231 if cat_result.retcode: |
| 232 cat_result.presentation.logs['stderr'] = [ |
| 233 self.m.step.active_result.stderr] |
| 234 assert re.search('No URLs matched', cat_result.stderr), ( |
| 235 'gsutil failed in an unexpected way; see stderr log') |
| 236 # We have never uploaded this change before. |
| 237 need_to_upload = True |
| 238 |
| 239 if not need_to_upload: |
| 240 # We have uploaded before, now let's check the diff hash to see if we |
| 241 # have uploaded this patchset before. |
| 242 change_data = json.loads(cat_result.stdout) |
| 243 if change_data['diff_digest'] != diff_digest: |
| 244 self.m.git('cl', 'issue', change_data['issue'], cwd=workdir) |
| 245 need_to_upload = True |
| 246 rebase = True |
| 247 cat_result.presentation.links['Issue %s' % change_data['issue']] = ( |
| 248 change_data['issue_url']) |
| 249 |
| 250 if need_to_upload: |
| 251 commit_message = ( |
| 252 'Rebase' if rebase else get_commit_message(roll_result)) |
| 253 if roll_result['trivial']: |
| 254 # Land immediately. |
| 255 upload_args = ['--use-commit-queue'] |
| 256 if not rebase: |
| 257 commit_message += '\nTBR=%s\n' % ','.join(TRIVIAL_ROLL_TBR_EMAILS) |
| 258 else: |
| 259 upload_args = ['--send-mail', '--cq-dry-run'] |
| 260 upload_args.extend(['--bypass-hooks', '-f', '-m', commit_message]) |
| 261 upload_args.extend([ |
| 262 '--auth-refresh-token-json=/creds/refresh_tokens/recipe-roller']) |
| 263 self.m.git('cl', 'upload', *upload_args, name='git cl upload', cwd=workdir
) |
| 264 issue_result = self.m.git( |
| 265 'cl', 'issue', |
| 266 name='git cl issue', stdout=self.m.raw_io.output(), |
| 267 cwd=workdir, |
| 268 step_test_data=lambda: self.m.raw_io.test_api.stream_output( |
| 269 'Issue number: ' |
| 270 '123456789 (https://codereview.chromium.org/123456789)')) |
| 271 |
| 272 m = re.match('Issue number: (\d+) \((\S*)\)', issue_result.stdout.strip()) |
| 273 if not m: |
| 274 self.m.python.failing_step( |
| 275 'git cl upload failed', 'git cl issue output "%s" is not valid' % |
| 276 issue_result.stdout.strip()) |
| 277 |
| 278 change_data = { |
| 279 'issue': m.group(1), |
| 280 'issue_url': m.group(2), |
| 281 'diff_digest': diff_digest, |
| 282 } |
| 283 issue_result.presentation.links['Issue %s' % change_data['issue']] = ( |
| 284 change_data['issue_url']) |
| 285 self.m.gsutil.upload( |
| 286 self.m.json.input(change_data), |
| 287 'recipe-roller-cl-uploads', |
| 288 cfg_digest) |
OLD | NEW |