OLD | NEW |
1 # coding=utf8 | 1 # coding=utf8 |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 """Commit queue manager class. | 5 """Commit queue manager class. |
6 | 6 |
7 Security implications: | 7 Security implications: |
8 | 8 |
9 The following hypothesis are made: | 9 The following hypothesis are made: |
10 - Commit queue: | 10 - Commit queue: |
(...skipping 179 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
190 | 190 |
191 raise | 191 raise |
192 | 192 |
193 # If there is an issue in processed_issues that is not in new_issues, | 193 # If there is an issue in processed_issues that is not in new_issues, |
194 # discard it. | 194 # discard it. |
195 for pending in self.queue.iterate(): | 195 for pending in self.queue.iterate(): |
196 # Note that pending.issue is a int but self.queue.pending_commits keys | 196 # Note that pending.issue is a int but self.queue.pending_commits keys |
197 # are str due to json support. | 197 # are str due to json support. |
198 if pending.issue not in new_issues: | 198 if pending.issue not in new_issues: |
199 logging.info('Flushing issue %d' % pending.issue) | 199 logging.info('Flushing issue %d' % pending.issue) |
200 self.context.status.send( | |
201 pending, | |
202 { 'verification': 'abort', | |
203 'payload': { | |
204 'output': 'CQ bit was unchecked on CL. Ignoring.' }}) | |
205 pending.get_state = lambda: base.IGNORED | 200 pending.get_state = lambda: base.IGNORED |
206 self._discard_pending(pending, None) | 201 self._discard_pending(pending, 'CQ bit was unchecked on CL. Ignoring.') |
207 | 202 |
208 # Find new issues. | 203 # Find new issues. |
209 for issue_id in new_issues: | 204 for issue_id in new_issues: |
210 if str(issue_id) not in self.queue.pending_commits: | 205 if str(issue_id) not in self.queue.pending_commits: |
211 try: | 206 try: |
212 issue_data = self.context.rietveld.get_issue_properties( | 207 issue_data = self.context.rietveld.get_issue_properties( |
213 issue_id, True) | 208 issue_id, True) |
214 except urllib2.HTTPError as e: | 209 except urllib2.HTTPError as e: |
215 if e.code in (500, 502, 503): | 210 if e.code in (500, 502, 503): |
216 # Temporary AppEngine hiccup. Just log it and continue. | 211 # Temporary AppEngine hiccup. Just log it and continue. |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
264 # Take in account the case where a verifier was removed. | 259 # Take in account the case where a verifier was removed. |
265 done = set(pending.verifications.keys()) | 260 done = set(pending.verifications.keys()) |
266 missing = expected - done | 261 missing = expected - done |
267 if (not missing or pending.get_state() != base.PROCESSING): | 262 if (not missing or pending.get_state() != base.PROCESSING): |
268 continue | 263 continue |
269 logging.info( | 264 logging.info( |
270 'Processing issue %s (%s, %d)' % ( | 265 'Processing issue %s (%s, %d)' % ( |
271 pending.issue, missing, pending.get_state())) | 266 pending.issue, missing, pending.get_state())) |
272 self._verify_pending(pending) | 267 self._verify_pending(pending) |
273 except base.DiscardPending as e: | 268 except base.DiscardPending as e: |
274 self._discard_pending(e.pending, e.status) | 269 message = e.status |
| 270 if not message: |
| 271 message = 'process_new_pending_commit: ' + self.FAILED_NO_MESSAGE |
| 272 self._discard_pending(e.pending, message) |
275 | 273 |
276 def update_status(self): | 274 def update_status(self): |
277 """Updates the status for each pending commit verifier.""" | 275 """Updates the status for each pending commit verifier.""" |
278 why_nots = dict((p.issue, p.why_not()) for p in self.queue.iterate()) | 276 why_nots = dict((p.issue, p.why_not()) for p in self.queue.iterate()) |
279 | 277 |
280 for verifier in self.all_verifiers: | 278 for verifier in self.all_verifiers: |
281 try: | 279 try: |
282 verifier.update_status(self.queue.iterate()) | 280 verifier.update_status(self.queue.iterate()) |
283 except base.DiscardPending as e: | 281 except base.DiscardPending as e: |
284 # It's not efficient since it takes a full loop for each pending | 282 # It's not efficient since it takes a full loop for each pending |
285 # commit to discard. | 283 # commit to discard. |
286 self._discard_pending(e.pending, e.status) | 284 message = e.status |
| 285 if not message: |
| 286 message = 'update_status: ' + self.FAILED_NO_MESSAGE |
| 287 self._discard_pending(e.pending, message) |
287 | 288 |
288 for pending in self.queue.iterate(): | 289 for pending in self.queue.iterate(): |
289 why_not = pending.why_not() | 290 why_not = pending.why_not() |
290 if why_nots[pending.issue] != why_not: | 291 if why_nots[pending.issue] != why_not: |
291 self.context.status.send( | 292 self.context.status.send( |
292 pending, | 293 pending, |
293 {'verification': 'why not', | 294 {'verification': 'why not', |
294 'payload': {'message': why_not}}) | 295 'payload': {'message': why_not}}) |
295 | 296 |
296 | 297 |
297 def scan_results(self): | 298 def scan_results(self): |
298 """Scans pending commits that can be committed or discarded.""" | 299 """Scans pending commits that can be committed or discarded.""" |
299 for pending in self.queue.iterate(): | 300 for pending in self.queue.iterate(): |
300 state = pending.get_state() | 301 state = pending.get_state() |
301 if state == base.FAILED: | 302 if state == base.FAILED: |
302 self._discard_pending( | 303 message = pending.error_message() |
303 pending, pending.error_message() or self.FAILED_NO_MESSAGE) | 304 if not message: |
| 305 message = 'scan_results(FAILED): ' + self.FAILED_NO_MESSAGE |
| 306 self._discard_pending(pending, message) |
304 elif state == base.SUCCEEDED: | 307 elif state == base.SUCCEEDED: |
305 if self._throttle(pending): | 308 if self._throttle(pending): |
306 continue | 309 continue |
307 try: | 310 try: |
308 # Runs checks. It's be nice to run the test before the postpone, | 311 # Runs checks. It's be nice to run the test before the postpone, |
309 # especially if the tree is closed for a long moment but at the same | 312 # especially if the tree is closed for a long moment but at the same |
310 # time it would keep fetching the rietveld status constantly. | 313 # time it would keep fetching the rietveld status constantly. |
311 self._last_minute_checks(pending) | 314 self._last_minute_checks(pending) |
312 self.context.status.send( | 315 self.context.status.send( |
313 pending, | 316 pending, |
314 {'verification': 'why not', | 317 {'verification': 'why not', |
315 'payload': {'message': ''}}) | 318 'payload': {'message': ''}}) |
316 | 319 |
317 self._commit_patch(pending) | 320 self._commit_patch(pending) |
318 except base.DiscardPending as e: | 321 except base.DiscardPending as e: |
319 self._discard_pending(e.pending, e.status) | 322 message = e.status |
| 323 if not message: |
| 324 message = 'scan_results(discard): ' + self.FAILED_NO_MESSAGE |
| 325 self._discard_pending(e.pending, message) |
320 except Exception as e: | 326 except Exception as e: |
321 self._discard_pending(pending, self.INTERNAL_EXCEPTION) | 327 message = 'scan_result(Exception): ' + self.INTERNAL_EXCEPTION |
| 328 self._discard_pending(pending, message) |
322 raise | 329 raise |
323 else: | 330 else: |
324 # When state is IGNORED, we need to keep this issue so it's not fetched | 331 # When state is IGNORED, we need to keep this issue so it's not fetched |
325 # another time but we can't discard it since we don't want to remove the | 332 # another time but we can't discard it since we don't want to remove the |
326 # commit bit for another project hosted on the same code review | 333 # commit bit for another project hosted on the same code review |
327 # instance. | 334 # instance. |
328 assert state in (base.PROCESSING, base.IGNORED) | 335 assert state in (base.PROCESSING, base.IGNORED) |
329 | 336 |
330 def _verify_pending(self, pending): | 337 def _verify_pending(self, pending): |
331 """Initiates all the verifiers on a pending change.""" | 338 """Initiates all the verifiers on a pending change.""" |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
416 pending, | 423 pending, |
417 'List of reviewers changed. %s did a drive-by without LGTM\'ing!' % | 424 'List of reviewers changed. %s did a drive-by without LGTM\'ing!' % |
418 ','.join(drivers_by)) | 425 ','.join(drivers_by)) |
419 if pending.patchset != pending_data['patchsets'][-1]: | 426 if pending.patchset != pending_data['patchsets'][-1]: |
420 raise base.DiscardPending(pending, | 427 raise base.DiscardPending(pending, |
421 'Commit queue failed due to new patchset.') | 428 'Commit queue failed due to new patchset.') |
422 | 429 |
423 def _discard_pending(self, pending, message): | 430 def _discard_pending(self, pending, message): |
424 """Discards a pending commit. Attach an optional message to the review.""" | 431 """Discards a pending commit. Attach an optional message to the review.""" |
425 logging.debug('_discard_pending(%s, %s)', pending.issue, message) | 432 logging.debug('_discard_pending(%s, %s)', pending.issue, message) |
| 433 msg = message or self.FAILED_NO_MESSAGE |
426 try: | 434 try: |
427 try: | 435 try: |
428 if pending.get_state() != base.IGNORED: | 436 if pending.get_state() != base.IGNORED: |
429 self.context.rietveld.set_flag( | 437 self.context.rietveld.set_flag( |
430 pending.issue, pending.patchset, 'commit', False) | 438 pending.issue, pending.patchset, 'commit', False) |
431 except urllib2.HTTPError as e: | 439 except urllib2.HTTPError as e: |
432 logging.error( | 440 logging.error( |
433 'Failed to set the flag to False for %s with message %s' % ( | 441 'Failed to set the flag to False for %s with message %s' % ( |
434 pending.pending_name(), message)) | 442 pending.pending_name(), msg)) |
435 traceback.print_stack() | 443 traceback.print_stack() |
436 logging.error(str(e)) | 444 logging.error(str(e)) |
437 errors.send_stack(e) | 445 errors.send_stack(e) |
438 if message: | 446 try: |
439 try: | 447 self.context.rietveld.add_comment(pending.issue, msg) |
440 self.context.rietveld.add_comment(pending.issue, message) | 448 except urllib2.HTTPError as e: |
441 except urllib2.HTTPError as e: | 449 logging.error( |
442 logging.error( | 450 'Failed to add comment for %s with message %s' % ( |
443 'Failed to add comment for %s with message %s' % ( | 451 pending.pending_name(), msg)) |
444 pending.pending_name(), message)) | 452 traceback.print_stack() |
445 traceback.print_stack() | 453 errors.send_stack(e) |
446 errors.send_stack(e) | 454 self.context.status.send( |
447 self.context.status.send( | 455 pending, |
448 pending, | 456 { 'verification': 'abort', |
449 { 'verification': 'abort', | 457 'payload': { |
450 'payload': { | 458 'output': msg }}) |
451 'output': message }}) | |
452 finally: | 459 finally: |
453 # Most importantly, remove the PendingCommit from the queue. | 460 # Most importantly, remove the PendingCommit from the queue. |
454 self.queue.remove(pending.issue) | 461 self.queue.remove(pending.issue) |
455 | 462 |
456 def _commit_patch(self, pending): | 463 def _commit_patch(self, pending): |
457 """Commits the pending patch to the repository. | 464 """Commits the pending patch to the repository. |
458 | 465 |
459 Do the checkout and applies the patch. | 466 Do the checkout and applies the patch. |
460 """ | 467 """ |
461 try: | 468 try: |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
520 except ( | 527 except ( |
521 checkout.PatchApplicationFailed, patch.UnsupportedPatchFormat) as e: | 528 checkout.PatchApplicationFailed, patch.UnsupportedPatchFormat) as e: |
522 raise base.DiscardPending(pending, str(e)) | 529 raise base.DiscardPending(pending, str(e)) |
523 except subprocess2.CalledProcessError as e: | 530 except subprocess2.CalledProcessError as e: |
524 stdout = getattr(e, 'stdout', None) | 531 stdout = getattr(e, 'stdout', None) |
525 out = 'Failed to apply the patch.' | 532 out = 'Failed to apply the patch.' |
526 if stdout: | 533 if stdout: |
527 out += '\n%s' % stdout | 534 out += '\n%s' % stdout |
528 raise base.DiscardPending(pending, out) | 535 raise base.DiscardPending(pending, out) |
529 except base.DiscardPending as e: | 536 except base.DiscardPending as e: |
530 self._discard_pending(e.pending, e.status) | 537 message = e.status |
| 538 if not message: |
| 539 message = '_commit_patch: ' + self.FAILED_NO_MESSAGE |
| 540 self._discard_pending(e.pending, message) |
531 | 541 |
532 def _throttle(self, pending): | 542 def _throttle(self, pending): |
533 """Returns True if a commit should be delayed.""" | 543 """Returns True if a commit should be delayed.""" |
534 if pending.postpone(): | 544 if pending.postpone(): |
535 self.context.status.send( | 545 self.context.status.send( |
536 pending, | 546 pending, |
537 {'verification': 'why not', | 547 {'verification': 'why not', |
538 'payload': { | 548 'payload': { |
539 'message': pending.why_not()}}) | 549 'message': pending.why_not()}}) |
540 return True | 550 return True |
(...skipping 19 matching lines...) Expand all Loading... |
560 """Loads the commit queue state from a JSON file.""" | 570 """Loads the commit queue state from a JSON file.""" |
561 self.queue = model.load_from_json_file(filename) | 571 self.queue = model.load_from_json_file(filename) |
562 | 572 |
563 def save(self, filename): | 573 def save(self, filename): |
564 """Save the commit queue state in a simple JSON file.""" | 574 """Save the commit queue state in a simple JSON file.""" |
565 model.save_to_json_file(filename, self.queue) | 575 model.save_to_json_file(filename, self.queue) |
566 | 576 |
567 def close(self): | 577 def close(self): |
568 """Close all the active pending manager items.""" | 578 """Close all the active pending manager items.""" |
569 self.context.status.close() | 579 self.context.status.close() |
OLD | NEW |