| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 """Commit queue executable. | |
| 6 | |
| 7 Reuse Rietveld and the Chromium Try Server to process and automatically commit | |
| 8 patches. | |
| 9 """ | |
| 10 | |
| 11 import logging | |
| 12 import logging.handlers | |
| 13 import optparse | |
| 14 import os | |
| 15 import shutil | |
| 16 import signal | |
| 17 import socket | |
| 18 import sys | |
| 19 import tempfile | |
| 20 import time | |
| 21 | |
| 22 import find_depot_tools # pylint: disable=W0611 | |
| 23 import checkout | |
| 24 import fix_encoding | |
| 25 import rietveld | |
| 26 import subprocess2 | |
| 27 | |
| 28 import async_push | |
| 29 import cq_alerts | |
| 30 import creds | |
| 31 import errors | |
| 32 import projects | |
| 33 import sig_handler | |
| 34 | |
| 35 | |
| 36 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| 37 | |
| 38 | |
| 39 class OnlyIssueRietveld(rietveld.Rietveld): | |
| 40 """Returns a single issue for end-to-end in prod testing.""" | |
| 41 def __init__(self, url, email, password, extra_headers, only_issue): | |
| 42 super(OnlyIssueRietveld, self).__init__(url, email, password, extra_headers) | |
| 43 self._only_issue = only_issue | |
| 44 | |
| 45 def get_pending_issues(self): | |
| 46 """If it's set to return a single issue, only return this one.""" | |
| 47 if self._only_issue: | |
| 48 return [self._only_issue] | |
| 49 return [] | |
| 50 | |
| 51 def get_issue_properties(self, issue, messages): | |
| 52 """Hacks the result to fake that the issue has the commit bit set.""" | |
| 53 data = super(OnlyIssueRietveld, self).get_issue_properties(issue, messages) | |
| 54 if issue == self._only_issue: | |
| 55 data['commit'] = True | |
| 56 return data | |
| 57 | |
| 58 def set_flag(self, issue, patchset, flag, value): | |
| 59 if issue == self._only_issue and flag == 'commit' and value == 'False': | |
| 60 self._only_issue = None | |
| 61 return super(OnlyIssueRietveld, self).set_flag(issue, patchset, flag, value) | |
| 62 | |
| 63 | |
| 64 class FakeCheckout(object): | |
| 65 def __init__(self): | |
| 66 self.project_path = os.getcwd() | |
| 67 self.project_name = os.path.basename(self.project_path) | |
| 68 | |
| 69 @staticmethod | |
| 70 def prepare(_revision): | |
| 71 logging.info('FakeCheckout is syncing') | |
| 72 return unicode('FAKE') | |
| 73 | |
| 74 @staticmethod | |
| 75 def apply_patch(*_args): | |
| 76 logging.info('FakeCheckout is applying a patch') | |
| 77 | |
| 78 @staticmethod | |
| 79 def commit(*_args): | |
| 80 logging.info('FakeCheckout is committing patch') | |
| 81 return 'FAKED' | |
| 82 | |
| 83 @staticmethod | |
| 84 def get_settings(_key): | |
| 85 return None | |
| 86 | |
| 87 @staticmethod | |
| 88 def revisions(*_args): | |
| 89 return None | |
| 90 | |
| 91 | |
| 92 def AlertOnUncleanCheckout(): | |
| 93 """Sends an alert if the cq is running live with local edits.""" | |
| 94 diff = subprocess2.capture(['gclient', 'diff'], cwd=ROOT_DIR).strip() | |
| 95 if diff: | |
| 96 cq_alerts.SendAlert( | |
| 97 'CQ running with local diff.', | |
| 98 ('Ruh-roh! Commit queue was started with an unclean checkout.\n\n' | |
| 99 '$ gclient diff\n%s' % diff)) | |
| 100 | |
| 101 | |
| 102 def SetupLogging(options): | |
| 103 """Configures the logging module.""" | |
| 104 logging.getLogger().setLevel(logging.DEBUG) | |
| 105 if options.verbose: | |
| 106 level = logging.DEBUG | |
| 107 else: | |
| 108 level = logging.INFO | |
| 109 console_logging = logging.StreamHandler() | |
| 110 console_logging.setFormatter(logging.Formatter( | |
| 111 '%(asctime)s %(levelname)7s %(message)s')) | |
| 112 console_logging.setLevel(level) | |
| 113 logging.getLogger().addHandler(console_logging) | |
| 114 | |
| 115 log_directory = 'logs-' + options.project | |
| 116 if not os.path.exists(log_directory): | |
| 117 os.mkdir(log_directory) | |
| 118 | |
| 119 logging_rotating_file = logging.handlers.RotatingFileHandler( | |
| 120 filename=os.path.join(log_directory, 'commit_queue.log'), | |
| 121 maxBytes= 10*1024*1024, | |
| 122 backupCount=50) | |
| 123 logging_rotating_file.setLevel(logging.DEBUG) | |
| 124 logging_rotating_file.setFormatter(logging.Formatter( | |
| 125 '%(asctime)s %(levelname)-8s %(module)15s(%(lineno)4d): %(message)s')) | |
| 126 logging.getLogger().addHandler(logging_rotating_file) | |
| 127 | |
| 128 | |
| 129 class SignalInterrupt(Exception): | |
| 130 """Exception that indicates being interrupted by a caught signal.""" | |
| 131 | |
| 132 def __init__(self, signal_set=None, *args, **kwargs): | |
| 133 super(SignalInterrupt, self).__init__(*args, **kwargs) | |
| 134 self.signal_set = signal_set | |
| 135 | |
| 136 | |
| 137 def SaveDatabaseCopyForDebugging(db_path): | |
| 138 """Saves database file for debugging. Returns name of the saved file.""" | |
| 139 with tempfile.NamedTemporaryFile( | |
| 140 dir=os.path.dirname(db_path), | |
| 141 prefix='db.debug.', | |
| 142 suffix='.json', | |
| 143 delete=False) as tmp_file: | |
| 144 with open(db_path) as db_file: | |
| 145 shutil.copyfileobj(db_file, tmp_file) | |
| 146 return tmp_file.name | |
| 147 | |
| 148 | |
| 149 def main(): | |
| 150 # Set a default timeout for sockets. This is critical when talking to remote | |
| 151 # services like AppEngine and buildbot. | |
| 152 # TODO(phajdan.jr): This used to be 70s. Investigate lowering it again. | |
| 153 socket.setdefaulttimeout(60.0 * 15) | |
| 154 | |
| 155 parser = optparse.OptionParser( | |
| 156 description=sys.modules['__main__'].__doc__) | |
| 157 project_choices = projects.supported_projects() | |
| 158 parser.add_option('-v', '--verbose', action='store_true') | |
| 159 parser.add_option( | |
| 160 '--no-dry-run', | |
| 161 action='store_false', | |
| 162 dest='dry_run', | |
| 163 default=True, | |
| 164 help='Run for real instead of dry-run mode which is the default. ' | |
| 165 'WARNING: while the CQ won\'t touch rietveld in dry-run mode, the ' | |
| 166 'Try Server will. So it is recommended to use --only-issue') | |
| 167 parser.add_option( | |
| 168 '--only-issue', | |
| 169 type='int', | |
| 170 help='Limits to a single issue. Useful for live testing; WARNING: it ' | |
| 171 'will fake that the issue has the CQ bit set, so only try with an ' | |
| 172 'issue you don\'t mind about.') | |
| 173 parser.add_option( | |
| 174 '--fake', | |
| 175 action='store_true', | |
| 176 help='Run with a fake checkout to speed up testing') | |
| 177 parser.add_option( | |
| 178 '--no-try', | |
| 179 action='store_true', | |
| 180 help='Don\'t send try jobs.') | |
| 181 parser.add_option( | |
| 182 '-p', | |
| 183 '--poll-interval', | |
| 184 type='int', | |
| 185 default=10, | |
| 186 help='Minimum delay between each polling loop, default: %default') | |
| 187 parser.add_option( | |
| 188 '--query-only', | |
| 189 action='store_true', | |
| 190 help='Return internal state') | |
| 191 parser.add_option( | |
| 192 '--project', | |
| 193 choices=project_choices, | |
| 194 help='Project to run the commit queue against: %s' % | |
| 195 ', '.join(project_choices)) | |
| 196 parser.add_option( | |
| 197 '-u', | |
| 198 '--user', | |
| 199 default='commit-bot@chromium.org', | |
| 200 help='User to use instead of %default') | |
| 201 parser.add_option( | |
| 202 '--rietveld', | |
| 203 default='https://codereview.chromium.org', | |
| 204 help='Rietveld server to use instead of %default') | |
| 205 options, args = parser.parse_args() | |
| 206 if args: | |
| 207 parser.error('Unsupported args: %s' % args) | |
| 208 if not options.project: | |
| 209 parser.error('Need to pass a valid project to --project.\nOptions are: %s' % | |
| 210 ', '.join(project_choices)) | |
| 211 | |
| 212 SetupLogging(options) | |
| 213 try: | |
| 214 work_dir = os.path.join(ROOT_DIR, 'workdir') | |
| 215 # Use our specific subversion config. | |
| 216 checkout.SvnMixIn.svn_config = checkout.SvnConfig( | |
| 217 os.path.join(ROOT_DIR, 'subversion_config')) | |
| 218 | |
| 219 url = options.rietveld | |
| 220 gaia_creds = creds.Credentials(os.path.join(work_dir, '.gaia_pwd')) | |
| 221 if options.dry_run: | |
| 222 logging.debug('Dry run - skipping SCM check.') | |
| 223 if options.only_issue: | |
| 224 parser.error('--only-issue is not supported with dry run') | |
| 225 else: | |
| 226 print('Using read-only Rietveld') | |
| 227 # Make sure rietveld is not modified. Pass empty email and | |
| 228 # password to bypass authentication; this additionally | |
| 229 # guarantees rietveld will not allow any changes. | |
| 230 rietveld_obj = rietveld.ReadOnlyRietveld(url, email='', password='') | |
| 231 else: | |
| 232 AlertOnUncleanCheckout() | |
| 233 print('WARNING: The Commit Queue is going to commit stuff') | |
| 234 if options.only_issue: | |
| 235 print('Using only issue %d' % options.only_issue) | |
| 236 rietveld_obj = OnlyIssueRietveld( | |
| 237 url, | |
| 238 options.user, | |
| 239 gaia_creds.get(options.user), | |
| 240 None, | |
| 241 options.only_issue) | |
| 242 else: | |
| 243 rietveld_obj = rietveld.Rietveld( | |
| 244 url, | |
| 245 options.user, | |
| 246 gaia_creds.get(options.user), | |
| 247 None) | |
| 248 | |
| 249 pc = projects.load_project( | |
| 250 options.project, | |
| 251 options.user, | |
| 252 work_dir, | |
| 253 rietveld_obj, | |
| 254 options.no_try) | |
| 255 | |
| 256 if options.dry_run: | |
| 257 if options.fake: | |
| 258 # Disable the checkout. | |
| 259 print 'Using no checkout' | |
| 260 pc.context.checkout = FakeCheckout() | |
| 261 else: | |
| 262 print 'Using read-only checkout' | |
| 263 pc.context.checkout = checkout.ReadOnlyCheckout(pc.context.checkout) | |
| 264 # Save pushed events on disk. | |
| 265 print 'Using read-only chromium-status interface' | |
| 266 pc.context.status = async_push.AsyncPushStore() | |
| 267 | |
| 268 landmine_path = os.path.join(work_dir, | |
| 269 pc.context.checkout.project_name + '.landmine') | |
| 270 db_path = os.path.join(work_dir, pc.context.checkout.project_name + '.json') | |
| 271 if os.path.isfile(db_path): | |
| 272 if os.path.isfile(landmine_path): | |
| 273 debugging_path = SaveDatabaseCopyForDebugging(db_path) | |
| 274 os.remove(db_path) | |
| 275 logging.warning(('Deleting database because previous shutdown ' | |
| 276 'was unclean. The copy of the database is saved ' | |
| 277 'as %s.') % debugging_path) | |
| 278 else: | |
| 279 try: | |
| 280 pc.load(db_path) | |
| 281 except ValueError as e: | |
| 282 debugging_path = SaveDatabaseCopyForDebugging(db_path) | |
| 283 os.remove(db_path) | |
| 284 logging.warning(('Failed to parse database (%r), deleting it. ' | |
| 285 'The copy of the database is saved as %s.') % | |
| 286 (e, debugging_path)) | |
| 287 raise e | |
| 288 | |
| 289 # Create a file to indicate unclean shutdown. | |
| 290 with open(landmine_path, 'w'): | |
| 291 pass | |
| 292 | |
| 293 sig_handler.installHandlers( | |
| 294 signal.SIGINT, | |
| 295 signal.SIGHUP | |
| 296 ) | |
| 297 | |
| 298 # Sync every 5 minutes. | |
| 299 SYNC_DELAY = 5*60 | |
| 300 try: | |
| 301 if options.query_only: | |
| 302 pc.look_for_new_pending_commit() | |
| 303 pc.update_status() | |
| 304 print(str(pc.queue)) | |
| 305 os.remove(landmine_path) | |
| 306 return 0 | |
| 307 | |
| 308 now = time.time() | |
| 309 next_loop = now + options.poll_interval | |
| 310 # First sync is on second loop. | |
| 311 next_sync = now + options.poll_interval * 2 | |
| 312 while True: | |
| 313 # In theory, we would gain in performance to parallelize these tasks. In | |
| 314 # practice I'm not sure it matters. | |
| 315 pc.look_for_new_pending_commit() | |
| 316 pc.process_new_pending_commit() | |
| 317 pc.update_status() | |
| 318 pc.scan_results() | |
| 319 if sig_handler.getTriggeredSignals(): | |
| 320 raise SignalInterrupt(signal_set=sig_handler.getTriggeredSignals()) | |
| 321 # Save the db at each loop. The db can easily be in the 1mb range so | |
| 322 # it's slowing down the CQ a tad but it in the 100ms range even for that | |
| 323 # size. | |
| 324 pc.save(db_path) | |
| 325 | |
| 326 # More than a second to wait and due to sync. | |
| 327 now = time.time() | |
| 328 if (next_loop - now) >= 1 and (next_sync - now) <= 0: | |
| 329 if sys.stdout.isatty(): | |
| 330 sys.stdout.write('Syncing while waiting \r') | |
| 331 sys.stdout.flush() | |
| 332 try: | |
| 333 pc.context.checkout.prepare(None) | |
| 334 except subprocess2.CalledProcessError as e: | |
| 335 # Don't crash, most of the time it's the svn server that is dead. | |
| 336 # How fun. Send a stack trace to annoy the maintainer. | |
| 337 errors.send_stack(e) | |
| 338 next_sync = time.time() + SYNC_DELAY | |
| 339 | |
| 340 now = time.time() | |
| 341 next_loop = max(now, next_loop) | |
| 342 while True: | |
| 343 # Abort if any signals are set | |
| 344 if sig_handler.getTriggeredSignals(): | |
| 345 raise SignalInterrupt(signal_set=sig_handler.getTriggeredSignals()) | |
| 346 delay = next_loop - now | |
| 347 if delay <= 0: | |
| 348 break | |
| 349 if sys.stdout.isatty(): | |
| 350 sys.stdout.write('Sleeping for %1.1f seconds \r' % delay) | |
| 351 sys.stdout.flush() | |
| 352 time.sleep(min(delay, 0.1)) | |
| 353 now = time.time() | |
| 354 if sys.stdout.isatty(): | |
| 355 sys.stdout.write('Running (please do not interrupt) \r') | |
| 356 sys.stdout.flush() | |
| 357 next_loop = time.time() + options.poll_interval | |
| 358 except: # Catch all fatal exit conditions. | |
| 359 logging.exception('CQ loop terminating') | |
| 360 raise | |
| 361 finally: | |
| 362 logging.warning('Saving db...') | |
| 363 pc.save(db_path) | |
| 364 pc.close() | |
| 365 logging.warning('db save successful.') | |
| 366 except SignalInterrupt: | |
| 367 # This is considered a clean shutdown: we only throw this exception | |
| 368 # from selected places in the code where the database should be | |
| 369 # in a known and consistent state. | |
| 370 os.remove(landmine_path) | |
| 371 | |
| 372 print 'Bye bye (SignalInterrupt)' | |
| 373 # 23 is an arbitrary value to signal loop.sh that it must stop looping. | |
| 374 return 23 | |
| 375 except KeyboardInterrupt: | |
| 376 # This is actually an unclean shutdown. Do not remove the landmine file. | |
| 377 # One example of this is user hitting ctrl-c twice at an arbitrary point | |
| 378 # inside the CQ loop. There are no guarantees about consistent state | |
| 379 # of the database then. | |
| 380 | |
| 381 print 'Bye bye (KeyboardInterrupt - this is considered unclean shutdown)' | |
| 382 # 23 is an arbitrary value to signal loop.sh that it must stop looping. | |
| 383 return 23 | |
| 384 except errors.ConfigurationError as e: | |
| 385 parser.error(str(e)) | |
| 386 return 1 | |
| 387 | |
| 388 # CQ generally doesn't exit by itself, but if we ever get here, it looks | |
| 389 # like a clean shutdown so remove the landmine file. | |
| 390 # TODO(phajdan.jr): Do we ever get here? | |
| 391 os.remove(landmine_path) | |
| 392 return 0 | |
| 393 | |
| 394 | |
| 395 if __name__ == '__main__': | |
| 396 fix_encoding.fix_encoding() | |
| 397 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 135363007:
Delete public commit queue to avoid confusion after move to internal repo (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/