Log when leaving main and list remaining threads in builder_alerts

infra/services/builder_alerts/__main__.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import argparse
 import contextlib
 import cStringIO
 import datetime
 import gzip
 import json
 import logging
 import multiprocessing
 import os
 import sys
+import threading
 import traceback
 
 import requests
 import requests_cache
 
 from infra_libs import logs
 from infra_libs import ts_mon
 from infra.libs.service_utils import outer_loop
 
 from infra.services.builder_alerts import alert_builder
@@ skipping 27 matching lines @@
 class SubProcess(object):
 
   def __init__(self, cache, old_alerts, builder_filter, jobs):
     super(SubProcess, self).__init__()
     self._cache = cache
     self._old_alerts = old_alerts
     self._builder_filter = builder_filter
     self._jobs = jobs
 
   def __call__(self, master_url):
+    logging.debug('Thread for master %s has started', master_url)
     try:
       master_json = buildbot.fetch_master_json(master_url)
       if not master_json:
         return (None, None, None, master_url)
 
       master_alerts, stale_master_alert = alert_builder.alerts_for_master(
           self._cache, master_url, master_json, self._old_alerts,
           self._builder_filter, self._jobs)
 
       # FIXME: The builder info doesn't really belong here. The builder
       # revisions tool uses this and we happen to have the builder json cached
       # at this point so it's cheap to compute, but it should be moved
       # to a different feed.
       data, stale_builder_alerts = (
           buildbot.latest_builder_info_and_alerts_for_master(
               self._cache, master_url, master_json))
       if stale_master_alert:
         stale_builder_alerts.append(stale_master_alert)
       return (master_alerts, data, stale_builder_alerts, master_url)
     except:
       # Put all exception text into an exception and raise that so it doesn't
       # get eaten by the multiprocessing code.
       msg = '%s for master url %s' % (
           ''.join(traceback.format_exception(*sys.exc_info())),
           master_url,
       )
       raise Exception(msg)
+    finally:
+      logging.debug('Thread for master %s has finished', master_url)
 
 
 def query_findit(findit_api_url, alerts):
   """Get analysis results from Findit for failures in the given alerts.
 
   Args:
     findit_api_url (str): The URL to findit's api for build failure analysis.
     alerts (list): A non-empty list of failure alerts.
 
   Returns:
@@ skipping 82 matching lines @@
 
 
 def gzipped(data):
   s = cStringIO.StringIO()
   with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
     g.write(data)
   return s.getvalue()
 
 
 def inner_loop(args):
+  logging.debug('Starting inner loop')
   old_api_endpoint = string_helpers.slash_join(args.api_endpoint_prefix,
       args.old_api_path) if args.old_api_path else None
   if not old_api_endpoint:
     logging.warn(
         'No /data url passed, will write to builder_alerts.json. JSON posted '
         'to new API endpoints will be written to builder_alerts_<tree>.json '
         'files.')
 
   if args.use_cache:
     requests_cache.install_cache('failure_stats')
@@ skipping 40 matching lines @@
                 ' %s' % (master, builder, step, reason))
 
           old_alerts[alert_key] = alert
 
   latest_builder_info = {}
   stale_builder_alerts = []
   missing_masters = []
   alerts = []
   suspected_cls = []
 
+  logging.debug('Processing all masters via process pool')
   pool = multiprocessing.Pool(processes=args.processes)
   master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
                                      args.jobs), master_urls)
+  logging.debug('Closing all threads in master process pool')
   pool.close()
   pool.join()
+  logging.debug('Joined all threads in master process pool')
 
   for data in master_datas:
     # TODO(ojan): We should put an alert in the JSON for this master so
     # we can show that the master is down in the sheriff-o-matic UI.
     if not data[0]:
       missing_masters.extend([data[3]])
       continue
     alerts.extend(data[0])
     latest_builder_info.update(data[1])
     stale_builder_alerts.extend(data[2])
@@ skipping 70 matching lines @@
                               resp.status_code, resp.reason, resp.content)
             ret = False
         else:
           with open('builder_alerts_%s.json' % tree, 'w') as f:
             f.write(json.dumps(json_data, indent=1))
   else:
     logging.error(
         '--crbug-service-account was not specified, can not get crbug issues')
     ret = False
 
+  logging.debug('Returning from inner loop')
   return ret
 
 
 def main(args):
   parser = argparse.ArgumentParser(prog='run.py %s' % __package__)
   parser.add_argument('data_url', action='store', nargs='*')  # Deprecated
   parser.add_argument('--use-cache', action='store_true')
   parser.add_argument('--master-filter', action='store')
   parser.add_argument('--builder-filter', action='store')
   parser.add_argument('--processes', default=PARALLEL_TASKS, action='store',
@@ skipping 64 matching lines @@
       def filter(record):
         if record.levelno == logging.INFO:
           return False
         return True
     logging.getLogger('requests.packages.urllib3.connectionpool').addFilter(
         _ConnectionpoolFilter())
 
   def outer_loop_iteration():
     return inner_loop(args)
 
+  logging.debug('Starting outer loop')
   loop_results = outer_loop.loop(
       task=outer_loop_iteration,
       sleep_timeout=lambda: 5,
       **loop_args)
+  logging.debug('Finished outer loop')
 
   logging.debug('Flushing ts_mon starting')
   ts_mon.flush()
   logging.debug('Flushing ts_mon completed')
   return 0 if loop_results.success else 1
 
 
 if __name__ == '__main__':
+  logging.debug('Started main')
   sys.exit(main(sys.argv[1:]))
+  current_thread_descriptions = [t.name + (' (daemon)' if t.isDaemon() else '')

[tandrii(chromium), 2016/07/26 09:56:35]

this will never be executed. sys.exit raises SystemExit exception.

+                                 for t in threading.enumerate()]
+  logging.debug(
+      'Leaving main. Threads: %s', ', '.join(current_thread_descriptions))