[Android] Make failure to find tombstones nonfatal. (RELAND)

build/android/tombstones.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
 # Find the most recent tombstone file(s) on all connected devices
 # and prints their stacks.
 #
 # Assumes tombstone file was created with current symbols.
 
 import datetime
+import itertools
+import logging
 import multiprocessing
 import os
 import re
 import subprocess
 import sys
 import optparse
 
 from pylib import android_commands
 from pylib.device import device_errors
 from pylib.device import device_utils
+from pylib.utils import run_tests_helper
 
 
 _TZ_UTC = {'TZ': 'UTC'}
 
 def _ListTombstones(device):
   """List the tombstone files on the device.
 
   Args:
     device: An instance of DeviceUtils.
 
   Yields:
     Tuples of (tombstone filename, date time of file on device).
   """
-  lines = device.RunShellCommand(
-      ['ls', '-a', '-l', '/data/tombstones'],
-      as_root=True, check_return=True, env=_TZ_UTC, timeout=60)
-  for line in lines:
-    if 'tombstone' in line and not 'No such file or directory' in line:
-      details = line.split()
-      t = datetime.datetime.strptime(details[-3] + ' ' + details[-2],
-                                     '%Y-%m-%d %H:%M')
-      yield details[-1], t
+  try:
+    lines = device.RunShellCommand(
+        ['ls', '-a', '-l', '/data/tombstones'],
+        as_root=True, check_return=True, env=_TZ_UTC, timeout=60)
+    for line in lines:
+      if 'tombstone' in line and not 'No such file or directory' in line:
+        details = line.split()
+        t = datetime.datetime.strptime(details[-3] + ' ' + details[-2],
+                                       '%Y-%m-%d %H:%M')
+        yield details[-1], t
+  except device_errors.CommandFailedError:
+    logging.exception('Could not retrieve tombstones.')
 
 
 def _GetDeviceDateTime(device):
   """Determine the date time on the device.
 
   Args:
     device: An instance of DeviceUtils.
 
   Returns:
     A datetime instance.
@@ skipping 71 matching lines @@
       break
     yield line
 
 
 def _ResolveTombstone(tombstone):
   lines = []
   lines += [tombstone['file'] + ' created on ' + str(tombstone['time']) +
             ', about this long ago: ' +
             (str(tombstone['device_now'] - tombstone['time']) +
             ' Device: ' + tombstone['serial'])]
-  print '\n'.join(lines)
-  print 'Resolving...'
+  logging.info('\n'.join(lines))
+  logging.info('Resolving...')
   lines += _ResolveSymbols(tombstone['data'], tombstone['stack'],
                            tombstone['device_abi'])
   return lines
 
 
 def _ResolveTombstones(jobs, tombstones):
   """Resolve a list of tombstones.
 
   Args:
     jobs: the number of jobs to use with multiprocess.
     tombstones: a list of tombstones.
   """
   if not tombstones:
-    print 'No device attached?  Or no tombstones?'
+    logging.warning('No tombstones to resolve.')
     return
   if len(tombstones) == 1:
     data = _ResolveTombstone(tombstones[0])
   else:
     pool = multiprocessing.Pool(processes=jobs)
     data = pool.map(_ResolveTombstone, tombstones)
-    data = ['\n'.join(d) for d in data]
-  print '\n'.join(data)
+  for d in data:
+    logging.info(d)
 
 
 def _GetTombstonesForDevice(device, options):
   """Returns a list of tombstones on a given device.
 
   Args:
     device: An instance of DeviceUtils.
     options: command line arguments from OptParse
   """
   ret = []
   all_tombstones = list(_ListTombstones(device))
   if not all_tombstones:
-    print 'No device attached?  Or no tombstones?'
+    logging.warning('No tombstones.')
     return ret
 
   # Sort the tombstones in date order, descending
   all_tombstones.sort(cmp=lambda a, b: cmp(b[1], a[1]))
 
   # Only resolve the most recent unless --all-tombstones given.
   tombstones = all_tombstones if options.all_tombstones else [all_tombstones[0]]
 
   device_now = _GetDeviceDateTime(device)
   try:
     for tombstone_file, tombstone_time in tombstones:
       ret += [{'serial': str(device),
                'device_abi': device.product_cpu_abi,
                'device_now': device_now,
                'time': tombstone_time,
                'file': tombstone_file,
                'stack': options.stack,
                'data': _GetTombstoneData(device, tombstone_file)}]
   except device_errors.CommandFailedError:
     for line in device.RunShellCommand(
         ['ls', '-a', '-l', '/data/tombstones'],
         as_root=True, check_return=True, env=_TZ_UTC, timeout=60):
-      print '%s: %s' % (str(device), line)
+      logging.info('%s: %s', str(device), line)
     raise
 
   # Erase all the tombstones if desired.
   if options.wipe_tombstones:
     for tombstone_file, _ in all_tombstones:
       _EraseTombstone(device, tombstone_file)
 
   return ret
 
 
 def main():
+  custom_handler = logging.StreamHandler(sys.stdout)
+  custom_handler.setFormatter(run_tests_helper.CustomFormatter())
+  logging.getLogger().addHandler(custom_handler)
+  logging.getLogger().setLevel(logging.INFO)
+
   parser = optparse.OptionParser()
   parser.add_option('--device',
                     help='The serial number of the device. If not specified '
                          'will use all devices.')
   parser.add_option('-a', '--all-tombstones', action='store_true',
                     help="""Resolve symbols for all tombstones, rather than just
                          the most recent""")
   parser.add_option('-s', '--stack', action='store_true',
                     help='Also include symbols for stack data')
   parser.add_option('-w', '--wipe-tombstones', action='store_true',
                     help='Erase all tombstones from device after processing')
   parser.add_option('-j', '--jobs', type='int',
                     default=4,
                     help='Number of jobs to use when processing multiple '
                          'crash stacks.')
   options, _ = parser.parse_args()
 
   if options.device:
     devices = [options.device]
   else:
     devices = android_commands.GetAttachedDevices()
 
+  # This must be done serially because strptime can hit a race condition if

[jbudorick, 2015/03/19 01:37:38]

Forgot to upload the original patchset, but the fix is here. I had reworked the
for loop below to instead use DeviceUtils.parallel and run
_GetTombstonesForDevice in parallel across all attached devices, but it ran into
this race condition. To avoid it, I just reverted to the original (serial)
version.

+  # used for the first time in a multithreaded environment.
+  # http://bugs.python.org/issue7980
   tombstones = []
   for device_serial in devices:
     device = device_utils.DeviceUtils(device_serial)
     tombstones += _GetTombstonesForDevice(device, options)
 
   _ResolveTombstones(options.jobs, tombstones)
 
 if __name__ == '__main__':
   sys.exit(main())