Add a snapshot of flutter/engine/src/build to our sdk

build/android/buildbot/bb_device_status_check.py

+#!/usr/bin/env python
+#
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A class to keep track of devices across builds and report state."""
+import json
+import logging
+import optparse
+import os
+import psutil
+import re
+import signal
+import smtplib
+import subprocess
+import sys
+import time
+import urllib
+
+import bb_annotations
+import bb_utils
+
+sys.path.append(os.path.join(os.path.dirname(__file__),
+                             os.pardir, os.pardir, 'util', 'lib',
+                             'common'))
+import perf_tests_results_helper  # pylint: disable=F0401
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from pylib import constants
+from pylib.cmd_helper import GetCmdOutput
+from pylib.device import adb_wrapper
+from pylib.device import battery_utils
+from pylib.device import device_blacklist
+from pylib.device import device_errors
+from pylib.device import device_list
+from pylib.device import device_utils
+from pylib.utils import run_tests_helper
+
+_RE_DEVICE_ID = re.compile('Device ID = (\d+)')
+
+def DeviceInfo(device, options):
+  """Gathers info on a device via various adb calls.
+
+  Args:
+    device: A DeviceUtils instance for the device to construct info about.
+
+  Returns:
+    Tuple of device type, build id, report as a string, error messages, and
+    boolean indicating whether or not device can be used for testing.
+  """
+  battery = battery_utils.BatteryUtils(device)
+
+  build_product = ''
+  build_id = ''
+  battery_level = 100
+  errors = []
+  dev_good = True
+  json_data = {}
+
+  try:
+    build_product = device.build_product
+    build_id = device.build_id
+
+    json_data = {
+      'serial': device.adb.GetDeviceSerial(),
+      'type': build_product,
+      'build': build_id,
+      'build_detail': device.GetProp('ro.build.fingerprint'),
+      'battery': {},
+      'imei_slice': 'Unknown',
+      'wifi_ip': device.GetProp('dhcp.wlan0.ipaddress'),
+    }
+
+    battery_info = {}
+    try:
+      battery_info = battery.GetBatteryInfo(timeout=5)
+      battery_level = int(battery_info.get('level', battery_level))
+      json_data['battery'] = battery_info
+    except device_errors.CommandFailedError:
+      logging.exception('Failed to get battery information for %s', str(device))
+
+    try:
+      for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
+                                      check_return=True, timeout=5):
+        m = _RE_DEVICE_ID.match(l)
+        if m:
+          json_data['imei_slice'] = m.group(1)[-6:]
+    except device_errors.CommandFailedError:
+      logging.exception('Failed to get IMEI slice for %s', str(device))
+
+    if battery_level < 15:
+      errors += ['Device critically low in battery.']
+      dev_good = False
+      if not battery.GetCharging():
+        battery.SetCharging(True)
+    if not options.no_provisioning_check:
+      setup_wizard_disabled = (
+          device.GetProp('ro.setupwizard.mode') == 'DISABLED')
+      if not setup_wizard_disabled and device.build_type != 'user':
+        errors += ['Setup wizard not disabled. Was it provisioned correctly?']
+    if (device.product_name == 'mantaray' and
+        battery_info.get('AC powered', None) != 'true'):
+      errors += ['Mantaray device not connected to AC power.']
+  except device_errors.CommandFailedError:
+    logging.exception('Failure while getting device status.')
+    dev_good = False
+  except device_errors.CommandTimeoutError:
+    logging.exception('Timeout while getting device status.')
+    dev_good = False
+
+  return (build_product, build_id, battery_level, errors, dev_good, json_data)
+
+
+def CheckForMissingDevices(options, devices):
+  """Uses file of previous online devices to detect broken phones.
+
+  Args:
+    options: out_dir parameter of options argument is used as the base
+      directory to load and update the cache file.
+    devices: A list of DeviceUtils instance for the currently visible and
+      online attached devices.
+  """
+  out_dir = os.path.abspath(options.out_dir)
+  device_serials = set(d.adb.GetDeviceSerial() for d in devices)
+
+  # last_devices denotes all known devices prior to this run
+  last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
+  last_missing_devices_path = os.path.join(out_dir,
+      device_list.LAST_MISSING_DEVICES_FILENAME)
+  try:
+    last_devices = device_list.GetPersistentDeviceList(last_devices_path)
+  except IOError:
+    # Ignore error, file might not exist
+    last_devices = []
+
+  try:
+    last_missing_devices = device_list.GetPersistentDeviceList(
+        last_missing_devices_path)
+  except IOError:
+    last_missing_devices = []
+
+  missing_devs = list(set(last_devices) - device_serials)
+  new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
+
+  if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
+    logging.info('new_missing_devs %s' % new_missing_devs)
+    devices_missing_msg = '%d devices not detected.' % len(missing_devs)
+    bb_annotations.PrintSummaryText(devices_missing_msg)
+
+    from_address = 'chrome-bot@chromium.org'
+    to_addresses = ['chrome-labs-tech-ticket@google.com',
+                    'chrome-android-device-alert@google.com']
+    cc_addresses = ['chrome-android-device-alert@google.com']
+    subject = 'Devices offline on %s, %s, %s' % (
+      os.environ.get('BUILDBOT_SLAVENAME'),
+      os.environ.get('BUILDBOT_BUILDERNAME'),
+      os.environ.get('BUILDBOT_BUILDNUMBER'))
+    msg = ('Please reboot the following devices:\n%s' %
+           '\n'.join(map(str, new_missing_devs)))
+    SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
+
+  all_known_devices = list(device_serials | set(last_devices))
+  device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
+  device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
+
+  if not all_known_devices:
+    # This can happen if for some reason the .last_devices file is not
+    # present or if it was empty.
+    return ['No online devices. Have any devices been plugged in?']
+  if missing_devs:
+    devices_missing_msg = '%d devices not detected.' % len(missing_devs)
+    bb_annotations.PrintSummaryText(devices_missing_msg)
+    return ['Current online devices: %s' % ', '.join(d for d in device_serials),
+            '%s are no longer visible. Were they removed?' % missing_devs]
+  else:
+    new_devs = device_serials - set(last_devices)
+    if new_devs and os.path.exists(last_devices_path):
+      bb_annotations.PrintWarning()
+      bb_annotations.PrintSummaryText(
+          '%d new devices detected' % len(new_devs))
+      logging.info('New devices detected:')
+      for d in new_devs:
+        logging.info('  %s', d)
+
+
+def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
+  msg_body = '\r\n'.join(['From: %s' % from_address,
+                          'To: %s' % ', '.join(to_addresses),
+                          'CC: %s' % ', '.join(cc_addresses),
+                          'Subject: %s' % subject, '', msg])
+  try:
+    server = smtplib.SMTP('localhost')
+    server.sendmail(from_address, to_addresses, msg_body)
+    server.quit()
+  except Exception:
+    logging.exception('Failed to send alert email.')
+
+
+def RestartUsb():
+  if not os.path.isfile('/usr/bin/restart_usb'):
+    logging.error('Could not restart usb. ''/usr/bin/restart_usb not '
+                  'installed on host (see BUG=305769).')
+    return False
+
+  lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
+  lsusb_output, _ = lsusb_proc.communicate()
+  if lsusb_proc.returncode:
+    logging.error('Could not get list of USB ports (i.e. lsusb).')
+    return lsusb_proc.returncode
+
+  usb_devices = [re.findall(r'Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
+                 for lsusb_line in lsusb_output.strip().split('\n')]
+
+  all_restarted = True
+  # Walk USB devices from leaves up (i.e reverse sorted) restarting the
+  # connection. If a parent node (e.g. usb hub) is restarted before the
+  # devices connected to it, the (bus, dev) for the hub can change, making the
+  # output we have wrong. This way we restart the devices before the hub.
+  for (bus, dev) in reversed(sorted(usb_devices)):
+    # Can not restart root usb connections
+    if dev != '001':
+      return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
+      if return_code:
+        logging.error('Error restarting USB device /dev/bus/usb/%s/%s',
+                      bus, dev)
+        all_restarted = False
+      else:
+        logging.info('Restarted USB device /dev/bus/usb/%s/%s', bus, dev)
+
+  return all_restarted
+
+
+def KillAllAdb():
+  def GetAllAdb():
+    for p in psutil.process_iter():
+      try:
+        if 'adb' in p.name:
+          yield p
+      except (psutil.NoSuchProcess, psutil.AccessDenied):
+        pass
+
+  for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
+    for p in GetAllAdb():
+      try:
+        logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
+                     ' '.join(p.cmdline))
+        p.send_signal(sig)
+      except (psutil.NoSuchProcess, psutil.AccessDenied):
+        pass
+  for p in GetAllAdb():
+    try:
+      logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
+                    ' '.join(p.cmdline))
+    except (psutil.NoSuchProcess, psutil.AccessDenied):
+      pass
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('', '--out-dir',
+                    help='Directory where the device path is stored',
+                    default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
+  parser.add_option('--no-provisioning-check', action='store_true',
+                    help='Will not check if devices are provisioned properly.')
+  parser.add_option('--device-status-dashboard', action='store_true',
+                    help='Output device status data for dashboard.')
+  parser.add_option('--restart-usb', action='store_true',
+                    help='Restart USB ports before running device check.')
+  parser.add_option('--json-output',
+                    help='Output JSON information into a specified file.')
+  parser.add_option('-v', '--verbose', action='count', default=1,
+                    help='Log more information.')
+
+  options, args = parser.parse_args()
+  if args:
+    parser.error('Unknown options %s' % args)
+
+  run_tests_helper.SetLogLevel(options.verbose)
+
+  # Remove the last build's "bad devices" before checking device statuses.
+  device_blacklist.ResetBlacklist()
+
+  try:
+    expected_devices = device_list.GetPersistentDeviceList(
+        os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
+  except IOError:
+    expected_devices = []
+  devices = device_utils.DeviceUtils.HealthyDevices()
+  device_serials = [d.adb.GetDeviceSerial() for d in devices]
+  # Only restart usb if devices are missing.
+  if set(expected_devices) != set(device_serials):
+    logging.warning('expected_devices: %s', expected_devices)
+    logging.warning('devices: %s', device_serials)
+    KillAllAdb()
+    retries = 5
+    usb_restarted = True
+    if options.restart_usb:
+      if not RestartUsb():
+        usb_restarted = False
+        bb_annotations.PrintWarning()
+        logging.error('USB reset stage failed, '
+                      'wait for any device to come back.')
+    while retries:
+      logging.info('retry adb devices...')
+      time.sleep(1)
+      devices = device_utils.DeviceUtils.HealthyDevices()
+      device_serials = [d.adb.GetDeviceSerial() for d in devices]
+      if set(expected_devices) == set(device_serials):
+        # All devices are online, keep going.
+        break
+      if not usb_restarted and devices:
+        # The USB wasn't restarted, but there's at least one device online.
+        # No point in trying to wait for all devices.
+        break
+      retries -= 1
+
+  types, builds, batteries, errors, devices_ok, json_data = (
+      [], [], [], [], [], [])
+  if devices:
+    types, builds, batteries, errors, devices_ok, json_data = (
+        zip(*[DeviceInfo(dev, options) for dev in devices]))
+
+  # Write device info to file for buildbot info display.
+  if os.path.exists('/home/chrome-bot'):
+    with open('/home/chrome-bot/.adb_device_info', 'w') as f:
+      for device in json_data:
+        try:
+          f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'],
+              device['build'], float(device['battery']['temperature']) / 10,
+              device['battery']['level']))
+        except Exception:
+          pass
+
+  err_msg = CheckForMissingDevices(options, devices) or []
+
+  unique_types = list(set(types))
+  unique_builds = list(set(builds))
+
+  bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
+                           % (len(devices), unique_types, unique_builds))
+
+  for j in json_data:
+    logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
+    logging.info('  Build: %s (%s)', j.get('build'), j.get('build_detail'))
+    logging.info('  Current Battery Service state:')
+    for k, v in j.get('battery', {}).iteritems():
+      logging.info('    %s: %s', k, v)
+    logging.info('  IMEI slice: %s', j.get('imei_slice'))
+    logging.info('  WiFi IP: %s', j.get('wifi_ip'))
+
+
+  for dev, dev_errors in zip(devices, errors):
+    if dev_errors:
+      err_msg += ['%s errors:' % str(dev)]
+      err_msg += ['    %s' % error for error in dev_errors]
+
+  if err_msg:
+    bb_annotations.PrintWarning()
+    for e in err_msg:
+      logging.error(e)
+    from_address = 'buildbot@chromium.org'
+    to_addresses = ['chromium-android-device-alerts@google.com']
+    bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
+    slave_name = os.environ.get('BUILDBOT_SLAVENAME')
+    subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
+    SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
+
+  if options.device_status_dashboard:
+    offline_devices = [
+        device_utils.DeviceUtils(a)
+        for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
+        if a.GetState() == 'offline']
+
+    perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
+                                              [len(devices)], 'devices')
+    perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
+                                              [len(offline_devices)], 'devices',
+                                              'unimportant')
+    for dev, battery in zip(devices, batteries):
+      perf_tests_results_helper.PrintPerfResult('DeviceBattery', str(dev),
+                                                [battery], '%',
+                                                'unimportant')
+
+  if options.json_output:
+    with open(options.json_output, 'wb') as f:
+      f.write(json.dumps(json_data, indent=4))
+
+  num_failed_devs = 0
+  for device_ok, device in zip(devices_ok, devices):
+    if not device_ok:
+      logging.warning('Blacklisting %s', str(device))
+      device_blacklist.ExtendBlacklist([str(device)])
+      num_failed_devs += 1
+
+  if num_failed_devs == len(devices):
+    return 2
+
+  if not devices:
+    return 1
+
+
+if __name__ == '__main__':
+  sys.exit(main())