factory_test_tools: merge latest improvements from ToT to R12 factory branch

gft_hwcomp.py

 #!/usr/bin/python
 #
 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import glob
 import os
 import pprint
 import re
 import sys
+import threading
 
 import flashrom_util
 import gft_common
 import gft_fwhash
 import vblock
 
 from gft_common import DebugMsg, VerboseMsg, WarningMsg, ErrorMsg, ErrorDie
 
 
 def Memorize(f):
-  """ Decorator for functions that need memorization."""
-  memorize_data = {}
-  def memorize_call(*args):
-    index = repr(args)
-    if index in memorize_data:
-      value = memorize_data[index]
-      # DebugMsg('Memorize: using cached value for: %s %s' % (repr(f), index))
-      return value
-    value = f(*args)
-    memorize_data[index] = value
-    return value
-  return memorize_call
+  return gft_common.ThreadSafe(gft_common.Memorize(f))
 
 
 class HardwareComponents(object):
   """ Hardware Components Scanner """
 
   # Function names in this class are used for reflection, so please don't change
   # the function names even if they are not comliant to coding style guide.
 
-  version = 5
+  version = 6
 
   # We divide all component IDs (cids) into 5 categories:
   #  - enumerable: able to get the results by running specific commands;
   #  - probable: returns existed or not by given some pre-defined choices;
   #  - pure data: data for some special purpose, can't be tested;
 
   _enumerable_cids = [
     'data_display_geometry',
     'hash_ec_firmware',
     'hash_ro_firmware',
@@ skipping 28 matching lines @@
   _probable_cids = [
     'key_recovery',
     'key_root',
     'part_id_cardreader',
     ]
   _pure_data_cids = [
     'data_bitmap_fv',
     'data_recovery_url',
     ]
 
+  # list of cids that should not be fetched asynchronously.
+  _non_async_cids = [
+    # Reading EC will become very slow and cause inaccurate results if we try to
+    # probe components that also fires EC command at the same time.
+    'part_id_ec_flash_chip',
+    ]
+
   # _not_test_cids and _to_be_tested_cids will be re-created for each match.
   _not_test_cids = []
   _to_be_tested_cids = []
 
   # TODO(hungte) unify the 'not available' style messages
   _not_present = ''
   _no_match = 'No match'
 
   def __init__(self, verbose=False):
     self._initialized = False
@@ skipping 27 matching lines @@
   @Memorize
   def load_module(self, name):
     grep_cmd = ('lsmod 2>/dev/null | grep -q %s' % name)
     loaded = (os.system(grep_cmd) == 0)
     if not loaded:
       if os.system('modprobe %s >/dev/null 2>&1' % name) != 0:
         ErrorMsg("Cannot load module: %s" % name)
     return loaded
 
   @Memorize
+  def is_legacy_device_record(self, record):
+    """ Returns if a matching record looks like a legacy device. """
+    # Current format: [0-9a-f]{4}:[0-9a-f]{4}
+    return True if re.match('[0-9a-f]{4}:[0-9a-f]{4}', record) else False
+
+  @Memorize
+  def _get_legacy_device_list(self):
+    # pci: cat /proc/bus/pci/devices | cut -f 2 # 0.004s < lspci=0.012s
+    device_list = []
+    pci_device_file = '/proc/bus/pci/devices'
+    if os.path.exists(pci_device_file):
+      with open(pci_device_file) as handle:
+        pci_list = [data.split('\t', 2)[1]
+                    for data in handle.readlines()]
+        device_list += ['%s:%s' % (entry[:4], entry[4:])
+                        for entry in pci_list]
+    else:
+      DebugMsg('Failed to read %s. Execute lspci.' % pci_device_list)
+      pci_list = [entry.split()[2:4]
+                  for entry in
+                  gft_common.SystemOutput('lspci -n -mm').splitlines()]
+      device_list += ['%s:%s' % (vendor.strip('"'), product.strip('"'))
+                      for (vendor, product) in pci_list]
+    # usb: realpath(/sys/bus/usb/devices/*:*)/../id* # 0.05s < lspci=0.078s
+    usb_devs = glob.glob('/sys/bus/usb/devices/*:*')
+    for dev in usb_devs:
+      path = os.path.join(os.path.realpath(dev), '..')
+      device_list += ['%s:%s' %
+                      (gft_common.ReadOneLine(os.path.join(path, 'idVendor')),
+                       gft_common.ReadOneLine(os.path.join(path, 'idProduct')))]
+    DebugMsg('Legacy device list: ' + ', '.join(device_list))
+    return device_list
+
+  @Memorize
   def _get_all_connection_info(self):
     """ Probes available connectivity and device information """
     connection_info = {
         'wireless': '/sys/class/net/wlan0/device',
         'ethernet': '/sys/class/net/eth0/device',
         '3g': '/sys/class/tty/ttyUSB0/device',
     }
     flimflam_scripts_base = '/usr/local/lib/flimflam/test'
     if os.path.exists(flimflam_scripts_base):
       # TODO(hungte) use flimflam to update the device list.
@@ skipping 478 matching lines @@
   def read_approved_from_file(self, filename):
     approved = gft_common.LoadComponentsDatabaseFile(filename)
     for cid in self._to_be_tested_cids + self._not_test_cids:
       if cid not in approved:
         # If we don't have any listing for this type
         # of part in HWID, it's not required.
         WarningMsg('gft_hwcomp: Bypassing unlisted cid %s' % cid)
         approved[cid] = '*'
     return approved
 
-  def get_all_enumerable_components(self):
+  def get_all_enumerable_components(self, async):
     results = {}
-    for cid in self._enumerable_cids:
+    thread_pools = []
+
+    def fetch_enumerable_component(cid):
+      """ Fetch an enumerable component and update the results """
       if self._verbose:
         sys.stdout.flush()
         sys.stderr.write('<Fetching property %s>\n' % cid)
       components = self.force_get_property('get_' + cid)
       if not isinstance(components, list):
         components = [components]
       results[cid] = components
+
+    class FetchThread(threading.Thread):
+      """ Thread object for parallel enumerating """
+      def __init__(self, cid):
+        threading.Thread.__init__(self)
+        self.cid = cid
+
+      def run(self):
+        fetch_enumerable_component(self.cid)
+
+    for cid in self._enumerable_cids:
+      if async and cid not in self._non_async_cids:
+        thread_pools.append(FetchThread(cid))
+      else:
+        fetch_enumerable_component(cid)
+
+    # Complete the threads
+    for thread in thread_pools:
+      thread.start()
+    for thread in thread_pools:
+      thread.join()
     return results
 
   def check_enumerable_component(self, cid, exact_values, approved_values):
     if '*' in approved_values:
       return
 
-    for value in exact_values:
-      if value not in approved_values:
-        if cid in self._failures:
-          self._failures[cid].append(value)
-        else:
-          self._failures[cid] = [value]
+    unmatched = [value for value in exact_values
+                 if value not in approved_values]
+    if not unmatched:
+      return
+
+    # there's some error, let's try to match them in legacy list
+    legacy_approved = filter(self.is_legacy_device_record, approved_values)
+    if set(legacy_approved) == set(approved_values):
+      DebugMsg('Start legacy search for cid: ' + cid)
+      # safe for legacy match
+      # TODO(hungte) prefetch this list in async batch process.
+      legacy_list = self._get_legacy_device_list()
+      matched = list(set(legacy_list).intersection(set(approved_values)))
+      if matched:
+        DebugMsg('Changed detected list: %s->%s' % (self._system[cid], matched))
+        self._system[cid] = matched
+        return
+    # update with remaining error.
+    self._failures[cid] = unmatched
 
   @Memorize
   def verify_probable_component(self, cid, approved_values):
     if '*' in approved_values:
       return (True, ['*'])
 
     for value in approved_values:
       present = getattr(self, 'probe_' + cid)(value)
       if present:
         return (True, [value])
     return (False, [self._no_match])
 
   def check_probable_component(self, cid, approved_values):
     (probed, value) = self.verify_probable_component(cid, approved_values)
     if probed:
       self._system[cid] = value
     else:
       self._failures[cid] = value
 
   def pformat(self, obj):
     return '\n' + self._pp.pformat(obj) + '\n'
 
-  def initialize(self, force=False):
+  def initialize(self, force=False, async=False):
     if self._initialized and not force:
       return
     # probe current system components
     DebugMsg('Starting to detect system components...')
-    self._enumerable_system = self.get_all_enumerable_components()
+    self._enumerable_system = self.get_all_enumerable_components(async)
     self._initialized = True
 
   def match_current_system(self, filename, ignored_cids=[]):
     """ Matches a given component list to current system.
         Returns: (current, failures)
     """
 
     # assert self._initialized, 'Not initialized.'
     self._to_be_tested_cids = (self._enumerable_cids +
                                self._probable_cids)
@@ skipping 18 matching lines @@
       else:
         self.check_probable_component(cid, approved[cid])
 
     return (self._system, self._failures)
 
 
 #############################################################################
 # Console main entry
 @gft_common.GFTConsole
 def _main(self_path, args):
-  if not args:
-    print 'Usage: %s components_db_files...\n' % self_path
-    sys.exit(1)
+  do_async = True
+
+  # preprocess args
+  compdb_list = []
+  for arg in args:
+    if arg == '--sync':
+      do_async = False
+    elif arg == '--async':
+      do_async = True
+    elif not os.path.exists(arg):
+      print 'ERROR: unknown parameter: ' + arg
+      print 'Usage: %s [--sync|--async] [components_db_files...]\n' % self_path
+      sys.exit(1)
+    else:
+      compdb_list.append(arg)
+
   components = HardwareComponents(verbose=True)
-  print 'Starting to probe system...'
-  components.initialize()
+  print 'Starting to detect%s...' % (' asynchrounously' if do_async else '')
+  components.initialize(async=do_async)
+
+  if not compdb_list:
+    print 'Enumerable properties:'
+    print components.pformat(components._enumerable_system)
+    sys.exit(0)
+
   print 'Starting to match system...'
-
-  for arg in args:
+  for arg in compdb_list:
     (matched, failures) = components.match_current_system(arg)
     print 'Probed (%s):' % arg
     print components.pformat(matched)
     print 'Failures (%s):' % arg
     print components.pformat(failures)
 
 if __name__ == '__main__':
   _main(sys.argv[0], sys.argv[1:])