factory_test_tools: merge latest improvements from ToT to R12 factory branch

gft_hwcomp.py

Index: gft_hwcomp.py
diff --git a/gft_hwcomp.py b/gft_hwcomp.py
index ff2b8e0e36b5eeb191b070fc02b65b184adcbd42..e83a80c21fa9414938c51fb547397f8e4b4eb25e 100755
--- a/gft_hwcomp.py
+++ b/gft_hwcomp.py
@@ -9,6 +9,7 @@ import os
 import pprint
 import re
 import sys
+import threading
 
 import flashrom_util
 import gft_common
@@ -19,18 +20,7 @@ from gft_common import DebugMsg, VerboseMsg, WarningMsg, ErrorMsg, ErrorDie
 
 
 def Memorize(f):
-  """ Decorator for functions that need memorization."""
-  memorize_data = {}
-  def memorize_call(*args):
-    index = repr(args)
-    if index in memorize_data:
-      value = memorize_data[index]
-      # DebugMsg('Memorize: using cached value for: %s %s' % (repr(f), index))
-      return value
-    value = f(*args)
-    memorize_data[index] = value
-    return value
-  return memorize_call
+  return gft_common.ThreadSafe(gft_common.Memorize(f))
 
 
 class HardwareComponents(object):
@@ -39,7 +29,7 @@ class HardwareComponents(object):
   # Function names in this class are used for reflection, so please don't change
   # the function names even if they are not comliant to coding style guide.
 
-  version = 5
+  version = 6
 
   # We divide all component IDs (cids) into 5 categories:
   #  - enumerable: able to get the results by running specific commands;
@@ -88,6 +78,13 @@ class HardwareComponents(object):
     'data_recovery_url',
     ]
 
+  # list of cids that should not be fetched asynchronously.
+  _non_async_cids = [
+    # Reading EC will become very slow and cause inaccurate results if we try to
+    # probe components that also fires EC command at the same time.
+    'part_id_ec_flash_chip',
+    ]
+
   # _not_test_cids and _to_be_tested_cids will be re-created for each match.
   _not_test_cids = []
   _to_be_tested_cids = []
@@ -135,6 +132,40 @@ class HardwareComponents(object):
     return loaded
 
   @Memorize
+  def is_legacy_device_record(self, record):
+    """ Returns if a matching record looks like a legacy device. """
+    # Current format: [0-9a-f]{4}:[0-9a-f]{4}
+    return True if re.match('[0-9a-f]{4}:[0-9a-f]{4}', record) else False
+
+  @Memorize
+  def _get_legacy_device_list(self):
+    # pci: cat /proc/bus/pci/devices | cut -f 2 # 0.004s < lspci=0.012s
+    device_list = []
+    pci_device_file = '/proc/bus/pci/devices'
+    if os.path.exists(pci_device_file):
+      with open(pci_device_file) as handle:
+        pci_list = [data.split('\t', 2)[1]
+                    for data in handle.readlines()]
+        device_list += ['%s:%s' % (entry[:4], entry[4:])
+                        for entry in pci_list]
+    else:
+      DebugMsg('Failed to read %s. Execute lspci.' % pci_device_list)
+      pci_list = [entry.split()[2:4]
+                  for entry in
+                  gft_common.SystemOutput('lspci -n -mm').splitlines()]
+      device_list += ['%s:%s' % (vendor.strip('"'), product.strip('"'))
+                      for (vendor, product) in pci_list]
+    # usb: realpath(/sys/bus/usb/devices/*:*)/../id* # 0.05s < lspci=0.078s
+    usb_devs = glob.glob('/sys/bus/usb/devices/*:*')
+    for dev in usb_devs:
+      path = os.path.join(os.path.realpath(dev), '..')
+      device_list += ['%s:%s' %
+                      (gft_common.ReadOneLine(os.path.join(path, 'idVendor')),
+                       gft_common.ReadOneLine(os.path.join(path, 'idProduct')))]
+    DebugMsg('Legacy device list: ' + ', '.join(device_list))
+    return device_list
+
+  @Memorize
   def _get_all_connection_info(self):
     """ Probes available connectivity and device information """
     connection_info = {
@@ -633,9 +664,12 @@ class HardwareComponents(object):
         approved[cid] = '*'
     return approved
 
-  def get_all_enumerable_components(self):
+  def get_all_enumerable_components(self, async):
     results = {}
-    for cid in self._enumerable_cids:
+    thread_pools = []
+
+    def fetch_enumerable_component(cid):
+      """ Fetch an enumerable component and update the results """
       if self._verbose:
         sys.stdout.flush()
         sys.stderr.write('<Fetching property %s>\n' % cid)
@@ -643,18 +677,52 @@ class HardwareComponents(object):
       if not isinstance(components, list):
         components = [components]
       results[cid] = components
+
+    class FetchThread(threading.Thread):
+      """ Thread object for parallel enumerating """
+      def __init__(self, cid):
+        threading.Thread.__init__(self)
+        self.cid = cid
+
+      def run(self):
+        fetch_enumerable_component(self.cid)
+
+    for cid in self._enumerable_cids:
+      if async and cid not in self._non_async_cids:
+        thread_pools.append(FetchThread(cid))
+      else:
+        fetch_enumerable_component(cid)
+
+    # Complete the threads
+    for thread in thread_pools:
+      thread.start()
+    for thread in thread_pools:
+      thread.join()
     return results
 
   def check_enumerable_component(self, cid, exact_values, approved_values):
     if '*' in approved_values:
       return
 
-    for value in exact_values:
-      if value not in approved_values:
-        if cid in self._failures:
-          self._failures[cid].append(value)
-        else:
-          self._failures[cid] = [value]
+    unmatched = [value for value in exact_values
+                 if value not in approved_values]
+    if not unmatched:
+      return
+
+    # there's some error, let's try to match them in legacy list
+    legacy_approved = filter(self.is_legacy_device_record, approved_values)
+    if set(legacy_approved) == set(approved_values):
+      DebugMsg('Start legacy search for cid: ' + cid)
+      # safe for legacy match
+      # TODO(hungte) prefetch this list in async batch process.
+      legacy_list = self._get_legacy_device_list()
+      matched = list(set(legacy_list).intersection(set(approved_values)))
+      if matched:
+        DebugMsg('Changed detected list: %s->%s' % (self._system[cid], matched))
+        self._system[cid] = matched
+        return
+    # update with remaining error.
+    self._failures[cid] = unmatched
 
   @Memorize
   def verify_probable_component(self, cid, approved_values):
@@ -677,12 +745,12 @@ class HardwareComponents(object):
   def pformat(self, obj):
     return '\n' + self._pp.pformat(obj) + '\n'
 
-  def initialize(self, force=False):
+  def initialize(self, force=False, async=False):
     if self._initialized and not force:
       return
     # probe current system components
     DebugMsg('Starting to detect system components...')
-    self._enumerable_system = self.get_all_enumerable_components()
+    self._enumerable_system = self.get_all_enumerable_components(async)
     self._initialized = True
 
   def match_current_system(self, filename, ignored_cids=[]):
@@ -721,15 +789,33 @@ class HardwareComponents(object):
 # Console main entry
 @gft_common.GFTConsole
 def _main(self_path, args):
-  if not args:
-    print 'Usage: %s components_db_files...\n' % self_path
-    sys.exit(1)
-  components = HardwareComponents(verbose=True)
-  print 'Starting to probe system...'
-  components.initialize()
-  print 'Starting to match system...'
+  do_async = True
 
+  # preprocess args
+  compdb_list = []
   for arg in args:
+    if arg == '--sync':
+      do_async = False
+    elif arg == '--async':
+      do_async = True
+    elif not os.path.exists(arg):
+      print 'ERROR: unknown parameter: ' + arg
+      print 'Usage: %s [--sync|--async] [components_db_files...]\n' % self_path
+      sys.exit(1)
+    else:
+      compdb_list.append(arg)
+
+  components = HardwareComponents(verbose=True)
+  print 'Starting to detect%s...' % (' asynchrounously' if do_async else '')
+  components.initialize(async=do_async)
+
+  if not compdb_list:
+    print 'Enumerable properties:'
+    print components.pformat(components._enumerable_system)
+    sys.exit(0)
+
+  print 'Starting to match system...'
+  for arg in compdb_list:
     (matched, failures) = components.match_current_system(arg)
     print 'Probed (%s):' % arg
     print components.pformat(matched)