factory_test_tools: improve gft_hwcomp (hardware_Components) performance

gft_hwcomp.py

 #!/usr/bin/python
 #
 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import glob
 import os
 import pprint
 import re
 import sys
+import threading
 
 import flashrom_util
 import gft_common
 import gft_fwhash
 import vblock
 
 from gft_common import DebugMsg, VerboseMsg, WarningMsg, ErrorMsg, ErrorDie
 
 
 def Memorize(f):
-  """ Decorator for functions that need memorization."""
-  memorize_data = {}
-  def memorize_call(*args):
-    index = repr(args)
-    if index in memorize_data:
-      value = memorize_data[index]
-      # DebugMsg('Memorize: using cached value for: %s %s' % (repr(f), index))
-      return value
-    value = f(*args)
-    memorize_data[index] = value
-    return value
-  return memorize_call
+  return gft_common.ThreadSafe(gft_common.Memorize(f))
 
 
 class HardwareComponents(object):
   """ Hardware Components Scanner """
 
   # Function names in this class are used for reflection, so please don't change
   # the function names even if they are not comliant to coding style guide.
 
-  version = 5
+  version = 6
 
   # We divide all component IDs (cids) into 5 categories:
   #  - enumerable: able to get the results by running specific commands;
   #  - probable: returns existed or not by given some pre-defined choices;
   #  - pure data: data for some special purpose, can't be tested;
 
   _enumerable_cids = [
     'data_display_geometry',
     'hash_ec_firmware',
     'hash_ro_firmware',
@@ skipping 28 matching lines @@
   _probable_cids = [
     'key_recovery',
     'key_root',
     'part_id_cardreader',
     ]
   _pure_data_cids = [
     'data_bitmap_fv',
     'data_recovery_url',
     ]
 
+  # list of cids that should not be fetched asynchronously.
+  _non_async_cids = [
+    # Reading EC will become very slow and cause inaccurate results if we try to
+    # probe components that also fires EC command at the same time.
+    'part_id_ec_flash_chip',
+    ]
+
   # _not_test_cids and _to_be_tested_cids will be re-created for each match.
   _not_test_cids = []
   _to_be_tested_cids = []
 
   # TODO(hungte) unify the 'not available' style messages
   _not_present = ''
   _no_match = 'No match'
 
   def __init__(self, verbose=False):
     self._initialized = False
@@ skipping 525 matching lines @@
   def read_approved_from_file(self, filename):
     approved = gft_common.LoadComponentsDatabaseFile(filename)
     for cid in self._to_be_tested_cids + self._not_test_cids:
       if cid not in approved:
         # If we don't have any listing for this type
         # of part in HWID, it's not required.
         WarningMsg('gft_hwcomp: Bypassing unlisted cid %s' % cid)
         approved[cid] = '*'
     return approved
 
-  def get_all_enumerable_components(self):
+  def get_all_enumerable_components(self, async):
     results = {}
-    for cid in self._enumerable_cids:
+    thread_pools = []
+
+    def fetch_enumerable_component(cid):
+      """ Fetch an enumerable component and update the results """
       if self._verbose:
         sys.stdout.flush()
         sys.stderr.write('<Fetching property %s>\n' % cid)
       components = self.force_get_property('get_' + cid)
       if not isinstance(components, list):
         components = [components]
       results[cid] = components
+
+    class FetchThread(threading.Thread):
+      """ Thread object for parallel enumerating """
+      def __init__(self, cid):
+        threading.Thread.__init__(self)
+        self.cid = cid
+
+      def run(self):
+        fetch_enumerable_component(self.cid)
+
+    for cid in self._enumerable_cids:
+      if async and cid not in self._non_async_cids:
+        thread_pools.append(FetchThread(cid))
+      else:
+        fetch_enumerable_component(cid)
+
+    # Complete the threads
+    for thread in thread_pools:
+      thread.start()
+    for thread in thread_pools:
+      thread.join()
     return results
 
   def check_enumerable_component(self, cid, exact_values, approved_values):
     if '*' in approved_values:
       return
 
     for value in exact_values:
       if value not in approved_values:
         if cid in self._failures:
           self._failures[cid].append(value)
@@ skipping 14 matching lines @@
   def check_probable_component(self, cid, approved_values):
     (probed, value) = self.verify_probable_component(cid, approved_values)
     if probed:
       self._system[cid] = value
     else:
       self._failures[cid] = value
 
   def pformat(self, obj):
     return '\n' + self._pp.pformat(obj) + '\n'
 
-  def initialize(self, force=False):
+  def initialize(self, force=False, async=False):
     if self._initialized and not force:
       return
     # probe current system components
     DebugMsg('Starting to detect system components...')
-    self._enumerable_system = self.get_all_enumerable_components()
+    self._enumerable_system = self.get_all_enumerable_components(async)
     self._initialized = True
 
   def match_current_system(self, filename, ignored_cids=[]):
     """ Matches a given component list to current system.
         Returns: (current, failures)
     """
 
     # assert self._initialized, 'Not initialized.'
     self._to_be_tested_cids = (self._enumerable_cids +
                                self._probable_cids)
@@ skipping 18 matching lines @@
       else:
         self.check_probable_component(cid, approved[cid])
 
     return (self._system, self._failures)
 
 
 #############################################################################
 # Console main entry
 @gft_common.GFTConsole
 def _main(self_path, args):
-  if not args:
-    print 'Usage: %s components_db_files...\n' % self_path
-    sys.exit(1)
+  do_async = True
+
+  # preprocess args
+  compdb_list = []
+  for arg in args:
+    if arg == '--sync':
+      do_async = False
+    elif arg == '--async':
+      do_async = True
+    elif not os.path.exists(arg):
+      print 'ERROR: unknown parameter: ' + arg
+      print 'Usage: %s [--sync|--async] [components_db_files...]\n' % self_path
+      sys.exit(1)
+    else:
+      compdb_list.append(arg)
+
   components = HardwareComponents(verbose=True)
-  print 'Starting to probe system...'
-  components.initialize()
+  print 'Starting to detect %s...' % ('Asynchronously' if do_async else '')

[Tom Wai-Hong Tam, 2011/04/13 01:19:36]

Asynchronously -> asynchronously

+  components.initialize(async=do_async)
+
+  if not compdb_list:
+    print 'Enumerable properties:'
+    print components.pformat(components._enumerable_system)
+    sys.exit(0)
+
   print 'Starting to match system...'
-
-  for arg in args:
+  for arg in compdb_list:
     (matched, failures) = components.match_current_system(arg)
     print 'Probed (%s):' % arg
     print components.pformat(matched)
     print 'Failures (%s):' % arg
     print components.pformat(failures)
 
 if __name__ == '__main__':
   _main(sys.argv[0], sys.argv[1:])