[telemetry] Add Python dynamic analysis find_dependencies.

tools/telemetry/telemetry/internal/util/find_dependencies.py

Index: tools/telemetry/telemetry/internal/util/find_dependencies.py
diff --git a/tools/telemetry/telemetry/internal/util/find_dependencies.py b/tools/telemetry/telemetry/internal/util/find_dependencies.py
index 2082565ea83ad0671204fac62240b4fc6b4a82d6..6e652ac708094c5d76c0f95ad660d207dc1efa60 100644
--- a/tools/telemetry/telemetry/internal/util/find_dependencies.py
+++ b/tools/telemetry/telemetry/internal/util/find_dependencies.py
@@ -22,6 +22,12 @@ from telemetry.internal.util import path_set
 DEPS_FILE = 'bootstrap_deps'
 
 
+def _LoadModules(module_paths):
+  for module_path in module_paths:
+    imp.load_source(
+        os.path.splitext(os.path.basename(module_path))[0], module_path)
+
+
 def FindBootstrapDependencies(base_dir):
   deps_file = os.path.join(base_dir, DEPS_FILE)
   if not os.path.exists(deps_file):
@@ -32,18 +38,49 @@ def FindBootstrapDependencies(base_dir):
       for deps_path in deps_paths)
 
 
-def FindPythonDependencies(module_path):
+def FindDynamicAnalysisDependencies(module_paths):

[nednguyen, 2015/08/04 22:22:02]

the method name is ambiguous.

+  logging.info('Finding Python dependencies of %s' % module_paths)
+
+  # Load the modules to inherit their sys.path modifications and imports.
+  _LoadModules(module_paths)

[nednguyen, 2015/08/04 22:22:01]

Wouldn't this includes module imported by this file?

+
+  # Analyze the imports.
+  for module in sys.modules.itervalues():

[nednguyen, 2015/08/04 22:22:02]

So I guess the difference between this and other method is that this also
discovers modules that only become importable due to loading others in
module_paths?

Can you add unittest to show the differences?

+    if not module:
+      continue
+    if not hasattr(module, '__file__'):
+      continue
+
+    module_path = os.path.realpath(module.__file__)
+
+    # If it's a .pyc, change it to .py instead.
+    root, ext = os.path.splitext(module_path)
+    if ext == '.pyc':
+      module_path = root + '.py'
+    if not os.path.exists(module_path):
+      continue
+
+    # If it's an __init__.py, use the package's folder.
+    if module_path.endswith('__init__.py'):
+      module_path = os.path.split(module_path)[0]
+
+    # Filter for only imports in Chromium.
+    if not path.IsSubpath(module_path, path.GetChromiumSrcDir()):
+      continue
+
+    yield module_path
+
+
+def FindStaticAnalysisDependencies(module_path):

[nednguyen, 2015/08/04 22:22:01]

ditto

   logging.info('Finding Python dependencies of %s' % module_path)
 
   # Load the module to inherit its sys.path modifications.
-  imp.load_source(
-      os.path.splitext(os.path.basename(module_path))[0], module_path)
+  _LoadModules([module_path])

[nednguyen, 2015/08/04 22:22:01]

Doesn't modulefinder also execute the sys.path modification?

 
   # Analyze the module for its imports.
   finder = modulefinder.ModuleFinder()
   finder.run_script(module_path)
 
-  # Filter for only imports in Chromium.
   for module in finder.modules.itervalues():
     # If it's an __init__.py, module.__path__ gives the package's folder.
     module_path = module.__path__[0] if module.__path__ else module.__file__
@@ -51,6 +88,8 @@ def FindPythonDependencies(module_path):
       continue
 
     module_path = os.path.realpath(module_path)
+
+    # Filter for only imports in Chromium.
     if not path.IsSubpath(module_path, path.GetChromiumSrcDir()):
       continue
 
@@ -126,23 +165,38 @@ def FindDependencies(target_paths, options):
 
   dependencies = path_set.PathSet()
 
+  # Bootstrap deps.
+  dependencies |= FindBootstrapDependencies(path.GetTelemetryDir())
+
+  # Deps for Telemetry files.
+  def IsSourceDir(dir_name):
+    return dir_name not in ('experimental', 'image_processing', 'third_party')
+  def IsPythonModule(file_name):
+    return os.path.splitext(file_name)[1] == '.py'
+  dependencies |= FindDynamicAnalysisDependencies(
+      path.ListFiles(path.GetTelemetryDir(), IsSourceDir, IsPythonModule))
+
   # Including Telemetry's major entry points will (hopefully) include Telemetry
   # and all its dependencies. If the user doesn't pass any arguments, we just
   # have Telemetry.
-  dependencies |= FindPythonDependencies(os.path.realpath(
+  dependencies |= FindStaticAnalysisDependencies(os.path.realpath(
     os.path.join(path.GetTelemetryDir(), 'telemetry', 'benchmark_runner.py')))
-  dependencies |= FindPythonDependencies(os.path.realpath(
+  dependencies |= FindStaticAnalysisDependencies(os.path.realpath(
     os.path.join(path.GetTelemetryDir(),
                  'telemetry', 'testing', 'run_tests.py')))
-  dependencies |= FindBootstrapDependencies(path.GetTelemetryDir())
 
   # Add dependencies.
   for target_path in target_paths:
-    base_dir = os.path.dirname(os.path.realpath(target_path))
+    target_path = os.path.realpath(target_path)
+    if os.path.isdir(target_path):
+      base_dir = target_path
+    else:
+      dependencies |= FindStaticAnalysisDependencies(target_path)
+      base_dir = os.path.dirname(os.path.realpath(target_path))
 
     dependencies.add(base_dir)
     dependencies |= FindBootstrapDependencies(base_dir)
-    dependencies |= FindPythonDependencies(target_path)
+    dependencies |= FindDynamicAnalysisDependencies(base_dir)
     if options.include_page_set_data:
       dependencies |= FindPageSetDependencies(base_dir)