[bisect] - Parse DEPS file manually if execfile fails.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 58 matching lines @@
 # svn: Needed for git workflow to resolve hashes to svn revisions.
 # from: Parent depot that must be bisected before this is bisected.
 # deps_var: Key name in vars varible in DEPS file that has revision information.
 DEPOT_DEPS_NAME = {
   'chromium' : {
     "src" : "src",
     "recurse" : True,
     "depends" : None,
     "from" : ['cros', 'android-chrome'],
     'viewvc': 'http://src.chromium.org/viewvc/chrome?view=revision&revision=',
-    'deps_var': None
+    'deps_var': 'chromium_rev'
   },
   'webkit' : {
     "src" : "src/third_party/WebKit",
     "recurse" : True,
     "depends" : None,
     "from" : ['chromium'],
     'viewvc': 'http://src.chromium.org/viewvc/blink?view=revision&revision=',
     'deps_var': 'webkit_revision'
   },
   'angle' : {
@@ skipping 643 matching lines @@
         if 'ninja' in os.getenv('GYP_GENERATORS'):
           opts.build_preference = 'ninja'
         else:
           opts.build_preference = 'make'
 
       SetBuildSystemDefault(opts.build_preference)
 
     if not bisect_utils.SetupPlatformBuildEnvironment(opts):
       raise RuntimeError('Failed to set platform environment.')
 
     bisect_utils.RunGClient(['runhooks'])

[shatch, 2014/04/25 22:01:40]

Unnecessary call, and ends up running when we run the bisect unit tests as well.

 
   @staticmethod
   def FromOpts(opts):
     builder = None
     if opts.target_platform == 'cros':
       builder = CrosBuilder(opts)
     elif opts.target_platform == 'android':
       builder = AndroidBuilder(opts)
     elif opts.target_platform == 'android-chrome':
       builder = AndroidChromeBuilder(opts)
@@ skipping 606 matching lines @@
 
     bleeding_edge_revision = None
 
     for c in commits:
       bleeding_edge_revision = self._GetV8BleedingEdgeFromV8TrunkIfMappable(c)
       if bleeding_edge_revision:
         break
 
     return bleeding_edge_revision
 
-  def Get3rdPartyRevisionsFromCurrentRevision(self, depot, revision):
-    """Parses the DEPS file to determine WebKit/v8/etc... versions.
+  def _ParseRevisionsFromDEPSFileManually(self, depot):
+    """Manually parses the vars section of the DEPS file to determine
+    chromium/blink/etc... revisions.
+
+    Args:
+      depot: Depot being bisected.
 
     Returns:
       A dict in the format {depot:revision} if successful, otherwise None.
     """
-    cwd = os.getcwd()
-    self.ChangeToDepotWorkingDirectory(depot)
+    # We'll parse the "vars" section of the DEPS file.
+    rxp = re.compile("vars = {(?P<vars_body>[^}]+)", re.MULTILINE)
+    re_results = rxp.search(ReadStringFromFile(bisect_utils.FILE_DEPS_GIT))
+    locals = {}
 
-    results = {}
+    if not re_results:
+      return None
 
-    if depot == 'chromium' or depot == 'android-chrome':
+    # We should be left with a series of entries in the vars component of
+    # the DEPS file with the following format:
+    # 'depot_name': 'revision',
+    vars_body = re_results.group("vars_body")

[qyearsley, 2014/04/25 20:53:30]

Line 1381 and 1391 -- single quotes preferred over double quotes for
consistency.

[shatch, 2014/04/25 22:01:40]

Done.

+    rxp = re.compile("'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'",
+        re.MULTILINE)

[qyearsley, 2014/04/25 20:53:30]

Style nit: line break style.
http://google-styleguide.googlecode.com/svn/trunk/pyguide.html?showone=Indent...

[shatch, 2014/04/25 22:01:40]

Done.

+    re_results = rxp.findall(vars_body)
+
+    for depot_name, depot_revision in re_results:
+      depot_revision = depot_revision.strip('@')
+      for current_name, current_data in DEPOT_DEPS_NAME.iteritems():
+        if (current_data.has_key('deps_var') and
+            current_data['deps_var'] == depot_name):
+          src_name = current_name
+          locals[src_name] = depot_revision
+          break
+    return locals
+
+  def _ParseRevisionsFromDEPSFile(self, depot):
+    """Parses the local DEPS file to determine blink/skia/v8 revisions which may
+    be needed if the bisect recurses into those depots later.
+
+    Args:
+      depot: Depot being bisected.
+
+    Returns:
+      A dict in the format {depot:revision} if successful, otherwise None.
+    """
+    try:
       locals = {'Var': lambda _: locals["vars"][_],
                 'From': lambda *args: None}
       execfile(bisect_utils.FILE_DEPS_GIT, {}, locals)
-
-      os.chdir(cwd)
+      locals = locals['deps']
+      results = {}
 
       rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)")
 
       for d in DEPOT_NAMES:
         if DEPOT_DEPS_NAME[d].has_key('platform'):
           if DEPOT_DEPS_NAME[d]['platform'] != os.name:
             continue
 
         if (DEPOT_DEPS_NAME[d]['recurse'] and
             depot in DEPOT_DEPS_NAME[d]['from']):
-          if (locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src']) or
-              locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src_old'])):
-            if locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src']):
-              re_results = rxp.search(locals['deps'][DEPOT_DEPS_NAME[d]['src']])
+          if (locals.has_key(DEPOT_DEPS_NAME[d]['src']) or
+              locals.has_key(DEPOT_DEPS_NAME[d]['src_old'])):
+            if locals.has_key(DEPOT_DEPS_NAME[d]['src']):
+              re_results = rxp.search(locals[DEPOT_DEPS_NAME[d]['src']])
               self.depot_cwd[d] = \
                   os.path.join(self.src_cwd, DEPOT_DEPS_NAME[d]['src'][4:])
-            elif locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src_old']):
+            elif (DEPOT_DEPS_NAME[d].has_key('src_old') and
+                locals.has_key(DEPOT_DEPS_NAME[d]['src_old'])):
               re_results = \
-                  rxp.search(locals['deps'][DEPOT_DEPS_NAME[d]['src_old']])
+                  rxp.search(locals[DEPOT_DEPS_NAME[d]['src_old']])
               self.depot_cwd[d] = \
                   os.path.join(self.src_cwd, DEPOT_DEPS_NAME[d]['src_old'][4:])
 
             if re_results:
               results[d] = re_results.group('revision')
             else:
-              print 'Couldn\'t parse revision for %s.' % d
-              print
-              return None
+              warning_text = ('Couldn\'t parse revision for %s while bisecting '
+                  '%s' % (d, depot))
+              if not warningText in self.warnings:
+                self.warnings.append(warningText)
           else:
             print 'Couldn\'t find %s while parsing .DEPS.git.' % d
             print
             return None
+      return results
+    except ImportError:

[qyearsley, 2014/04/25 20:53:30]

This ImportError is thrown on line 1419 (in execfile) when the DEPS file
contains an import statement, right?

Would it be more complicated if we just always "manually" parsed the DEPS file,
even if the DEPS file doesn't contain import statements?

[shatch, 2014/04/25 22:01:40]

Yep.

I'm thinking it would be a lot more complicated to manually parse it. If you
want to go that route, might be better as a follow up patch later on so that we
can get android-chrome folks on their feet in the meantime.

On 2014/04/25 20:53:30, qyearsley wrote:

+      return self._ParseRevisionsFromDEPSFileManually(depot)
+
+  def Get3rdPartyRevisionsFromCurrentRevision(self, depot, revision):
+    """Parses the DEPS file to determine WebKit/v8/etc... versions.
+
+    Returns:
+      A dict in the format {depot:revision} if successful, otherwise None.
+    """
+    cwd = os.getcwd()
+    self.ChangeToDepotWorkingDirectory(depot)
+
+    results = {}
+
+    if depot == 'chromium' or depot == 'android-chrome':
+      results = self._ParseRevisionsFromDEPSFile(depot)
+      os.chdir(cwd)
     elif depot == 'cros':
       cmd = [CROS_SDK_PATH, '--', 'portageq-%s' % self.opts.cros_board,
              'best_visible', '/build/%s' % self.opts.cros_board, 'ebuild',
              CROS_CHROMEOS_PATTERN]
       (output, return_code) = RunProcessAndRetrieveOutput(cmd)
 
       assert not return_code, 'An error occurred while running' \
                               ' "%s"' % ' '.join(cmd)
 
       if len(output) > CROS_CHROMEOS_PATTERN:
@@ skipping 2290 matching lines @@
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())