[bisect] - Parse DEPS file manually if execfile fails.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 58 matching lines @@
 # svn: Needed for git workflow to resolve hashes to svn revisions.
 # from: Parent depot that must be bisected before this is bisected.
 # deps_var: Key name in vars varible in DEPS file that has revision information.
 DEPOT_DEPS_NAME = {
   'chromium' : {
     "src" : "src",
     "recurse" : True,
     "depends" : None,
     "from" : ['cros', 'android-chrome'],
     'viewvc': 'http://src.chromium.org/viewvc/chrome?view=revision&revision=',
-    'deps_var': None
+    'deps_var': 'chromium_rev'
   },
   'webkit' : {
     "src" : "src/third_party/WebKit",
     "recurse" : True,
     "depends" : None,
     "from" : ['chromium'],
     'viewvc': 'http://src.chromium.org/viewvc/blink?view=revision&revision=',
     'deps_var': 'webkit_revision'
   },
   'angle' : {
@@ skipping 1270 matching lines @@
 
     bleeding_edge_revision = None
 
     for c in commits:
       bleeding_edge_revision = self._GetV8BleedingEdgeFromV8TrunkIfMappable(c)
       if bleeding_edge_revision:
         break
 
     return bleeding_edge_revision
 
-  def Get3rdPartyRevisionsFromCurrentRevision(self, depot, revision):
-    """Parses the DEPS file to determine WebKit/v8/etc... versions.
+  def _ParseRevisionsFromDEPSFileManually(self, depot):
+    """Manually parses the vars section of the DEPS file to determine
+    chromium/blink/etc... revisions.
+
+    Args:
+      depot: Depot being bisected.
 
     Returns:
       A dict in the format {depot:revision} if successful, otherwise None.
     """
-    cwd = os.getcwd()
-    self.ChangeToDepotWorkingDirectory(depot)
+    # We'll parse the "vars" section of the DEPS file.
+    rxp = re.compile("vars = {(?P<vars_body>[^}]+)", re.MULTILINE)
+    re_results = rxp.search(ReadStringFromFile(bisect_utils.FILE_DEPS_GIT))
+    locals = {}
 
-    results = {}
+    if not re_results:
+      return None
 
-    if depot == 'chromium' or depot == 'android-chrome':
+    for current_line in re_results.group("vars_body").splitlines():
+      try:
+        # We should be left with a series of entries in the vars component of
+        # the DEPS file with the following format:
+        # 'depot_name': 'revision',

[qyearsley, 2014/04/25 18:11:18]

I think this might not work for some .DEPS.git files, since in some cases the
depot name and revision are split up onto different lines, like:

vars = {
    'ffmpeg_hash':
         '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
    'webkit_url':
         'https://chromium.googlesource.com/chromium/blink.git',
    'git_url':
         'https://chromium.googlesource.com',
    'webkit_rev':
         '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
    'angle_revision':
         '74697cf2064c0a2c0d7e1b1b28db439286766a05'
}

https://code.google.com/p/chromium/codesearch#chromium/src/.DEPS.git

Will this still work in that case? Maybe in that case we'd need to do something
else, like use a regex to match each depot_name / depot_revision pair even if
there's a newline between them?

[shatch, 2014/04/25 20:24:16]

True, changed it to use a regex to match up name/revision pairs.

On 2014/04/25 18:11:18, qyearsley wrote:

+        depot_name = current_line.split("'")[1].split("'")[0]
+        depot_revision = current_line.split(": '")[1].split("'")[0]
+        for current_name, current_data in DEPOT_DEPS_NAME.iteritems():
+          if (current_data.has_key('deps_var') and
+              current_data['deps_var'] == depot_name):
+            src_name = current_name
+            locals[src_name] = depot_revision
+            break
+      except IndexError:
+        pass
+    return locals
+
+  def _ParseRevisionsFromDEPSFile(self, depot):
+    """Parses the local DEPS file to determine blink/skia/v8 revisions which may
+    be needed if the bisect recurses into those depots later.
+
+    Args:
+      depot: Depot being bisected.
+
+    Returns:
+      A dict in the format {depot:revision} if successful, otherwise None.
+    """
+    try:
       locals = {'Var': lambda _: locals["vars"][_],
                 'From': lambda *args: None}
       execfile(bisect_utils.FILE_DEPS_GIT, {}, locals)
-
-      os.chdir(cwd)
+      locals = locals['deps']
+      results = {}
 
       rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)")
 
       for d in DEPOT_NAMES:
         if DEPOT_DEPS_NAME[d].has_key('platform'):
           if DEPOT_DEPS_NAME[d]['platform'] != os.name:
             continue
 
         if (DEPOT_DEPS_NAME[d]['recurse'] and
             depot in DEPOT_DEPS_NAME[d]['from']):
-          if (locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src']) or
-              locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src_old'])):
-            if locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src']):
-              re_results = rxp.search(locals['deps'][DEPOT_DEPS_NAME[d]['src']])
+          if (locals.has_key(DEPOT_DEPS_NAME[d]['src']) or
+              locals.has_key(DEPOT_DEPS_NAME[d]['src_old'])):
+            if locals.has_key(DEPOT_DEPS_NAME[d]['src']):
+              re_results = rxp.search(locals[DEPOT_DEPS_NAME[d]['src']])
               self.depot_cwd[d] = \
                   os.path.join(self.src_cwd, DEPOT_DEPS_NAME[d]['src'][4:])
-            elif locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src_old']):
+            elif (DEPOT_DEPS_NAME[d].has_key('src_old') and
+                locals.has_key(DEPOT_DEPS_NAME[d]['src_old'])):
               re_results = \
-                  rxp.search(locals['deps'][DEPOT_DEPS_NAME[d]['src_old']])
+                  rxp.search(locals[DEPOT_DEPS_NAME[d]['src_old']])
               self.depot_cwd[d] = \
                   os.path.join(self.src_cwd, DEPOT_DEPS_NAME[d]['src_old'][4:])
 
             if re_results:
               results[d] = re_results.group('revision')
             else:
-              print 'Couldn\'t parse revision for %s.' % d
-              print
-              return None
+              warning_text = ('Couldn\'t parse revision for %s while bisecting '
+                  '%s' % (d, depot))
+              if not warningText in self.warnings:
+                self.warnings.append(warningText)
           else:
             print 'Couldn\'t find %s while parsing .DEPS.git.' % d
             print
             return None
+      return results
+    except ImportError:
+      return self._ParseRevisionsFromDEPSFileManually(depot)
+
+  def Get3rdPartyRevisionsFromCurrentRevision(self, depot, revision):
+    """Parses the DEPS file to determine WebKit/v8/etc... versions.
+
+    Returns:
+      A dict in the format {depot:revision} if successful, otherwise None.
+    """
+    cwd = os.getcwd()
+    self.ChangeToDepotWorkingDirectory(depot)
+
+    results = {}
+
+    if depot == 'chromium' or depot == 'android-chrome':
+      results = self._ParseRevisionsFromDEPSFile(depot)
+      os.chdir(cwd)
     elif depot == 'cros':
       cmd = [CROS_SDK_PATH, '--', 'portageq-%s' % self.opts.cros_board,
              'best_visible', '/build/%s' % self.opts.cros_board, 'ebuild',
              CROS_CHROMEOS_PATTERN]
       (output, return_code) = RunProcessAndRetrieveOutput(cmd)
 
       assert not return_code, 'An error occurred while running' \
                               ' "%s"' % ' '.join(cmd)
 
       if len(output) > CROS_CHROMEOS_PATTERN:
@@ skipping 2290 matching lines @@
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())