Adding ability to bisect recipe to bisect into dependency repos.

scripts/slave/recipe_modules/auto_bisect/revision_state.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """An interface for holding state and result of revisions in a bisect job.
 
 When implementing support for tests other than perf, one should extend this
 class so that the bisect module and recipe can use it.
 
 See perf_revision_state for an example.
 """
+import hashlib
+import os
+import re
+
+from . import depot_config
 
 class RevisionState(object):

[qyearsley, 2015/02/22 20:37:55]

The other files have two blank lines between import section and first class or
function.

[RobertoCN, 2015/02/24 20:01:14]

Done.

   """Abstracts the state of a single revision on a bisect job."""
 
-  def __init__(self, revision_string, bisector):
+  def __init__(self, revision_string, bisector, depot='chromium',
+               base_revision=None, deps_revision=None):
     """Create a new instance to track the state of a revision.
 
+    There are two use cases for this constructor:
+      - Creating a revision state for a chromium revision, OR
+      - Creating a revision state for a chromium revision plus a deps change.

[qyearsley, 2015/02/22 20:37:55]

I'm not sure, but for other people without context I think the phrase "deps
change" might be unclear. Not sure if it would be more clear to say "...a
chromium revision plus an explicitly specified revision of a dependency
repository (a DEPS change)".

[RobertoCN, 2015/02/24 20:01:14]

Done.

+    In the first case a revision_string and a bisector are needed.
+    In the second case, revision_string must be None, and all of depot,
+    base_revision and deps_revision must be provided.
+
     Args:
-      revision_string: should be in the following format:
-        [(chromium|src)@](<commit_pos>|<commit_hash)(,<repo>@<commit>)*
-        E.g.:
-          'a0b1c2ffff89009909090' (full or abbrev. commit hash)
-          '123456'
-          'src@123456'
-          'chromium@123456'
-          'src@abc01234ffff,v8@00af5ceb888ff'
-      bisector: an instance of Bisector, the object performing the bisection.
+      revision_string (str): A git hash or a commit position in the chromium
+        repository. If None, all kwargs must be given.
+      bisector (Bisector): The object performing the bisection.
+      depot (dict): One of the entries in depot_config.DEPOT_DEPS_NAME that
+        specifies which dependency to do the DEPS change on. It is expected to
+        contain the 'chromium' string instead of None when not bisecting any
+        dependencies.
+      base_revision (RevisionState): The revision state to patch with the deps
+        change.
+      depot_revision: The commit hash of the dependency repo to put in place of
+        the one set for the base_revision.
     """
+    #  TODO(robertocn): Evaluate if the logic of this constructor should be
+    #    split into separate methods.
     super(RevisionState, self).__init__()
     self.bisector = bisector
     self._good = None
+    self.deps = None
     self.build_status_url = None
     self.in_progress = False
     self.aborted = False
     self.next_revision = None
     self.previous_revision = None
     self.revision_string = revision_string
-    self.commit_hash, self.commit_pos = self._commit_from_rev_string()
     self.build_job_name = None
     self.test_job_name = None
     self.built = False
+    if not self.revision_string:
+      assert base_revision
+      assert base_revision.deps_file_contents
+      assert depot != 'chromium'
+      assert deps_revision
+      self.needs_patch = True
+      self.depot = depot['deps_var']
+      self.revision_string = base_revision.revision_string + ',' + self.depot
+      self.revision_string += '@' + deps_revision
+      self.deps_patch, self.deps_file_contents = self.bisector.make_deps_patch(
+          base_revision.commit_hash, base_revision.deps_file_contents,
+          self.depot, deps_revision)
+      self.deps_sha = hashlib.sha1(self.deps_patch).hexdigest()
+      self.deps_patch += self.bisector.make_deps_sha_file(self.deps_sha)
+      self.deps = dict(base_revision.deps)
+      self.deps[self.depot] = deps_revision
+    else:
+      self.needs_patch = False
+      self.depot = depot
+    self.commit_hash, self.commit_pos = self._commit_from_rev_string()
     self.build_url = self.bisector.get_platform_gs_prefix() + self._gs_suffix()
 
   @property
   def good(self):
     return self._good == True
 
   @property
   def bad(self):
     return self._good == False
 
@@ skipping 40 matching lines @@
     """Uses `git show` to see if a given commit contains a DEPS change."""
     api = self.bisector.api
     step_result = api.m.git('show', '--name-only', '--pretty=format:',
                self.commit_hash, stdout=api.m.raw_io.output())
     if self.bisector.dummy_builds:
       return False
     if 'DEPS' in step_result.stdout.splitlines():
       return True
     return False
 
+
+  def _gen_deps_local_scope(self):
+    """Defines the Var and From functions in a dict for calling exec.
+
+    This is needed for executing the DEPS file.
+    """
+    deps_data =  {
+        'Var': lambda _: deps_data["vars"][_],

[qyearsley, 2015/02/22 20:37:55]

Single quotes.

[RobertoCN, 2015/02/24 20:01:14]

Done.

+        'From': lambda *args: None,
+    }
+    return deps_data
+
+  def read_deps(self):
+    """Sets the dependencies for this revision from the contents of DEPS."""
+    if self.deps:
+      return
+    deps_contents = self.m.git.cat_file_at_revision(depot_config.FILE_DEPS)
+    try:
+      deps_data = self._gen_deps_local_scope()
+      exec(deps_contents, {}, deps_data)
+      deps_data = deps_data['deps']
+    except ImportError:
+      # TODO(robertocn): Implement manual parsing of DEPS when exec fails.
+      raise NotImplementedError('Path not implemented to manually parse DEPS')
+
+    rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)")

[qyearsley, 2015/02/22 20:37:55]

1. Variable name could probably be improved.
2. Not sure if you want to use [a-fA-F0-9]{40} instead of [a-fA-F0-9]+ (are all
the commit hashes in deps full sha1 hashes?)
3. Single quotes.

[RobertoCN, 2015/02/24 20:01:14]

Done.

+    results = {}
+    for depot_name, depot_data in depot_config.DEPOT_DEPS_NAME.iteritems():
+      if (depot_data.get('platform') and
+          depot_data.get('platform') != os.name):
+        # TODO(robertocn) we shouldn't be checking the os of the bot running the
+        # bisector, but the os the tester would be running on.
+        continue
+
+      if depot_data.get('recurse') and self.depot in depot_data.get('from'):
+        depot_data_src = depot_data.get('src') or depot_data.get('src_old')
+        src_dir = deps_data.get(depot_data_src)
+        if src_dir:
+          re_results = rxp.search(src_dir)
+          if re_results:
+            results[depot_name] = re_results.group('revision')
+          else:
+            warning_text = ('Could not parse revision for %s while bisecting '
+                            '%s' % (depot_name, self.depot))
+            if not warning_text in self.bisector.warnings:
+              self.bisector.warnings.append(warning_text)
+        else:
+          results[depot_name] = None
+    self.deps = results
+    return
+
   def update_status(self):
     """Checks on the pending jobs and updates status accordingly.
 
     This method will check for the build to complete and then trigger the test,
     or will wait for the test as appropriate.
 
     To wait for the test we try to get the buildbot job url from GS, and if
     available, we query the status of such job.
     """
     if not self.in_progress:
@@ skipping 17 matching lines @@
     result = api.gsutil_file_exists(self.build_url)
     if self.bisector.dummy_builds:
       return self.in_progress
     return result
 
   def _gs_suffix(self):
     """Provides the expected right half of the build filename.
 
     This takes into account whether the build has a deps patch.
     """
-    # TODO: Implement the logic for deps patch changes.
-    return self.commit_hash + '.zip'
+    name_parts = [self.commit_hash]
+    if self.needs_patch:
+      name_parts.append(self.deps_sha)
+    return '%s.zip' % '_'.join(name_parts)
 
   def _commit_from_rev_string(self):
     """Gets the chromium repo commit hash and position for this revision.
 
     If there are specified dependency revisions in the string, we don't compute
     either the position or hash"""
     pieces = self.revision_string.split(',')
     if len(pieces) > 1:
       return None, None
     if (pieces[0].startswith('chromium@') or
@@ skipping 22 matching lines @@
     int(s, 16)
     return True
 
   def _get_pos_from_hash(self, sha):
     api = self.bisector.api
     return api.m.commit_position.chromium_commit_position_from_hash(sha)
 
   def _get_hash_from_pos(self, pos):
     api = self.bisector.api
     return api.m.commit_position.chromium_hash_from_commit_position(pos)