Trigger build request using git try instead of try_job_http on bisect bots

tools/auto_bisect/bisect_perf_regression.py

Index: tools/auto_bisect/bisect_perf_regression.py
diff --git a/tools/auto_bisect/bisect_perf_regression.py b/tools/auto_bisect/bisect_perf_regression.py
index 9a555533eed55c7f14b255f5dd270ea9780996ff..d1cc0e64ba4618d40a163561057a95ec8a6e14ed 100755
--- a/tools/auto_bisect/bisect_perf_regression.py
+++ b/tools/auto_bisect/bisect_perf_regression.py
@@ -176,10 +176,10 @@ HIGH_CONFIDENCE = 95
 # When a build requested is posted with a patch, bisect builders on try server,
 # once build is produced, it reads SHA value from this file and appends it
 # to build archive filename.
-DEPS_SHA_PATCH = """diff --git src/DEPS.sha src/DEPS.sha
+DEPS_SHA_PATCH = """diff --git DEPS.sha DEPS.sha
 new file mode 100644
 --- /dev/null
-+++ src/DEPS.sha
++++ DEPS.sha
 @@ -0,0 +1 @@
 +%(deps_sha)s
 """
@@ -263,6 +263,20 @@ bisecting.
 .                 /     /==)
 .                ._____."""
 
+# Git branch name used to run bisect try jobs.
+BISECT_TRYJOB_BRANCH = 'bisect-tryjob'
+# Git master branch name.
+BISECT_MASTER_BRANCH = 'master'
+# File to store 'git diff' content.
+BISECT_PATCH_FILE = 'deps_patch.txt'
+# SVN repo where the bisect try jobs are submitted.
+SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try-perf'
+
+class RunGitError(Exception):
+
+  def __str__(self):
+    return '%s\nError executing git command.' % self.args[0]
+
 
 def _AddAdditionalDepotInfo(depot_info):
   """Adds additional depot info to the global depot variables."""
@@ -859,6 +873,110 @@ class DepotDirectoryRegistry(object):
     """
     os.chdir(self.GetDepotDir(depot_name))
 
+def _PrepareBisectBranch(parent_branch, new_branch):
+  """Creates a new branch to submit bisect try job.
+
+  Args:
+    parent_branch: Parent branch to be used to create new branch.
+    new_branch: New branch name.
+
+  Returns:
+    True if new branch is created succesfully, otherwise raises RunGitError
+    exception.

[qyearsley, 2014/09/27 01:45:33]

No need to return anything if the lack of an exception indicates success.

[prasadv, 2014/09/29 17:36:13]

Done.

+  """
+  current_branch, returncode = bisect_utils.RunGit(
+      ['rev-parse', '--abbrev-ref', 'HEAD'])
+  if returncode:
+    raise RunGitError('Must be in a git repository to send changes to trybots.')
+
+  current_branch = current_branch.strip()
+  # Make sure current branch is master.
+  if current_branch != parent_branch:
+    output, returncode = bisect_utils.RunGit(['checkout', '-f', parent_branch])
+    if returncode:
+      raise RunGitError('Failed to checkout branch: %s.' % output)
+
+  # Delete new branch if exists.
+  output, returncode = bisect_utils.RunGit(['branch', '--list' ])
+  if new_branch in output:
+    output, returncode = bisect_utils.RunGit(['branch', '-D', new_branch])
+    if returncode:
+      raise RunGitError('Deleting branch failed, %s', output)
+
+  # Check if the tree is dirty: make sure the index is up to date and then
+  # run diff-index

[qyearsley, 2014/09/27 01:45:33]

Nit: should add a period at the end of the sentence.
Also, below we're running "git diff-index HEAD", not just "diff-index". What
does "git diff-index HEAD" do? I guess it outputs the changes that are staged
for commit but haven't yet been committed?

Maybe these few lines could be extracted to a IsGitTreeDirty function?

[prasadv, 2014/09/29 17:36:13]

Correct it lists if we have any uncommitted changes. 
Not sure, we want to extract a new method  IsGitTreeDirty(), since it is used
only here.
Let me know if you still want to it.

On 2014/09/27 01:45:33, qyearsley wrote:

+  bisect_utils.RunGit(['update-index', '--refresh', '-q'])
+  output, returncode = bisect_utils.RunGit(['diff-index', 'HEAD'])
+  if output:
+    raise RunGitError('Cannot send a try job with a dirty tree.')
+
+  # Create/check out the telemetry-tryjob branch, and edit the configs
+  # for the tryjob there.
+  output, returncode = bisect_utils.RunGit(['checkout', '-b', new_branch])
+  if returncode:
+    raise RunGitError('Failed to checkout branch: %s.' % output)
+
+  output, returncode = bisect_utils.RunGit(
+      ['branch', '--set-upstream-to', parent_branch])
+  if returncode:
+    raise RunGitError('Error in git branch --set-upstream-to')
+
+  return True
+
+
+def _BuilderTryjob(git_revision, bot_name, bisect_job_name, patch=None):
+  """Attempts to run a tryjob from the current directory.
+
+  Args:
+    git_revision: A Git hash revision.
+    bot_name: Name of the bisect bot to be used for try job.
+    bisect_job_name: Bisect try job name.
+    patch: A DEPS patch (used while bisecting 3rd party repositories).
+
+  Returns:
+    True if a tryjob was sent otherwise raises RunGitError exception.
+  """
+  if not patch:
+    raise RunGitError('No associated patch to try')
+
+  try:
+    # Temporary branch for running tryjob.
+    _PrepareBisectBranch(BISECT_MASTER_BRANCH, BISECT_TRYJOB_BRANCH)
+    patch_content = '/dev/null'
+    # Create a temporary patch file, if it fails raise an exception.
+    if patch:
+      WriteStringToFile(patch, BISECT_PATCH_FILE)
+      patch_content = BISECT_PATCH_FILE
+
+    try_cmd = ['try',
+               '-b', bot_name,
+               '-r', git_revision,
+               '-n', bisect_job_name,
+               '--svn_repo=%s' % SVN_REPO_URL,
+               '--diff=%s' % patch_content
+              ]
+    # Execute try job to build revision.
+    output, returncode = bisect_utils.RunGit(try_cmd)
+
+    if returncode:
+      raise RunGitError('Could not execute tryjob: %s.\n Error: %s' % (
+                         'git %s' % ' '.join(try_cmd), output))
+    print ('Try job successfully submitted.\n TryJob Details: %s\n%s' % (
+           'git %s' % ' '.join(try_cmd), output))
+
+  finally:
+    # Delete patch file if exists
+    try:
+      os.remove(BISECT_PATCH_FILE)
+    except OSError as e:
+      if e.errno != errno.ENOENT:
+        raise
+    # Checkout master branch and delete bisect-tryjob branch.
+    bisect_utils.RunGit(['checkout', '-f', BISECT_MASTER_BRANCH])
+    bisect_utils.RunGit(['branch', '-D', BISECT_TRYJOB_BRANCH])
+
+  return True
+
 
 class BisectPerformanceMetrics(object):
   """This class contains functionality to perform a bisection of a range of
@@ -1195,7 +1313,7 @@ class BisectPerformanceMetrics(object):
         return FetchFromCloudStorage(gs_bucket, source_file, out_dir)
     return downloaded_archive
 
-  def DownloadCurrentBuild(self, revision, build_type='Release', patch=None):
+  def DownloadCurrentBuild(self, revision, depot, build_type='Release'):
     """Downloads the build archive for the given revision.
 
     Args:
@@ -1206,7 +1324,12 @@ class BisectPerformanceMetrics(object):
     Returns:
       True if download succeeds, otherwise False.
     """
+    patch = None
     patch_sha = None
+    if depot != 'chromium':
+      # Create a DEPS patch with new revision for dependency repository.
+      revision, patch = self.CreateDEPSPatch(depot, revision)
+
     if patch:
       # Get the SHA of the DEPS changes patch.
       patch_sha = GetSHA1HexDigest(patch)
@@ -1303,37 +1426,31 @@ class BisectPerformanceMetrics(object):
     if not fetch_build:
       return False
 
-    bot_name, build_timeout = GetBuilderNameAndBuildTime(
-       self.opts.target_platform, self.opts.target_arch)
-    builder_host = self.opts.builder_host
-    builder_port = self.opts.builder_port
     # Create a unique ID for each build request posted to try server builders.
     # This ID is added to "Reason" property of the build.
     build_request_id = GetSHA1HexDigest(
         '%s-%s-%s' % (git_revision, patch, time.time()))
 
-    # Creates a try job description.
-    # Always use Git hash to post build request since Commit positions are
-    # not supported by builders to build.
-    job_args = {
-        'revision': 'src@%s' % git_revision,
-        'bot': bot_name,
-        'name': build_request_id,
-    }
-    # Update patch information if supplied.
-    if patch:
-      job_args['patch'] = patch
-    # Posts job to build the revision on the server.
-    if request_build.PostTryJob(builder_host, builder_port, job_args):
+    # Reverts any changes to DEPS file.
+    self.source_control.CheckoutFileAtRevision(
+      bisect_utils.FILE_DEPS, git_revision, cwd=self.src_cwd)
+
+    bot_name, build_timeout = GetBuilderNameAndBuildTime(
+       self.opts.target_platform, self.opts.target_arch)
+    target_file = None
+    try:
+      # Execute try job request to build revision with patch.
+      _BuilderTryjob(git_revision, bot_name, build_request_id, patch)
       target_file, error_msg = _WaitUntilBuildIsReady(
-          fetch_build, bot_name, builder_host, builder_port, build_request_id,
-          build_timeout)
+          fetch_build, bot_name, self.opts.builder_host,
+          self.opts.builder_port, build_request_id, build_timeout)
       if not target_file:
         print '%s [revision: %s]' % (error_msg, git_revision)
-        return None
-      return target_file
-    print 'Failed to post build request for revision: [%s]' % git_revision
-    return None
+    except RunGitError, e:

[qyearsley, 2014/09/27 01:45:33]

The style guide suggests using "as", e.g. "except RunGitError as e".
https://google-styleguide.googlecode.com/svn/trunk/pyguide.html?showone=Excep...

+      print ('Failed to post builder try job for revision: [%s].\n'
+             'Error: %s' % (git_revision, e))
+
+    return target_file
 
   def IsDownloadable(self, depot):
     """Checks if build can be downloaded based on target platform and depot."""
@@ -1430,6 +1547,7 @@ class BisectPerformanceMetrics(object):
       print 'Something went wrong while updating DEPS file. [%s]' % e
     return False
 
+
   def CreateDEPSPatch(self, depot, revision):
     """Modifies DEPS and returns diff as text.
 
@@ -1457,8 +1575,8 @@ class BisectPerformanceMetrics(object):
         if self.UpdateDeps(revision, depot, deps_file_path):
           diff_command = [
               'diff',
-              '--src-prefix=src/',
-              '--dst-prefix=src/',
+              '--src-prefix=',
+              '--dst-prefix=',
               '--no-ext-diff',
                bisect_utils.FILE_DEPS,
           ]
@@ -1487,16 +1605,7 @@ class BisectPerformanceMetrics(object):
     # Fetch build archive for the given revision from the cloud storage when
     # the storage bucket is passed.
     if self.IsDownloadable(depot) and revision:
-      deps_patch = None
-      if depot != 'chromium':
-        # Create a DEPS patch with new revision for dependency repository.
-        revision, deps_patch = self.CreateDEPSPatch(depot, revision)
-      if self.DownloadCurrentBuild(revision, patch=deps_patch):
-        if deps_patch:
-          # Reverts the changes to DEPS file.
-          self.source_control.CheckoutFileAtRevision(
-              bisect_utils.FILE_DEPS, revision, cwd=self.src_cwd)
-        build_success = True
+      build_success = self.DownloadCurrentBuild(revision, depot)
     else:
       # These codes are executed when bisect bots builds binaries locally.
       build_success = self.builder.Build(depot, self.opts)
@@ -1792,17 +1901,7 @@ class BisectPerformanceMetrics(object):
     Returns:
       True if successful.
     """
-    if depot == 'chromium' or depot == 'android-chrome':
-      # Removes third_party/libjingle. At some point, libjingle was causing
-      # issues syncing when using the git workflow (crbug.com/266324).
-      os.chdir(self.src_cwd)
-      if not bisect_utils.RemoveThirdPartyDirectory('libjingle'):
-        return False
-      # Removes third_party/skia. At some point, skia was causing
-      # issues syncing when using the git workflow (crbug.com/377951).
-      if not bisect_utils.RemoveThirdPartyDirectory('skia'):
-        return False
-    elif depot == 'cros':
+    if depot == 'cros':
       return self.PerformCrosChrootCleanup()
     return True