bisect-builds.py: Minor bug fixes for blink bisect and better error handling.

tools/bisect-builds.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Snapshot Build Bisect Tool
 
 This script bisects a snapshot archive using binary search. It starts at
 a bad revision (it will try to guess HEAD) and asks for a last known-good
 revision. It will then binary search across this revision range by downloading,
@@ skipping 298 matching lines @@
     # until an entire directory listing is done.
     while next_marker:
       next_url = self.GetListingURL(next_marker)
       (new_revisions, next_marker, new_dict) = _FetchAndParse(next_url)
       revisions.extend(new_revisions)
       self.githash_svn_dict.update(new_dict)
     return revisions
 
   def _GetSVNRevisionFromGitHashWithoutGitCheckout(self, git_sha1, depot):
     json_url = GITHASH_TO_SVN_URL[depot] % git_sha1
-    try:
-      response = urllib.urlopen(json_url)
-    except urllib.HTTPError as error:
-      msg = 'HTTP Error %d for %s' % (error.getcode(), git_sha1)
-      return None
-    data = json.loads(response.read()[4:])
+    response = urllib.urlopen(json_url)
+    if response.getcode() == 200:
+      try:
+        data = json.loads(response.read()[4:])
+      except ValueError:
+        print 'ValueError for JSON URL: %s' % json_url
+        raise ValueError
+    else:
+      raise ValueError
     if 'message' in data:
       message = data['message'].split('\n')
       message = [line for line in message if line.strip()]
       search_pattern = re.compile(SEARCH_PATTERN[depot])
       result = search_pattern.search(message[len(message)-1])
       if result:
         return result.group(1)
     print 'Failed to get svn revision number for %s' % git_sha1
     raise ValueError
 
@@ skipping 294 matching lines @@
 
   def WaitFor(self):
     """Prints a message and waits for the download to complete. The download
     must have been started previously."""
     assert self.thread, 'DownloadJob must be started before WaitFor is called.'
     print 'Downloading revision %s...' % str(self.rev)
     self.progress_event.set()  # Display progress of download.
     self.thread.join()
 
 
-def Bisect(base_url,
-           platform,
-           official_builds,
-           is_aura,
-           is_asan,
-           use_local_repo,
-           good_rev=0,
-           bad_rev=0,
+def Bisect(context,
            num_runs=1,
            command='%p %a',
            try_args=(),
            profile=None,
-           flash_path=None,
-           pdf_path=None,
            interactive=True,
            evaluate=AskIsGoodBuild):
   """Given known good and known bad revisions, run a binary search on all
   archived revisions to determine the last known good revision.
 
-  @param platform Which build to download/run ('mac', 'win', 'linux64', etc.).
-  @param official_builds Specify build type (Chromium or Official build).
-  @param good_rev Number/tag of the known good revision.
-  @param bad_rev Number/tag of the known bad revision.
+  @param context PathContext object initialized with user provided parameters.
   @param num_runs Number of times to run each build for asking good/bad.
   @param try_args A tuple of arguments to pass to the test application.
   @param profile The name of the user profile to run with.
   @param interactive If it is false, use command exit code for good or bad
                      judgment of the argument build.
   @param evaluate A function which returns 'g' if the argument build is good,
                   'b' if it's bad or 'u' if unknown.
 
   Threading is used to fetch Chromium revisions in the background, speeding up
   the user's experience. For example, suppose the bounds of the search are
   good_rev=0, bad_rev=100. The first revision to be checked is 50. Depending on
   whether revision 50 is good or bad, the next revision to check will be either
   25 or 75. So, while revision 50 is being checked, the script will download
   revisions 25 and 75 in the background. Once the good/bad verdict on rev 50 is
   known:
 
     - If rev 50 is good, the download of rev 25 is cancelled, and the next test
       is run on rev 75.
 
     - If rev 50 is bad, the download of rev 75 is cancelled, and the next test
       is run on rev 25.
   """
 
   if not profile:
     profile = 'profile'
 
-  context = PathContext(base_url, platform, good_rev, bad_rev,
-                        official_builds, is_aura, is_asan, use_local_repo,
-                        flash_path, pdf_path)
+  good_rev = context.good_revision
+  bad_rev = context.bad_revision
   cwd = os.getcwd()
 
   print 'Downloading list of known revisions...',
-  if not use_local_repo:
+  if not context.use_local_repo:
     print '(use --use-local-repo for speed if you have a local checkout)'
   else:
     print
   _GetDownloadPath = lambda rev: os.path.join(cwd,
       '%s-%s' % (str(rev), context.archive_name))
-  if official_builds:
+  if context.is_official:
     revlist = context.GetOfficialBuildsList()
   else:
     revlist = context.GetRevList()
 
   # Get a list of revisions to bisect across.
   if len(revlist) < 2:  # Don't have enough builds to bisect.
     msg = 'We don\'t have enough builds to bisect. revlist: %s' % revlist
     raise RuntimeError(msg)
 
   # Figure out our bookends and first pivot point; fetch the pivot revision.
@@ skipping 56 matching lines @@
     # other, as described in the comments above.
     try:
       if not interactive:
         if status:
           answer = 'b'
           print 'Bad revision: %s' % rev
         else:
           answer = 'g'
           print 'Good revision: %s' % rev
       else:
-        answer = evaluate(rev, official_builds, status, stdout, stderr)
+        answer = evaluate(rev, context.is_official, status, stdout, stderr)
       if ((answer == 'g' and good_rev < bad_rev)
           or (answer == 'b' and bad_rev < good_rev)):
         fetch.Stop()
         minrev = pivot
         if down_fetch:
           down_fetch.Stop()  # Kill the download of the older revision.
           fetch = None
         if up_fetch:
           up_fetch.WaitFor()
           pivot = up_pivot
@@ skipping 47 matching lines @@
       for f in [_GetDownloadPath(revlist[down_pivot]),
                 _GetDownloadPath(revlist[up_pivot])]:
         try:
           os.unlink(f)
         except OSError:
           pass
       sys.exit(0)
 
     rev = revlist[pivot]
 
-  return (revlist[minrev], revlist[maxrev])
+  return (revlist[minrev], revlist[maxrev], context)
 
 
 def GetBlinkDEPSRevisionForChromiumRevision(rev):
   """Returns the blink revision that was in REVISIONS file at
   chromium revision |rev|."""
   # . doesn't match newlines without re.DOTALL, so this is safe.
   blink_re = re.compile(r'webkit_revision\D*(\d+)')
   url = urllib.urlopen(DEPS_FILE % rev)
   m = blink_re.search(url.read())
   url.close()
   if m:
     return int(m.group(1))
   else:
     raise Exception('Could not get Blink revision for Chromium rev %d' % rev)
 
 
-def GetBlinkRevisionForChromiumRevision(self, rev):
+def GetBlinkRevisionForChromiumRevision(context, rev):
   """Returns the blink revision that was in REVISIONS file at
   chromium revision |rev|."""
   def _IsRevisionNumber(revision):
     if isinstance(revision, int):
       return True
     else:
       return revision.isdigit()
-  if str(rev) in self.githash_svn_dict:
-    rev = self.githash_svn_dict[str(rev)]
-  file_url = '%s/%s%s/REVISIONS' % (self.base_url,
-                                    self._listing_platform_dir, rev)
+  if str(rev) in context.githash_svn_dict:
+    rev = context.githash_svn_dict[str(rev)]
+  file_url = '%s/%s%s/REVISIONS' % (context.base_url,
+                                    context._listing_platform_dir, rev)
   url = urllib.urlopen(file_url)
-  data = json.loads(url.read())
+  if url.getcode() == 200:
+    try:
+      data = json.loads(url.read())
+    except ValueError:
+      print 'ValueError for JSON URL: %s' % file_url
+      raise ValueError
+  else:
+    raise ValueError
   url.close()
   if 'webkit_revision' in data:
     blink_rev = data['webkit_revision']
     if not _IsRevisionNumber(blink_rev):
-      blink_rev = self.GetSVNRevisionFromGitHash(blink_rev, 'blink')
+      blink_rev = int(context.GetSVNRevisionFromGitHash(blink_rev, 'blink'))
     return blink_rev
   else:
     raise Exception('Could not get blink revision for cr rev %d' % rev)
 
 
 def FixChromiumRevForBlink(revisions_final, revisions, self, rev):
   """Returns the chromium revision that has the correct blink revision
   for blink bisect, DEPS and REVISIONS file might not match since
   blink snapshots point to tip of tree blink.
   Note: The revisions_final variable might get modified to include
@@ skipping 146 matching lines @@
       return 1
 
   if opts.asan:
     base_url = ASAN_BASE_URL
   elif opts.blink:
     base_url = WEBKIT_BASE_URL
   else:
     base_url = CHROMIUM_BASE_URL
 
   # Create the context. Initialize 0 for the revisions as they are set below.
-  context = PathContext(base_url, opts.archive, 0, 0,
+  context = PathContext(base_url, opts.archive, opts.good, opts.bad,
                         opts.official_builds, opts.aura, opts.asan,
-                        opts.use_local_repo, None)
+                        opts.use_local_repo, opts.flash_path, opts.pdf_path)
   # Pick a starting point, try to get HEAD for this.
-  if opts.bad:
-    bad_rev = opts.bad
-  else:
-    bad_rev = '999.0.0.0'
-    if not opts.official_builds:
-      bad_rev = GetChromiumRevision(context, context.GetLastChangeURL())
+  if not opts.bad:
+    context.bad_revision = '999.0.0.0'
+    context.bad_revision = GetChromiumRevision(
+        context, context.GetLastChangeURL())
 
   # Find out when we were good.
-  if opts.good:
-    good_rev = opts.good
-  else:
-    good_rev = '0.0.0.0' if opts.official_builds else 0
+  if not opts.good:
+    context.good_revision = '0.0.0.0' if opts.official_builds else 0
 
   if opts.flash_path:
-    flash_path = opts.flash_path
-    msg = 'Could not find Flash binary at %s' % flash_path
-    assert os.path.exists(flash_path), msg
+    msg = 'Could not find Flash binary at %s' % opts.flash_path
+    assert os.path.exists(opts.flash_path), msg
 
   if opts.pdf_path:
-    pdf_path = opts.pdf_path
-    msg = 'Could not find PDF binary at %s' % pdf_path
-    assert os.path.exists(pdf_path), msg
+    msg = 'Could not find PDF binary at %s' % opts.pdf_path
+    assert os.path.exists(opts.pdf_path), msg
 
   if opts.official_builds:
-    good_rev = LooseVersion(good_rev)
-    bad_rev = LooseVersion(bad_rev)
+    context.good_revision = LooseVersion(context.good_revision)
+    context.bad_revision = LooseVersion(context.bad_revision)
   else:
-    good_rev = int(good_rev)
-    bad_rev = int(bad_rev)
+    context.good_revision = int(context.good_revision)
+    context.bad_revision = int(context.bad_revision)
 
   if opts.times < 1:
     print('Number of times to run (%d) must be greater than or equal to 1.' %
           opts.times)
     parser.print_help()
     return 1
 
   if opts.asan:
     evaluator = IsGoodASANBuild
   else:
     evaluator = AskIsGoodBuild
 
-  (min_chromium_rev, max_chromium_rev) = Bisect(
-      base_url, opts.archive, opts.official_builds, opts.aura, opts.asan,
-      opts.use_local_repo, good_rev, bad_rev, opts.times, opts.command,
-      args, opts.profile, opts.flash_path, opts.pdf_path,
+  # Save these revision numbers to compare when showing the changelog URL
+  # after the bisect.
+  good_rev = context.good_revision
+  bad_rev = context.bad_revision
+
+  (min_chromium_rev, max_chromium_rev, context) = Bisect(
+      context, opts.times, opts.command, args, opts.profile,
       not opts.not_interactive, evaluator)
 
   # Get corresponding blink revisions.
   try:
     min_blink_rev = GetBlinkRevisionForChromiumRevision(context,
                                                         min_chromium_rev)
     max_blink_rev = GetBlinkRevisionForChromiumRevision(context,
                                                         max_chromium_rev)
   except Exception:
     # Silently ignore the failure.
@@ skipping 23 matching lines @@
 
     print 'CHANGELOG URL:'
     if opts.official_builds:
       print OFFICIAL_CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
     else:
       print '  ' + CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
 
 
 if __name__ == '__main__':
   sys.exit(main())