bisect-builds.py: Fix official Google Chrome build bisection.

tools/bisect-builds.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Snapshot Build Bisect Tool
 
 This script bisects a snapshot archive using binary search. It starts at
 a bad revision (it will try to guess HEAD) and asks for a last known-good
 revision. It will then binary search across this revision range by downloading,
 unzipping, and opening Chromium for you. After testing the specific revision,
 it will ask you whether it is good or bad before continuing the search.
 """
 
 # The base URL for stored build archives.
 CHROMIUM_BASE_URL = ('http://commondatastorage.googleapis.com'
                      '/chromium-browser-snapshots')
 WEBKIT_BASE_URL = ('http://commondatastorage.googleapis.com'
                    '/chromium-webkit-snapshots')
 ASAN_BASE_URL = ('http://commondatastorage.googleapis.com'
                  '/chromium-browser-asan')
 
 # The base URL for official builds.
-OFFICIAL_BASE_URL = 'http://master.chrome.corp.google.com/official_builds'
+OFFICIAL_BASE_URL = ('http://commondatastorage.googleapis.com/'
+                     'chrome-unsigned/desktop-W15K3Y')
 
 # URL template for viewing changelogs between revisions.
 CHANGELOG_URL = ('http://build.chromium.org'
                  '/f/chromium/perf/dashboard/ui/changelog.html'
                  '?url=/trunk/src&range=%d%%3A%d')
 
 # URL template for viewing changelogs between official versions.
 OFFICIAL_CHANGELOG_URL = ('http://omahaproxy.appspot.com/changelog'
                           '?old_version=%s&new_version=%s')
 
 # DEPS file URL.
 DEPS_FILE = 'http://src.chromium.org/viewvc/chrome/trunk/src/DEPS?revision=%d'
 
+# GSUtil download URL.

[DaleCurtis, 2014/08/20 22:56:19]

Remove?

[pshenoy, 2014/08/20 23:21:55]

Done.

+GSUTIL_URL = 'http://storage.googleapis.com/pub/gsutil.tar.gz'
+
 # Blink changelogs URL.
 BLINK_CHANGELOG_URL = ('http://build.chromium.org'
                       '/f/chromium/perf/dashboard/ui/changelog_blink.html'
                       '?url=/trunk&range=%d%%3A%d')
 
 DONE_MESSAGE_GOOD_MIN = ('You are probably looking for a change made after %s ('
                          'known good), but no later than %s (first known bad).')
 DONE_MESSAGE_GOOD_MAX = ('You are probably looking for a change made after %s ('
                          'known bad), but no later than %s (first known good).')
 
@@ skipping 18 matching lines @@
 BLINK_SEARCH_PATTERN = (
     r'.*git-svn-id: svn://svn.chromium.org/blink/trunk@(\d+) ')
 
 SEARCH_PATTERN = {
     'chromium': CHROMIUM_SEARCH_PATTERN,
     'blink': BLINK_SEARCH_PATTERN,
 }
 
 ###############################################################################
 
+import cStringIO
+import httplib
 import json
 import optparse
 import os
 import re
 import shlex
 import shutil
 import subprocess
 import sys
+import tarfile
 import tempfile
 import threading
 import urllib
 from distutils.version import LooseVersion
 from xml.etree import ElementTree
 import zipfile
 
 
 class PathContext(object):
   """A PathContext is used to carry the information used to construct URLs and
@@ skipping 38 matching lines @@
       self._archive_extract_dir = 'chrome-mac'
     elif self.platform == 'win':
       self.archive_name = 'chrome-win32.zip'
       self._archive_extract_dir = 'chrome-win32'
       self._binary_name = 'chrome.exe'
     else:
       raise Exception('Invalid platform: %s' % self.platform)
 
     if is_official:
       if self.platform == 'linux':
-        self._listing_platform_dir = 'precise32bit/'
-        self.archive_name = 'chrome-precise32bit.zip'
-        self._archive_extract_dir = 'chrome-precise32bit'
+        self._listing_platform_dir = 'precise32/'
+        self.archive_name = 'chrome-precise32.zip'
+        self._archive_extract_dir = 'chrome-precise32'
       elif self.platform == 'linux64':
-        self._listing_platform_dir = 'precise64bit/'
-        self.archive_name = 'chrome-precise64bit.zip'
-        self._archive_extract_dir = 'chrome-precise64bit'
+        self._listing_platform_dir = 'precise64/'
+        self.archive_name = 'chrome-precise64.zip'
+        self._archive_extract_dir = 'chrome-precise64'
       elif self.platform == 'mac':
         self._listing_platform_dir = 'mac/'
         self._binary_name = 'Google Chrome.app/Contents/MacOS/Google Chrome'
       elif self.platform == 'win':
         self._listing_platform_dir = 'win/'
     else:
       if self.platform in ('linux', 'linux64', 'linux-arm'):
         self.archive_name = 'chrome-linux.zip'
         self._archive_extract_dir = 'chrome-linux'
         if self.platform == 'linux':
@@ skipping 227 matching lines @@
                                                     self.good_revision)
         self.bad_revision = FixChromiumRevForBlink(revlist,
                                                    revlist_all,
                                                    self,
                                                    self.bad_revision)
     return revlist
 
   def GetOfficialBuildsList(self):
     """Gets the list of official build numbers between self.good_revision and
     self.bad_revision."""
+
+    def CheckDeoptToolsInPath():

[DaleCurtis, 2014/08/20 22:56:18]

Name is wrong.

[pshenoy, 2014/08/20 23:21:55]

Done.

+      if os.environ['PATH'].find('depot_tools') != -1:

[DaleCurtis, 2014/08/20 22:56:19]

Might as well just split and look for it instead of double-checking.

[pshenoy, 2014/08/20 23:21:55]

Done.

+        delimiter = ';' if sys.platform.startswith('win') else ':'

[DaleCurtis, 2014/08/20 22:56:18]

Hmm, does this syntax actually work?

[pshenoy, 2014/08/20 23:21:55]

Yes. It does :-)

+        path_list = os.environ['PATH'].split(delimiter)
+        for path in path_list:
+          if path.find('depot_tools') != -1:
+            return path
+      return None
+
+    def RunGsutilCommand(args):
+      gsutil_path = CheckDeoptToolsInPath()
+      if gsutil_path is None:
+        print ('Follow the instructions in this document '
+               'http://dev.chromium.org/developers/how-tos/install-depot-tools'
+               ' to install depot_tools and then try again.'
+        sys.exit(1)
+      gsutil_path = os.path.join(gsutil_path, 'third_party', 'gsutil', 'gsutil')
+      gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                env=None)
+      stdout, stderr = gsutil.communicate()
+      if gsutil.returncode:
+        if ('status=403' in stderr or 'status 403' in stderr or

[DaleCurtis, 2014/08/20 22:56:18]

just re.findall(r'status[ |=]40[1|3]', stderr) ?

[pshenoy, 2014/08/20 23:21:55]

Done.

+            'status=401' in stderr or 'status 401' in stderr):
+          print ('Follow these steps to configure your credentials and try'
+                 ' running the bisect-builds.py again.:\n'
+                 '  1. Run "python %s config" and follow its instructions.\n'
+                 '  2. If you have a @google.com account, use that account.\n'
+                 '  3. For the project-id, just enter 0.' % gsutil_path)
+          sys.exit(1)
+        else:
+          raise Exception('Error running the gsutil command')
+      return stdout
+
+    def GsutilList(bucket):
+      query = 'gs://%s/' % bucket
+      stdout = RunGsutilCommand(['ls', query])
+      return [url[len(query):].strip('/') for url in stdout.splitlines()]
+
     # Download the revlist and filter for just the range between good and bad.
     minrev = min(self.good_revision, self.bad_revision)
     maxrev = max(self.good_revision, self.bad_revision)
-    handle = urllib.urlopen(OFFICIAL_BASE_URL)
-    dirindex = handle.read()
-    handle.close()
-    build_numbers = re.findall(r'<a href="([0-9][0-9].*)/">', dirindex)
+    build_numbers = GsutilList('chrome-unsigned/desktop-W15K3Y')

[DaleCurtis, 2014/08/20 22:56:19]

Constant.

[pshenoy, 2014/08/20 23:21:55]

Done.

+    revision_re = re.compile(r'(\d\d\.\d\.\d{4}\.\d+)')
+    build_numbers = filter(lambda b: revision_re.search(b), build_numbers)
     final_list = []
     i = 0
     parsed_build_numbers = [LooseVersion(x) for x in build_numbers]
+    connection = httplib.HTTPConnection('commondatastorage.googleapis.com')

[DaleCurtis, 2014/08/20 22:56:19]

Put this as a constant with the official_url ?

[pshenoy, 2014/08/20 23:21:55]

Done.

     for build_number in sorted(parsed_build_numbers):
-      path = (OFFICIAL_BASE_URL + '/' + str(build_number) + '/' +
+      if build_number > maxrev:
+        break
+      if build_number < minrev:
+        continue
+      path = ('/chrome-unsigned/desktop-W15K3Y/' + str(build_number) + '/' +

[DaleCurtis, 2014/08/20 22:56:19]

Ditto.

[pshenoy, 2014/08/20 23:21:55]

Done.

               self._listing_platform_dir + self.archive_name)
       i = i + 1
-      try:
-        connection = urllib.urlopen(path)
-        connection.close()
-        if build_number > maxrev:
-          break
-        if build_number >= minrev:
-          final_list.append(str(build_number))
-      except urllib.HTTPError:
-        pass
-    return final_list
+      connection.request('HEAD', path)
+      response = connection.getresponse()
+      if response.status == 200:
+        final_list.append(str(build_number))
+      response.read()
+    connection.close()
+    return sorted(list(set(final_list)))

[DaleCurtis, 2014/08/20 22:56:19]

I think I had a bug in my patch previously, this shouldn't be necessary.
LooseVersion should return 1:1 with the directory listing, so there shouldn't be
duplicates and they should already be sorted.

[pshenoy, 2014/08/20 23:21:55]

Done.

 
 def UnzipFilenameToDir(filename, directory):
   """Unzip |filename| to |directory|."""
   cwd = os.getcwd()
   if not os.path.isabs(filename):
     filename = os.path.join(cwd, filename)
   zf = zipfile.ZipFile(filename)
   # Make base.
   if not os.path.isdir(directory):
     os.mkdir(directory)
@@ skipping 220 matching lines @@
   """
 
   if not profile:
     profile = 'profile'
 
   good_rev = context.good_revision
   bad_rev = context.bad_revision
   cwd = os.getcwd()
 
   print 'Downloading list of known revisions...',
-  if not context.use_local_repo:
+  if not context.use_local_repo and not context.is_official:
     print '(use --use-local-repo for speed if you have a local checkout)'
   else:
     print
   _GetDownloadPath = lambda rev: os.path.join(cwd,
       '%s-%s' % (str(rev), context.archive_name))
   if context.is_official:
     revlist = context.GetOfficialBuildsList()
   else:
     revlist = context.GetRevList()
 
@@ skipping 417 matching lines @@
 
     print 'CHANGELOG URL:'
     if opts.official_builds:
       print OFFICIAL_CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
     else:
       print '  ' + CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
 
 
 if __name__ == '__main__':
   sys.exit(main())