bisect-builds.py: Fix official Google Chrome build bisection.

tools/bisect-builds.py

Index: tools/bisect-builds.py
diff --git a/tools/bisect-builds.py b/tools/bisect-builds.py
index a7938dfa1f9c8049d7597bfbf3d42a3106a54e7c..dd21b0129d25d30fde3058e5c0fc1849e18a0a9f 100755
--- a/tools/bisect-builds.py
+++ b/tools/bisect-builds.py
@@ -21,7 +21,8 @@ ASAN_BASE_URL = ('http://commondatastorage.googleapis.com'
                  '/chromium-browser-asan')
 
 # The base URL for official builds.
-OFFICIAL_BASE_URL = 'http://master.chrome.corp.google.com/official_builds'
+OFFICIAL_BASE_URL = ('http://commondatastorage.googleapis.com/'
+                     'chrome-unsigned/desktop-W15K3Y')
 
 # URL template for viewing changelogs between revisions.
 CHANGELOG_URL = ('http://build.chromium.org'
@@ -35,6 +36,9 @@ OFFICIAL_CHANGELOG_URL = ('http://omahaproxy.appspot.com/changelog'
 # DEPS file URL.
 DEPS_FILE = 'http://src.chromium.org/viewvc/chrome/trunk/src/DEPS?revision=%d'
 
+# GSUtil download URL.

[DaleCurtis, 2014/08/20 22:56:19]

Remove?

[pshenoy, 2014/08/20 23:21:55]

Done.

+GSUTIL_URL = 'http://storage.googleapis.com/pub/gsutil.tar.gz'
+
 # Blink changelogs URL.
 BLINK_CHANGELOG_URL = ('http://build.chromium.org'
                       '/f/chromium/perf/dashboard/ui/changelog_blink.html'
@@ -73,6 +77,8 @@ SEARCH_PATTERN = {
 
 ###############################################################################
 
+import cStringIO
+import httplib
 import json
 import optparse
 import os
@@ -81,6 +87,7 @@ import shlex
 import shutil
 import subprocess
 import sys
+import tarfile
 import tempfile
 import threading
 import urllib
@@ -139,13 +146,13 @@ class PathContext(object):
 
     if is_official:
       if self.platform == 'linux':
-        self._listing_platform_dir = 'precise32bit/'
-        self.archive_name = 'chrome-precise32bit.zip'
-        self._archive_extract_dir = 'chrome-precise32bit'
+        self._listing_platform_dir = 'precise32/'
+        self.archive_name = 'chrome-precise32.zip'
+        self._archive_extract_dir = 'chrome-precise32'
       elif self.platform == 'linux64':
-        self._listing_platform_dir = 'precise64bit/'
-        self.archive_name = 'chrome-precise64bit.zip'
-        self._archive_extract_dir = 'chrome-precise64bit'
+        self._listing_platform_dir = 'precise64/'
+        self.archive_name = 'chrome-precise64.zip'
+        self._archive_extract_dir = 'chrome-precise64'
       elif self.platform == 'mac':
         self._listing_platform_dir = 'mac/'
         self._binary_name = 'Google Chrome.app/Contents/MacOS/Google Chrome'
@@ -393,30 +400,71 @@ class PathContext(object):
   def GetOfficialBuildsList(self):
     """Gets the list of official build numbers between self.good_revision and
     self.bad_revision."""
+
+    def CheckDeoptToolsInPath():

[DaleCurtis, 2014/08/20 22:56:18]

Name is wrong.

[pshenoy, 2014/08/20 23:21:55]

Done.

+      if os.environ['PATH'].find('depot_tools') != -1:

[DaleCurtis, 2014/08/20 22:56:19]

Might as well just split and look for it instead of double-checking.

[pshenoy, 2014/08/20 23:21:55]

Done.

+        delimiter = ';' if sys.platform.startswith('win') else ':'

[DaleCurtis, 2014/08/20 22:56:18]

Hmm, does this syntax actually work?

[pshenoy, 2014/08/20 23:21:55]

Yes. It does :-)

+        path_list = os.environ['PATH'].split(delimiter)
+        for path in path_list:
+          if path.find('depot_tools') != -1:
+            return path
+      return None
+
+    def RunGsutilCommand(args):
+      gsutil_path = CheckDeoptToolsInPath()
+      if gsutil_path is None:
+        print ('Follow the instructions in this document '
+               'http://dev.chromium.org/developers/how-tos/install-depot-tools'
+               ' to install depot_tools and then try again.'
+        sys.exit(1)
+      gsutil_path = os.path.join(gsutil_path, 'third_party', 'gsutil', 'gsutil')
+      gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                env=None)
+      stdout, stderr = gsutil.communicate()
+      if gsutil.returncode:
+        if ('status=403' in stderr or 'status 403' in stderr or

[DaleCurtis, 2014/08/20 22:56:18]

just re.findall(r'status[ |=]40[1|3]', stderr) ?

[pshenoy, 2014/08/20 23:21:55]

Done.

+            'status=401' in stderr or 'status 401' in stderr):
+          print ('Follow these steps to configure your credentials and try'
+                 ' running the bisect-builds.py again.:\n'
+                 '  1. Run "python %s config" and follow its instructions.\n'
+                 '  2. If you have a @google.com account, use that account.\n'
+                 '  3. For the project-id, just enter 0.' % gsutil_path)
+          sys.exit(1)
+        else:
+          raise Exception('Error running the gsutil command')
+      return stdout
+
+    def GsutilList(bucket):
+      query = 'gs://%s/' % bucket
+      stdout = RunGsutilCommand(['ls', query])
+      return [url[len(query):].strip('/') for url in stdout.splitlines()]
+
     # Download the revlist and filter for just the range between good and bad.
     minrev = min(self.good_revision, self.bad_revision)
     maxrev = max(self.good_revision, self.bad_revision)
-    handle = urllib.urlopen(OFFICIAL_BASE_URL)
-    dirindex = handle.read()
-    handle.close()
-    build_numbers = re.findall(r'<a href="([0-9][0-9].*)/">', dirindex)
+    build_numbers = GsutilList('chrome-unsigned/desktop-W15K3Y')

[DaleCurtis, 2014/08/20 22:56:19]

Constant.

[pshenoy, 2014/08/20 23:21:55]

Done.

+    revision_re = re.compile(r'(\d\d\.\d\.\d{4}\.\d+)')
+    build_numbers = filter(lambda b: revision_re.search(b), build_numbers)
     final_list = []
     i = 0
     parsed_build_numbers = [LooseVersion(x) for x in build_numbers]
+    connection = httplib.HTTPConnection('commondatastorage.googleapis.com')

[DaleCurtis, 2014/08/20 22:56:19]

Put this as a constant with the official_url ?

[pshenoy, 2014/08/20 23:21:55]

Done.

     for build_number in sorted(parsed_build_numbers):
-      path = (OFFICIAL_BASE_URL + '/' + str(build_number) + '/' +
+      if build_number > maxrev:
+        break
+      if build_number < minrev:
+        continue
+      path = ('/chrome-unsigned/desktop-W15K3Y/' + str(build_number) + '/' +

[DaleCurtis, 2014/08/20 22:56:19]

Ditto.

[pshenoy, 2014/08/20 23:21:55]

Done.

               self._listing_platform_dir + self.archive_name)
       i = i + 1
-      try:
-        connection = urllib.urlopen(path)
-        connection.close()
-        if build_number > maxrev:
-          break
-        if build_number >= minrev:
-          final_list.append(str(build_number))
-      except urllib.HTTPError:
-        pass
-    return final_list
+      connection.request('HEAD', path)
+      response = connection.getresponse()
+      if response.status == 200:
+        final_list.append(str(build_number))
+      response.read()
+    connection.close()
+    return sorted(list(set(final_list)))

[DaleCurtis, 2014/08/20 22:56:19]

I think I had a bug in my patch previously, this shouldn't be necessary.
LooseVersion should return 1:1 with the directory listing, so there shouldn't be
duplicates and they should already be sorted.

[pshenoy, 2014/08/20 23:21:55]

Done.

 
 def UnzipFilenameToDir(filename, directory):
   """Unzip |filename| to |directory|."""
@@ -657,7 +705,7 @@ def Bisect(context,
   cwd = os.getcwd()
 
   print 'Downloading list of known revisions...',
-  if not context.use_local_repo:
+  if not context.use_local_repo and not context.is_official:
     print '(use --use-local-repo for speed if you have a local checkout)'
   else:
     print