bisect-builds.py: Fix official Google Chrome build bisection.

tools/bisect-builds.py

Index: tools/bisect-builds.py
diff --git a/tools/bisect-builds.py b/tools/bisect-builds.py
index a7938dfa1f9c8049d7597bfbf3d42a3106a54e7c..c14fddbc2d2d33519f4b81f0101ddd4540f4c7be 100755
--- a/tools/bisect-builds.py
+++ b/tools/bisect-builds.py
@@ -21,7 +21,8 @@ ASAN_BASE_URL = ('http://commondatastorage.googleapis.com'
                  '/chromium-browser-asan')
 
 # The base URL for official builds.
-OFFICIAL_BASE_URL = 'http://master.chrome.corp.google.com/official_builds'
+OFFICIAL_BASE_URL = ('http://commondatastorage.googleapis.com/'
+                     'chrome-unsigned/desktop-W15K3Y')
 
 # URL template for viewing changelogs between revisions.
 CHANGELOG_URL = ('http://build.chromium.org'
@@ -35,6 +36,9 @@ OFFICIAL_CHANGELOG_URL = ('http://omahaproxy.appspot.com/changelog'
 # DEPS file URL.
 DEPS_FILE = 'http://src.chromium.org/viewvc/chrome/trunk/src/DEPS?revision=%d'
 
+# GSUtil download URL.
+GSUTIL_URL = 'http://storage.googleapis.com/pub/gsutil.tar.gz'
+
 # Blink changelogs URL.
 BLINK_CHANGELOG_URL = ('http://build.chromium.org'
                       '/f/chromium/perf/dashboard/ui/changelog_blink.html'
@@ -73,6 +77,8 @@ SEARCH_PATTERN = {
 
 ###############################################################################
 
+import cStringIO
+import httplib
 import json
 import optparse
 import os
@@ -81,6 +87,7 @@ import shlex
 import shutil
 import subprocess
 import sys
+import tarfile
 import tempfile
 import threading
 import urllib
@@ -139,13 +146,13 @@ class PathContext(object):
 
     if is_official:
       if self.platform == 'linux':
-        self._listing_platform_dir = 'precise32bit/'
-        self.archive_name = 'chrome-precise32bit.zip'
-        self._archive_extract_dir = 'chrome-precise32bit'
+        self._listing_platform_dir = 'precise32/'
+        self.archive_name = 'chrome-precise32.zip'
+        self._archive_extract_dir = 'chrome-precise32'
       elif self.platform == 'linux64':
-        self._listing_platform_dir = 'precise64bit/'
-        self.archive_name = 'chrome-precise64bit.zip'
-        self._archive_extract_dir = 'chrome-precise64bit'
+        self._listing_platform_dir = 'precise64/'
+        self.archive_name = 'chrome-precise64.zip'
+        self._archive_extract_dir = 'chrome-precise64'
       elif self.platform == 'mac':
         self._listing_platform_dir = 'mac/'
         self._binary_name = 'Google Chrome.app/Contents/MacOS/Google Chrome'
@@ -393,30 +400,72 @@ class PathContext(object):
   def GetOfficialBuildsList(self):
     """Gets the list of official build numbers between self.good_revision and
     self.bad_revision."""
+
+    def DownloadGsutil():
+      gsutil_path = os.path.join(os.getcwd(), 'gsutil')
+      if not os.path.exists(gsutil_path):
+        print 'Downloading gsutil'
+        response = urllib.urlopen(GSUTIL_URL)
+        if response.getcode() == 200:
+          tar_file = tarfile.open(fileobj=cStringIO.StringIO(response.read()))
+          tar_file.extractall(os.getcwd())
+          print 'Downloaded gsutil to %s' % gsutil_path
+        else:
+          raise Exception('HTTP Error %d' % response.getcode())
+      return os.path.join(gsutil_path, 'gsutil')
+
+    def RunGsutilCommand(args):
+      gsutil_path = DownloadGsutil()
+      gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                env=None)
+      stdout, stderr = gsutil.communicate()
+      if gsutil.returncode:
+        if ('status=403' in stderr or 'status 403' in stderr or
+            'status=401' in stderr or 'status 401' in stderr):
+          print ('Follow these steps to configure your credentials and try'
+                 ' running the bisect-builds.py again.:\n'
+                 '  1. Run "python %s config" and follow its instructions.\n'
+                 '  2. If you have a @google.com account, use that account.\n'
+                 '  3. For the project-id, just enter 0.' % gsutil_path)
+          sys.exit(1)
+        else:
+          raise Exception('Error running the gsutil command')
+      return stdout
+
+    def GsutilList(bucket):
+      query = 'gs://%s/' % bucket
+      stdout = RunGsutilCommand(['ls', query])
+      return [url[len(query):].strip('/') for url in stdout.splitlines()]
+
+    def IsValidBuildNumber(build):
+      revision_re = re.compile(r'(\d\d\.\d\.\d{4}\.\d+)')

[DaleCurtis, 2014/08/20 20:46:43]

No point in using re.compile inside the function. You should compile this and
pass the compiled re into filter (i.e. drop this function) if you want this
optimization. Alternatively just drop it if you want.

[pshenoy, 2014/08/20 22:42:43]

Done.

+      return revision_re.search(build)
+
     # Download the revlist and filter for just the range between good and bad.
     minrev = min(self.good_revision, self.bad_revision)
     maxrev = max(self.good_revision, self.bad_revision)
-    handle = urllib.urlopen(OFFICIAL_BASE_URL)
-    dirindex = handle.read()
-    handle.close()
-    build_numbers = re.findall(r'<a href="([0-9][0-9].*)/">', dirindex)
+    build_numbers = GsutilList('chrome-unsigned/desktop-W15K3Y')
+    build_numbers = filter(IsValidBuildNumber, build_numbers)
     final_list = []
     i = 0
     parsed_build_numbers = [LooseVersion(x) for x in build_numbers]
+    connection = httplib.HTTPConnection('commondatastorage.googleapis.com')
     for build_number in sorted(parsed_build_numbers):
-      path = (OFFICIAL_BASE_URL + '/' + str(build_number) + '/' +
+      if build_number > maxrev:
+        break
+      if build_number < minrev:
+        continue
+      path = ('/chrome-unsigned/desktop-W15K3Y/' + str(build_number) + '/' +
               self._listing_platform_dir + self.archive_name)
       i = i + 1
-      try:
-        connection = urllib.urlopen(path)
-        connection.close()
-        if build_number > maxrev:
-          break
-        if build_number >= minrev:
-          final_list.append(str(build_number))
-      except urllib.HTTPError:
-        pass
-    return final_list
+      connection.request('HEAD', path)
+      response = connection.getresponse()
+      if response.status == 200:
+        final_list.append(str(build_number))
+      response.read()
+    connection.close()
+    return sorted(list(set(final_list)))
 
 def UnzipFilenameToDir(filename, directory):
   """Unzip |filename| to |directory|."""
@@ -657,7 +706,7 @@ def Bisect(context,
   cwd = os.getcwd()
 
   print 'Downloading list of known revisions...',
-  if not context.use_local_repo:
+  if not context.use_local_repo and not context.is_official:
     print '(use --use-local-repo for speed if you have a local checkout)'
   else:
     print