bisect-builds.py: Fix official Google Chrome build bisection.

tools/bisect-builds.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Snapshot Build Bisect Tool
 
 This script bisects a snapshot archive using binary search. It starts at
 a bad revision (it will try to guess HEAD) and asks for a last known-good
 revision. It will then binary search across this revision range by downloading,
 unzipping, and opening Chromium for you. After testing the specific revision,
 it will ask you whether it is good or bad before continuing the search.
 """
 
 # The base URL for stored build archives.
 CHROMIUM_BASE_URL = ('http://commondatastorage.googleapis.com'
                      '/chromium-browser-snapshots')
 WEBKIT_BASE_URL = ('http://commondatastorage.googleapis.com'
                    '/chromium-webkit-snapshots')
 ASAN_BASE_URL = ('http://commondatastorage.googleapis.com'
                  '/chromium-browser-asan')
 
 # The base URL for official builds.
-OFFICIAL_BASE_URL = 'http://master.chrome.corp.google.com/official_builds'
+OFFICIAL_BASE_URL = ('http://commondatastorage.googleapis.com/'

[DaleCurtis, 2014/08/21 00:18:55]

Construct the official base_url using the bucket_name and google apis url
constants?

[pshenoy, 2014/08/21 16:27:23]

Done.

+                     'chrome-unsigned/desktop-W15K3Y')
+# GS bucket name.
+GS_BUCKET_NAME = 'chrome-unsigned/desktop-W15K3Y'
+
+# Base URL for downloading official builds.
+GOOGLE_APIS_URL = 'commondatastorage.googleapis.com'
 
 # URL template for viewing changelogs between revisions.
 CHANGELOG_URL = ('http://build.chromium.org'
                  '/f/chromium/perf/dashboard/ui/changelog.html'
                  '?url=/trunk/src&range=%d%%3A%d')
 
 # URL template for viewing changelogs between official versions.
 OFFICIAL_CHANGELOG_URL = ('http://omahaproxy.appspot.com/changelog'
                           '?old_version=%s&new_version=%s')
 
@@ skipping 31 matching lines @@
 BLINK_SEARCH_PATTERN = (
     r'.*git-svn-id: svn://svn.chromium.org/blink/trunk@(\d+) ')
 
 SEARCH_PATTERN = {
     'chromium': CHROMIUM_SEARCH_PATTERN,
     'blink': BLINK_SEARCH_PATTERN,
 }
 
 ###############################################################################
 
+import httplib
 import json
 import optparse
 import os
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import threading
@@ skipping 46 matching lines @@
       self._archive_extract_dir = 'chrome-mac'
     elif self.platform == 'win':
       self.archive_name = 'chrome-win32.zip'
       self._archive_extract_dir = 'chrome-win32'
       self._binary_name = 'chrome.exe'
     else:
       raise Exception('Invalid platform: %s' % self.platform)
 
     if is_official:
       if self.platform == 'linux':
-        self._listing_platform_dir = 'precise32bit/'
-        self.archive_name = 'chrome-precise32bit.zip'
-        self._archive_extract_dir = 'chrome-precise32bit'
+        self._listing_platform_dir = 'precise32/'
+        self.archive_name = 'chrome-precise32.zip'
+        self._archive_extract_dir = 'chrome-precise32'
       elif self.platform == 'linux64':
-        self._listing_platform_dir = 'precise64bit/'
-        self.archive_name = 'chrome-precise64bit.zip'
-        self._archive_extract_dir = 'chrome-precise64bit'
+        self._listing_platform_dir = 'precise64/'
+        self.archive_name = 'chrome-precise64.zip'
+        self._archive_extract_dir = 'chrome-precise64'
       elif self.platform == 'mac':
         self._listing_platform_dir = 'mac/'
         self._binary_name = 'Google Chrome.app/Contents/MacOS/Google Chrome'
       elif self.platform == 'win':
         self._listing_platform_dir = 'win/'
     else:
       if self.platform in ('linux', 'linux64', 'linux-arm'):
         self.archive_name = 'chrome-linux.zip'
         self._archive_extract_dir = 'chrome-linux'
         if self.platform == 'linux':
@@ skipping 227 matching lines @@
                                                     self.good_revision)
         self.bad_revision = FixChromiumRevForBlink(revlist,
                                                    revlist_all,
                                                    self,
                                                    self.bad_revision)
     return revlist
 
   def GetOfficialBuildsList(self):
     """Gets the list of official build numbers between self.good_revision and
     self.bad_revision."""
+
+    def CheckDepotToolsInPath():
+      delimiter = ';' if sys.platform.startswith('win') else ':'
+      path_list = os.environ['PATH'].split(delimiter)
+      for path in path_list:
+        if path.find('depot_tools') != -1:
+          return path
+      return None
+
+    def RunGsutilCommand(args):
+      gsutil_path = CheckDepotToolsInPath()
+      if gsutil_path is None:
+        print ('Follow the instructions in this document '
+               'http://dev.chromium.org/developers/how-tos/install-depot-tools'
+               ' to install depot_tools and then try again.')
+        sys.exit(1)
+      gsutil_path = os.path.join(gsutil_path, 'third_party', 'gsutil', 'gsutil')
+      gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                env=None)
+      stdout, stderr = gsutil.communicate()
+      if gsutil.returncode:
+        if re.findall(r'status[ |=]40[1|3]', stderr):
+          print ('Follow these steps to configure your credentials and try'
+                 ' running the bisect-builds.py again.:\n'
+                 '  1. Run "python %s config" and follow its instructions.\n'
+                 '  2. If you have a @google.com account, use that account.\n'
+                 '  3. For the project-id, just enter 0.' % gsutil_path)
+          sys.exit(1)
+        else:
+          raise Exception('Error running the gsutil command')
+      return stdout
+
+    def GsutilList(bucket):
+      query = 'gs://%s/' % bucket
+      stdout = RunGsutilCommand(['ls', query])
+      return [url[len(query):].strip('/') for url in stdout.splitlines()]
+
     # Download the revlist and filter for just the range between good and bad.
     minrev = min(self.good_revision, self.bad_revision)
     maxrev = max(self.good_revision, self.bad_revision)
-    handle = urllib.urlopen(OFFICIAL_BASE_URL)
-    dirindex = handle.read()
-    handle.close()
-    build_numbers = re.findall(r'<a href="([0-9][0-9].*)/">', dirindex)
+    build_numbers = GsutilList(GS_BUCKET_NAME)
+    revision_re = re.compile(r'(\d\d\.\d\.\d{4}\.\d+)')
+    build_numbers = filter(lambda b: revision_re.search(b), build_numbers)
     final_list = []
     i = 0
     parsed_build_numbers = [LooseVersion(x) for x in build_numbers]
+    connection = httplib.HTTPConnection(GOOGLE_APIS_URL)
     for build_number in sorted(parsed_build_numbers):
-      path = (OFFICIAL_BASE_URL + '/' + str(build_number) + '/' +
+      if build_number > maxrev:
+        break
+      if build_number < minrev:
+        continue
+      path = ('/' + GS_BUCKET_NAME + '/' + str(build_number) + '/' +
               self._listing_platform_dir + self.archive_name)
       i = i + 1
-      try:
-        connection = urllib.urlopen(path)
-        connection.close()
-        if build_number > maxrev:
-          break
-        if build_number >= minrev:
-          final_list.append(str(build_number))
-      except urllib.HTTPError:
-        pass
+      connection.request('HEAD', path)
+      response = connection.getresponse()
+      if response.status == 200:
+        final_list.append(str(build_number))
+      response.read()
+    connection.close()
     return final_list
 
 def UnzipFilenameToDir(filename, directory):
   """Unzip |filename| to |directory|."""
   cwd = os.getcwd()
   if not os.path.isabs(filename):
     filename = os.path.join(cwd, filename)
   zf = zipfile.ZipFile(filename)
   # Make base.
   if not os.path.isdir(directory):
@@ skipping 221 matching lines @@
   """
 
   if not profile:
     profile = 'profile'
 
   good_rev = context.good_revision
   bad_rev = context.bad_revision
   cwd = os.getcwd()
 
   print 'Downloading list of known revisions...',
-  if not context.use_local_repo:
+  if not context.use_local_repo and not context.is_official:
     print '(use --use-local-repo for speed if you have a local checkout)'
   else:
     print
   _GetDownloadPath = lambda rev: os.path.join(cwd,
       '%s-%s' % (str(rev), context.archive_name))
   if context.is_official:
     revlist = context.GetOfficialBuildsList()
   else:
     revlist = context.GetRevList()
 
@@ skipping 417 matching lines @@
 
     print 'CHANGELOG URL:'
     if opts.official_builds:
       print OFFICIAL_CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
     else:
       print '  ' + CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
 
 
 if __name__ == '__main__':
   sys.exit(main())