Add support to bisect nonofficial Android builds from tip of tree bot.

tools/bisect-builds.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Snapshot Build Bisect Tool
 
 This script bisects a snapshot archive using binary search. It starts at
 a bad revision (it will try to guess HEAD) and asks for a last known-good
 revision. It will then binary search across this revision range by downloading,
@@ skipping 14 matching lines @@
 
 # Base URL for downloading official builds.
 GOOGLE_APIS_URL = 'commondatastorage.googleapis.com'
 
 # The base URL for official builds.
 OFFICIAL_BASE_URL = 'http://%s/%s' % (GOOGLE_APIS_URL, GS_BUCKET_NAME)
 
 # URL template for viewing changelogs between revisions.
 CHANGELOG_URL = ('https://chromium.googlesource.com/chromium/src/+log/%s..%s')
 
+# GS bucket name for tip of tree android builds.
+ANDROID_TOT_BUCKET_NAME = ('chrome-android-tot/bisect')
+
 # GS bucket name for android unsigned official builds.
 ANDROID_BUCKET_NAME = 'chrome-unsigned/android-C4MPAR1'
 
 # The base URL for android official builds.
 ANDROID_OFFICIAL_BASE_URL = 'http://%s/%s' % (GOOGLE_APIS_URL, ANDROID_BUCKET_NAME)
 
 # URL to convert SVN revision to git hash.
 CRREV_URL = ('https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/')
 
 # URL template for viewing changelogs between official versions.
@@ skipping 89 matching lines @@
     self.is_official = is_official
     self.is_asan = is_asan
     self.build_type = 'release'
     self.flash_path = flash_path
     # Dictionary which stores svn revision number as key and it's
     # corresponding git hash as value. This data is populated in
     # _FetchAndParse and used later in GetDownloadURL while downloading
     # the build.
     self.githash_svn_dict = {}
     self.pdf_path = pdf_path
-    # android_apk defaults to Chrome.apk
-    self.android_apk = android_apk if android_apk else 'Chrome.apk'
     # The name of the ZIP file in a revision directory on the server.
     self.archive_name = None
-
+    # Whether the build should be downloaded using gsutil.
+    self.download_with_gsutil = False
     # If the script is run from a local Chromium checkout,
     # "--use-local-repo" option can be used to make the script run faster.
     # It uses "git svn find-rev <SHA1>" command to convert git hash to svn
     # revision number.
     self.use_local_repo = use_local_repo
 
     # If the script is being used for android builds.
-    self.android = self.platform.startswith('android')
+    self.is_android = self.platform.startswith('android')
+    # android_apk defaults to Chrome.apk
+    if self.is_android:
+      self.android_apk = android_apk if android_apk else 'Chrome.apk'
 
     # Set some internal members:
     #   _listing_platform_dir = Directory that holds revisions. Ends with a '/'.
     #   _archive_extract_dir = Uncompressed directory in the archive_name file.
     #   _binary_name = The name of the executable to run.
     if self.platform in ('linux', 'linux64', 'linux-arm'):
       self._binary_name = 'chrome'
     elif self.platform in ('mac', 'mac64'):
       self.archive_name = 'chrome-mac.zip'
       self._archive_extract_dir = 'chrome-mac'
     elif self.platform in ('win', 'win64'):
       self.archive_name = 'chrome-win32.zip'
       self._archive_extract_dir = 'chrome-win32'
       self._binary_name = 'chrome.exe'
-    elif self.android:
+    elif self.is_android:
       pass
     else:
       raise Exception('Invalid platform: %s' % self.platform)
 
     if is_official:
       if self.platform == 'linux':
         self._listing_platform_dir = 'precise32/'
         self.archive_name = 'chrome-precise32.zip'
         self._archive_extract_dir = 'chrome-precise32'
       elif self.platform == 'linux64':
@@ skipping 34 matching lines @@
           self._listing_platform_dir = 'Linux/'
         elif self.platform == 'linux64':
           self._listing_platform_dir = 'Linux_x64/'
         elif self.platform == 'linux-arm':
           self._listing_platform_dir = 'Linux_ARM_Cross-Compile/'
       elif self.platform == 'mac':
         self._listing_platform_dir = 'Mac/'
         self._binary_name = 'Chromium.app/Contents/MacOS/Chromium'
       elif self.platform == 'win':
         self._listing_platform_dir = 'Win/'
+      elif self.platform == 'android-arm':
+        self.archive_name = 'bisect_android.zip'
+        # Need to download builds using gsutil instead of visiting url for
+        # authentication reasons.
+        self.download_with_gsutil = True
 
   def GetASANPlatformDir(self):
     """ASAN builds are in directories like "linux-release", or have filenames
     like "asan-win32-release-277079.zip". This aligns to our platform names
     except in the case of Windows where they use "win32" instead of "win"."""
     if self.platform == 'win':
       return 'win32'
     else:
       return self.platform
 
   def GetListingURL(self, marker=None):
     """Returns the URL for a directory listing, with an optional marker."""
     marker_param = ''
     if marker:
       marker_param = '&marker=' + str(marker)
     if self.is_asan:
       prefix = '%s-%s' % (self.GetASANPlatformDir(), self.build_type)
       return self.base_url + '/?delimiter=&prefix=' + prefix + marker_param
     else:
       return (self.base_url + '/?delimiter=/&prefix=' +
               self._listing_platform_dir + marker_param)
 
   def GetDownloadURL(self, revision):
     """Gets the download URL for a build archive of a specific revision."""
     if self.is_asan:
       return '%s/%s-%s/%s-%d.zip' % (
           ASAN_BASE_URL, self.GetASANPlatformDir(), self.build_type,
           self.GetASANBaseName(), revision)
     if self.is_official:
-      if self.android:
+      if self.is_android:
         official_base_url = ANDROID_OFFICIAL_BASE_URL
       else:
         official_base_url = OFFICIAL_BASE_URL
       return '%s/%s/%s%s' % (
           official_base_url, revision, self._listing_platform_dir,
           self.archive_name)
     else:
-      if str(revision) in self.githash_svn_dict:
-        revision = self.githash_svn_dict[str(revision)]
-      return '%s/%s%s/%s' % (self.base_url, self._listing_platform_dir,
-                             revision, self.archive_name)
+      if self.is_android:
+        # These files need to be downloaded through gsutil.
+        return ('gs://%s/%s/%s' % (ANDROID_TOT_BUCKET_NAME, revision,
+                                       self.archive_name))
+      else:
+        if str(revision) in self.githash_svn_dict:
+          revision = self.githash_svn_dict[str(revision)]
+        return '%s/%s%s/%s' % (self.base_url, self._listing_platform_dir,
+                               revision, self.archive_name)
 
   def GetLastChangeURL(self):
     """Returns a URL to the LAST_CHANGE file."""
     return self.base_url + '/' + self._listing_platform_dir + 'LAST_CHANGE'
 
   def GetASANBaseName(self):
     """Returns the base name of the ASAN zip file."""
     if 'linux' in self.platform:
       return 'asan-symbolized-%s-%s' % (self.GetASANPlatformDir(),
                                         self.build_type)
@@ skipping 173 matching lines @@
         self.good_revision = FixChromiumRevForBlink(revlist,
                                                     revlist_all,
                                                     self,
                                                     self.good_revision)
         self.bad_revision = FixChromiumRevForBlink(revlist,
                                                    revlist_all,
                                                    self,
                                                    self.bad_revision)
     return revlist
 
+  def _GetHashToNumberDict(self):
+    """Gets the mapping of git hashes to git numbers from Google Storage."""
+    gs_file = 'gs://%s/gitnumbers_dict.json' % ANDROID_TOT_BUCKET_NAME
+    local_file = 'gitnumbers_dict.json'
+    GsutilDownload(gs_file, local_file)
+    json_data = open(local_file).read()
+    os.remove(local_file)
+    return json.loads(json_data)
+
+  def GetAndroidToTRevisions(self):
+    """Gets the ordered list of revisions between self.good_revision and
+    self.bad_revision from the Android tip of tree GS bucket.
+    """
+    # Dictionary that maps git hashes to git numbers. The git numbers
+    # let us order the revisions.
+    hash_to_num = self._GetHashToNumberDict()
+    try:
+      good_rev_num = hash_to_num[self.good_revision]
+      bad_rev_num = hash_to_num[self.bad_revision]
+    except KeyError:
+      exit('Error. Make sure the good and bad revisions are valid git hashes.')
+
+    # List of all builds by their git hashes in the storage bucket.
+    hash_list = GsutilList(ANDROID_TOT_BUCKET_NAME)
+
+    # Get list of builds that we want to bisect over.
+    final_list = []
+    minnum = min(good_rev_num, bad_rev_num)
+    maxnum = max(good_rev_num, bad_rev_num)
+    for githash in hash_list:
+      if len(githash) != 40:
+        continue
+      gitnumber = hash_to_num[githash]
+      if minnum < gitnumber < maxnum:
+        final_list.append(githash)
+    return sorted(final_list, key=lambda h: hash_to_num[h])
+
   def GetOfficialBuildsList(self):
     """Gets the list of official build numbers between self.good_revision and
     self.bad_revision."""
 
-    def CheckDepotToolsInPath():
-      delimiter = ';' if sys.platform.startswith('win') else ':'
-      path_list = os.environ['PATH'].split(delimiter)
-      for path in path_list:
-        if path.rstrip(os.path.sep).endswith('depot_tools'):
-          return path
-      return None
-
-    def RunGsutilCommand(args):
-      gsutil_path = CheckDepotToolsInPath()
-      if gsutil_path is None:
-        print ('Follow the instructions in this document '
-               'http://dev.chromium.org/developers/how-tos/install-depot-tools'
-               ' to install depot_tools and then try again.')
-        sys.exit(1)
-      gsutil_path = os.path.join(gsutil_path, 'third_party', 'gsutil', 'gsutil')
-      gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
-                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                env=None)
-      stdout, stderr = gsutil.communicate()
-      if gsutil.returncode:
-        if (re.findall(r'status[ |=]40[1|3]', stderr) or
-            stderr.startswith(CREDENTIAL_ERROR_MESSAGE)):
-          print ('Follow these steps to configure your credentials and try'
-                 ' running the bisect-builds.py again.:\n'
-                 '  1. Run "python %s config" and follow its instructions.\n'
-                 '  2. If you have a @google.com account, use that account.\n'
-                 '  3. For the project-id, just enter 0.' % gsutil_path)
-          sys.exit(1)
-        else:
-          raise Exception('Error running the gsutil command: %s' % stderr)
-      return stdout
-
-    def GsutilList(bucket):
-      query = 'gs://%s/' % bucket
-      stdout = RunGsutilCommand(['ls', query])
-      return [url[len(query):].strip('/') for url in stdout.splitlines()]
-
     # Download the revlist and filter for just the range between good and bad.
     minrev = min(self.good_revision, self.bad_revision)
     maxrev = max(self.good_revision, self.bad_revision)
-    if self.android:
+    if self.is_android:
       gs_bucket_name = ANDROID_BUCKET_NAME
     else:
       gs_bucket_name = GS_BUCKET_NAME
     build_numbers = GsutilList(gs_bucket_name)
     revision_re = re.compile(r'(\d\d\.\d\.\d{4}\.\d+)')
     build_numbers = filter(lambda b: revision_re.search(b), build_numbers)
     final_list = []
     parsed_build_numbers = [LooseVersion(x) for x in build_numbers]
     connection = httplib.HTTPConnection(GOOGLE_APIS_URL)
     for build_number in sorted(parsed_build_numbers):
       if build_number > maxrev:
         break
       if build_number < minrev:
         continue
       path = ('/' + gs_bucket_name + '/' + str(build_number) + '/' +
               self._listing_platform_dir + self.archive_name)
       connection.request('HEAD', path)
       response = connection.getresponse()
       if response.status == 200:
         final_list.append(str(build_number))
       response.read()
     connection.close()
     return final_list
 

[Robert Sesek, 2015/02/04 17:27:54]

nit: add another blank line

[mikecase (-- gone --), 2015/02/04 22:50:44]

Done.

+def CheckDepotToolsInPath():
+    delimiter = ';' if sys.platform.startswith('win') else ':'

[Robert Sesek, 2015/02/04 17:27:54]

nit: indent is wrong

[mikecase (-- gone --), 2015/02/04 22:50:44]

Good eye.

+    path_list = os.environ['PATH'].split(delimiter)
+    for path in path_list:
+      if path.rstrip(os.path.sep).endswith('depot_tools'):
+        return path
+    return None
+
+
+def RunGsutilCommand(args):
+  gsutil_path = CheckDepotToolsInPath()
+  if gsutil_path is None:
+    print ('Follow the instructions in this document '
+           'http://dev.chromium.org/developers/how-tos/install-depot-tools'
+           ' to install depot_tools and then try again.')
+    sys.exit(1)
+  gsutil_path = os.path.join(gsutil_path, 'third_party', 'gsutil', 'gsutil')
+  gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            env=None)
+  stdout, stderr = gsutil.communicate()
+  if gsutil.returncode:
+    if (re.findall(r'status[ |=]40[1|3]', stderr) or
+        stderr.startswith(CREDENTIAL_ERROR_MESSAGE)):
+      print ('Follow these steps to configure your credentials and try'
+             ' running the bisect-builds.py again.:\n'
+             '  1. Run "python %s config" and follow its instructions.\n'
+             '  2. If you have a @google.com account, use that account.\n'
+             '  3. For the project-id, just enter 0.' % gsutil_path)
+      sys.exit(1)
+    else:
+      raise Exception('Error running the gsutil command: %s' % stderr)
+  return stdout
+
+
+def GsutilList(bucket):
+  query = 'gs://%s/' % bucket
+  stdout = RunGsutilCommand(['ls', query])
+  return [url[len(query):].strip('/') for url in stdout.splitlines()]
+
+
+def GsutilDownload(gs_download_url, filename):
+  RunGsutilCommand(['cp', gs_download_url, filename])
+
+
 def UnzipFilenameToDir(filename, directory):
   """Unzip |filename| to |directory|."""
   cwd = os.getcwd()
   if not os.path.isabs(filename):
     filename = os.path.join(cwd, filename)
   zf = zipfile.ZipFile(filename)
   # Make base.
   if not os.path.isdir(directory):
     os.mkdir(directory)
   os.chdir(directory)
   # Extract files.
   for info in zf.infolist():
     name = info.filename
     if name.endswith('/'):  # dir
       if not os.path.isdir(name):
         os.makedirs(name)
     else:  # file
       directory = os.path.dirname(name)
-      if not os.path.isdir(directory):
+      if directory and not os.path.isdir(directory):
         os.makedirs(directory)
       out = open(name, 'wb')
       out.write(zf.read(name))
       out.close()
     # Set permissions. Permission info in external_attr is shifted 16 bits.
     os.chmod(name, info.external_attr >> 16L)
   os.chdir(cwd)
 
 
 def FetchRevision(context, rev, filename, quit_event=None, progress_event=None):
@@ skipping 14 matching lines @@
       size = blocknum * blocksize
       if totalsize == -1:  # Total size not known.
         progress = 'Received %d bytes' % size
       else:
         size = min(totalsize, size)
         progress = 'Received %d of %d bytes, %.2f%%' % (
             size, totalsize, 100.0 * size / totalsize)
       # Send a \r to let all progress messages use just one line of output.
       sys.stdout.write('\r' + progress)
       sys.stdout.flush()
-
   download_url = context.GetDownloadURL(rev)
   try:
-    urllib.urlretrieve(download_url, filename, ReportHook)
+    if context.download_with_gsutil:
+      GsutilDownload(download_url, filename)
+    else:
+      urllib.urlretrieve(download_url, filename, ReportHook)
     if progress_event and progress_event.isSet():
       print
+
   except RuntimeError:
     pass
 
 
+def RunADBCommand(args):
+  cmd = ['adb'] + args
+  adb = subprocess.Popen(['adb'] + args,
+                        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                        env=None)
+  stdout, stderr = adb.communicate()
+  return stdout
+
+
 def IsADBInstalled():
   """Checks if ADB is in the environment path."""
   try:
-    adb_output = subprocess.check_output(['adb', 'version'])
+    adb_output = RunADBCommand(['version'])
     return ('Android Debug Bridge' in adb_output)
   except OSError:
     return False
 
 
 def GetAndroidDeviceList():
   """Returns the list of Android devices attached to the host machine."""
-  lines = subprocess.check_output(['adb', 'devices']).split('\n')[1:]
+  lines = RunADBCommand(['devices']).split('\n')[1:]
   devices = []
   for line in lines:
     m = re.match('^(.*?)\s+device$', line)
     if not m:
       continue
     devices.append(m.group(1))
   return devices
 
 
-def RunAndroidRevision(context, revision, apk_file):
+def RunAndroidRevision(context, revision, zip_file):
   """Given a Chrome apk, install it on a local device, and launch Chrome."""
   devices = GetAndroidDeviceList()
   if len(devices) is not 1:
     sys.exit('Please have 1 Android device plugged in. %d devices found'
         % len(devices))
 
-  devnull = open(os.devnull, 'w')
+  if context.is_official:
+    # Downloaded file is just the .apk in this case.
+    apk_file = zip_file
+  else:
+    cwd = os.getcwd()
+    tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
+    UnzipFilenameToDir(zip_file, tempdir)
+    os.chdir(tempdir)
+    apk_file = context.android_apk
+
   package_name = ANDROID_CHROME_PACKAGE_NAME[context.android_apk]
-
-  print 'Installing new Chrome version...'
-  subprocess.call(['adb', 'install', '-r', '-d', apk_file],
-       stdout=devnull, stderr=devnull)
+  print 'Installing...'
+  RunADBCommand(['install', '-r', '-d', apk_file])
 
   print 'Launching Chrome...\n'
-  subprocess.call(['adb', 'shell', 'am', 'start', '-a',
+  RunADBCommand(['shell', 'am', 'start', '-a',
       'android.intent.action.VIEW', '-n', package_name +
-      '/com.google.android.apps.chrome.Main'], stdout=devnull, stderr=devnull)
+      '/com.google.android.apps.chrome.Main'])
 
 
 def RunRevision(context, revision, zip_file, profile, num_runs, command, args):
   """Given a zipped revision, unzip it and run the test."""
   print 'Trying revision %s...' % str(revision)
 
-  if context.android:
+  if context.is_android:
     RunAndroidRevision(context, revision, zip_file)
     # TODO(mikecase): Support running command to auto-bisect Android.
     return (None, None, None)
 
   # Create a temp directory and unzip the revision into it.
   cwd = os.getcwd()
   tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
   UnzipFilenameToDir(zip_file, tempdir)
   os.chdir(tempdir)
 
@@ skipping 26 matching lines @@
           replace('%s', ' '.join(testargs)))
 
   results = []
   for _ in range(num_runs):
     subproc = subprocess.Popen(runcommand,
                                bufsize=-1,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
     (stdout, stderr) = subproc.communicate()
     results.append((subproc.returncode, stdout, stderr))
-
   os.chdir(cwd)
   try:
     shutil.rmtree(tempdir, True)
   except Exception:
     pass
 
   for (returncode, stdout, stderr) in results:
     if returncode:
       return (returncode, stdout, stderr)
   return results[0]
@@ skipping 117 matching lines @@
 
   print 'Downloading list of known revisions...',
   if not context.use_local_repo and not context.is_official:
     print '(use --use-local-repo for speed if you have a local checkout)'
   else:
     print
   _GetDownloadPath = lambda rev: os.path.join(cwd,
       '%s-%s' % (str(rev), context.archive_name))
   if context.is_official:
     revlist = context.GetOfficialBuildsList()
+  elif context.is_android: # Android non-official
+    revlist = context.GetAndroidToTRevisions()
   else:
     revlist = context.GetRevList()
 
   # Get a list of revisions to bisect across.
   if len(revlist) < 2:  # Don't have enough builds to bisect.
     msg = 'We don\'t have enough builds to bisect. revlist: %s' % revlist
     raise RuntimeError(msg)
 
   # Figure out our bookends and first pivot point; fetch the pivot revision.
   minrev = 0
@@ skipping 362 matching lines @@
   elif opts.blink:
     base_url = WEBKIT_BASE_URL
   else:
     base_url = CHROMIUM_BASE_URL
 
   # Create the context. Initialize 0 for the revisions as they are set below.
   context = PathContext(base_url, opts.archive, opts.good, opts.bad,
                         opts.official_builds, opts.asan, opts.use_local_repo,
                         opts.flash_path, opts.pdf_path, opts.apk)
 
-  # TODO(mikecase): Add support to bisect on nonofficial builds for Android.
-  if context.android and not opts.official_builds:
-    sys.exit('Can only bisect on official builds for Android.')
+  if context.is_android and not opts.official_builds:
+    if(context.platform != 'android-arm' or

[Robert Sesek, 2015/02/04 17:27:54]

nit: space before (

[mikecase (-- gone --), 2015/02/04 22:50:44]

Done.

+        context.android_apk != 'Chrome.apk'):
+      sys.exit('For non-official builds, can only bisect'
+        ' Chrome.apk arm builds.')
 
   # If bisecting Android, we make sure we have ADB setup.
-  if context.android:
+  if context.is_android:
     if opts.adb_path:
       os.environ['PATH'] = '%s:%s' % (os.path.dirname(opts.adb_path),
           os.environ['PATH'])
     if not IsADBInstalled():
       sys.exit('Please have "adb" in PATH or use adb_path command line option'
           'to bisect Android builds.')
 
   # Pick a starting point, try to get HEAD for this.
   if not opts.bad:
     context.bad_revision = '999.0.0.0'
     context.bad_revision = GetChromiumRevision(
         context, context.GetLastChangeURL())
 
   # Find out when we were good.
   if not opts.good:
     context.good_revision = '0.0.0.0' if opts.official_builds else 0
 
   if opts.flash_path:
     msg = 'Could not find Flash binary at %s' % opts.flash_path
     assert os.path.exists(opts.flash_path), msg
 
   if opts.pdf_path:
     msg = 'Could not find PDF binary at %s' % opts.pdf_path
     assert os.path.exists(opts.pdf_path), msg
 
   if opts.official_builds:
     context.good_revision = LooseVersion(context.good_revision)
     context.bad_revision = LooseVersion(context.bad_revision)
+  elif context.is_android:
+    # Revisions are git hashes and should be left as strings.
+    pass
   else:
     context.good_revision = int(context.good_revision)
     context.bad_revision = int(context.bad_revision)
 
   if opts.times < 1:
     print('Number of times to run (%d) must be greater than or equal to 1.' %
           opts.times)
     parser.print_help()
     return 1
 
@@ skipping 45 matching lines @@
 
     print 'CHANGELOG URL:'
     if opts.official_builds:
       print OFFICIAL_CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
     else:
       PrintChangeLog(min_chromium_rev, max_chromium_rev)
 
 
 if __name__ == '__main__':
   sys.exit(main())