Converted Android Chromedriver buildbot scripts to recipes.

scripts/slave/recipe_modules/chromedriver/api.py

Index: scripts/slave/recipe_modules/chromedriver/api.py
diff --git a/scripts/slave/recipe_modules/chromedriver/api.py b/scripts/slave/recipe_modules/chromedriver/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..b26ee647a653fd13443ed2b819fcd95f9dac7454
--- /dev/null
+++ b/scripts/slave/recipe_modules/chromedriver/api.py
@@ -0,0 +1,672 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import bisect
+import csv
+import datetime
+import logging
+import re
+import time
+import urllib2
+
+from recipe_engine import recipe_api
+from . import archive as chromium_archive
+
+GS_CHROMEDRIVER_BUCKET = 'chromedriver'

[luqui, 2015/08/06 00:57:37]

Be aware that such things have a way of migrating into configs.  I think here is
good for now, as long as we aren't depending on it in a way that is inaccessible
from configs.

[mikecase (-- gone --), 2016/04/04 21:46:42]

Ack

+GS_CHROMEDRIVER_DATA_BUCKET = 'chromedriver-data'
+GS_CHROMEDRIVER_RELEASE_URL = 'chromedriver'
+GS_CONTINUOUS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/continuous'
+GS_PREBUILTS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/prebuilts'
+GS_SERVER_LOGS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/server_logs'
+
+SERVER_LOGS_LINK = (
+    'http://chromedriver-data.storage.googleapis.com/server_logs')
+TEST_LOG_FORMAT = '%s_log.json'
+GS_GIT_LOG_URL = (
+    'https://chromium.googlesource.com/chromium/src/+/%s?format=json')
+GS_SEARCH_PATTERN = (
+    r'Cr-Commit-Position: refs/heads/master@{#(\d+)}')
+CR_REV_URL = 'https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/%s'
+
+
+class ChromedriverApi(recipe_api.RecipeApi):
+
+  def __init__(self, *args, **kwargs):
+    super(ChromedriverApi, self).__init__(*args, **kwargs)
+    self.archive = chromium_archive.Archive(self)
+
+  @property
+  def chromedriver_dir(self):
+    """Gets the directory of the chromedriver code in Chromium."""
+    return self.m.path['checkout'].join('chrome', 'test', 'chromedriver')
+
+  def get_chromedriver_platform(self, is_android):
+    """Gets the platform name for chromedriver."""
+    bitness = '32'
+    if self.m.platform.is_linux and self.m.platform.bits == 64:
+      bitness = '64'
+    platform = '%s%s' % (self.m.platform.name, bitness)
+    if is_android:
+      platform = 'android'
+    return platform
+
+  def get_build_dir(self, required_paths):

[luqui, 2015/08/06 00:57:37]

Is it possible to compute what this ought to be instead of guessing & checking? 
Under what conditions do you expect each, and is that encodable with other info?

[mikecase (-- gone --), 2016/04/04 21:46:41]

I'm honestly not too familiar with this code (I'm just converting the buildbot
scripts) but this function seems unnecessary. I'm just going to get rid of it
and use api.chromium.output_dir to get the output build_dir. I expect this to
work, but if it doesn't I can always switch it back.

+    """Returns the preferred build directory that contains given paths."""
+    debug_dirs = [self.m.path['checkout'].join('out', 'Debug'),
+                  self.m.path['checkout'].join('build', 'Debug'),
+                  self.m.path['checkout'].join('xcodebuild', 'Debug')]
+    release_dirs = [self.m.path['checkout'].join('out', 'Release'),
+                    self.m.path['checkout'].join('build', 'Release'),
+                    self.m.path['checkout'].join('xcodebuild', 'Release')]
+    for build_dir in debug_dirs + release_dirs:
+      for required_path in required_paths:
+        self.m.path.mock_add_paths(self.m.path.join(build_dir, required_path))
+        if not self.m.path.exists(
+            self.m.path.join(build_dir, required_path)): # pragma: no cover
+          logging.debug('%s not in %s' % (required_path, build_dir))
+          break;
+      else:
+        return build_dir
+    raise RuntimeError('Cannot find build directory containing %s' %
+                       ', '.join(required_paths)) # pragma: no cover
+
+  def archive_prebuilts(self, commit_position):
+    """Uploads the prebuilts to google storage."""
+    chromedriver_dir = self.get_build_dir('chromedriver')
+    output_zip_path = self.m.path.mkdtemp('zip_to_archive').join(
+        'chromedriver.zip')
+    pkg = self.m.zip.make_package(root=chromedriver_dir, output=output_zip_path)
+    pkg.add_file(path=chromedriver_dir.join('chromedriver'))
+    pkg.zip(step_name='zip chromedriver')
+    self.m.gsutil.upload(
+        name='archive prebuilt',
+        source=output_zip_path,
+        bucket=GS_PREBUILTS_URL,
+        dest='r%s.zip' % commit_position,
+        link_name='Prebuilt chromedriver r%s.zip' % commit_position)
+
+  def download_prebuilts(self):
+    """Downloads the most recent prebuilts from google storage."""
+    prebuilt_dir = self.m.path.mkdtemp('prebuilt')
+    zipfile = prebuilt_dir.join('build.zip')
+    unzip_dir = prebuilt_dir.join('unzipped')
+
+    self.m.gsutil.download_latest_file(base_url=GS_PREBUILTS_URL,
+                                       partial_name=GS_PREBUILTS_URL + '/r',
+                                       destination=zipfile,
+                                       name='download latest prebuilt')
+    self.m.zip.unzip(step_name='unzip prebuilt',
+                     zip_file=zipfile,
+                     output=unzip_dir)
+    self.m.file.move(name='move prebuilt',
+                     source=unzip_dir.join('chromedriver'),
+                     dest=self.get_build_dir(['host_forwarder']),
+                     infra_step=False)
+
+  def archive_server_logs(self):
+    """Uploads chromedriver server logs to google storage."""
+    server_logs = self.m.file.glob(
+        name='glob search for server logs',
+        pattern=self.m.path['tmp_base'].join('chromedriver_*', '*'),

[luqui, 2015/08/06 00:57:36]

How do we know the logs will be put in tmp_base?  

If it's suitably clean, consider passing the log dir to this function and the
other function that writes to it, wherever that is instead of assuming
implicitly.   You could even use a temp dir instead of tmp_base so that you
don't have to sift through other junk to find what you need.

[mikecase (-- gone --), 2016/04/04 21:46:41]

Done. Made a chromedriver_log_dir to store the logs.

+        test_data=[self.m.path['tmp_base'].join('chromedriver_123'),
+                   self.m.path['tmp_base'].join('chromedriver_456'),
+                   self.m.path['tmp_base'].join('chromedriver_789')])
+    for server_log in server_logs:
+      self.m.gsutil.upload(name='upload %s' % server_log,
+                           source=server_log,
+                           bucket=GS_SERVER_LOGS_URL,
+                           dest=self.m.path.basename(server_log),
+                           link_name='server log %s' % server_log)
+
+  def get_test_results_log(self, chromedriver_platform):
+    """Gets the test results log for the given chromedriver platform.
+
+    Args:
+      chromedriver_platform: The platform that the test results log is for.
+
+    Returns:
+      A dictionary where the keys are commit positions and the values are
+      booleans indicating whether the tests passed.
+    """
+    log_name = TEST_LOG_FORMAT % chromedriver_platform
+    temp_log = self.m.path.mkdtemp('results_log').join(log_name)
+    try:
+      self.m.gsutil.download(name='download results log',
+                             source=log_name,
+                             bucket=GS_CHROMEDRIVER_DATA_BUCKET,
+                             dest=temp_log)
+      json_data = self.m.file.read(name='read results log file',
+                                   path=temp_log,
+                                   test_data='{"222222": true}')
+      json_dict = self.m.json.loads(json_data)
+    except self.m.step.StepFailure:
+      json_dict = {}
+    # Workaround for json encoding dictionary keys as strings.
+    return dict([(int(v[0]), v[1]) for v in json_dict.items()])

[luqui, 2015/08/06 00:57:37]

return { int(k): v for k, v in json_dict.items() }

[mikecase (-- gone --), 2016/04/04 21:46:41]

Done

+
+  def put_test_results_log(self, chromedriver_platform, test_results_log):
+    """Pushes the given test results log to google storage."""
+    temp_dir = self.m.path.mkdtemp('results_log')
+    log_name = TEST_LOG_FORMAT % chromedriver_platform
+    log_path = temp_dir.join(log_name)
+    self.m.file.write(name='write log to file %s' % log_name,
+                      path=log_path,
+                      data=self.m.json.dumps(test_results_log))
+    self.m.gsutil.upload(name='upload log %s' % log_name,
+                         source=log_path,
+                         bucket=GS_CHROMEDRIVER_DATA_BUCKET,
+                         dest=log_name,
+                         link_name='results log')
+
+  def update_test_results_log(self, chromedriver_platform,
+                              commit_position, passed):
+    """Updates the test results log for the given platform.

[luqui, 2015/08/06 00:57:37]

This description is a bit unclear, not sure what you mean by update.  Looks like
it uploads to GS.  Mention something about GS in the description.

[mikecase (-- gone --), 2016/04/04 21:46:42]

Done, mentioned the log is stored in GS.

+
+    Args:
+      chromedriver_platform: The platform name.
+      commit_position: The commit position number.
+      passed: Boolean indicating whether the tests passed at this
+          commit position.
+    """
+    log = self.get_test_results_log(chromedriver_platform)
+    if len(log) > 500: # pragma: no cover

[luqui, 2015/08/06 00:57:36]

I know we're only adding one at a time so it should only go at most once, but if
you change this 'if' to 'while' your intent to truncate is clearer.

[mikecase (-- gone --), 2016/04/04 21:46:41]

Agree, done.

+      del log[min(log.keys())]
+    assert commit_position not in log, \

[luqui, 2015/08/06 00:57:36]

Should this really be a failure, or just a skip & warn?

[mikecase (-- gone --), 2016/04/04 21:46:42]

Changed to skip and warn.

+        'Results already exist for commit position %s' % commit_position
+    log[commit_position] = bool(passed)
+    self.put_test_results_log(chromedriver_platform, log)
+
+  def get_version(self):
+    """Get the current chromedriver version."""
+    version_file = self.chromedriver_dir.join('VERSION')
+    return self.m.file.read(name='read version file',
+                            path=version_file,
+                            test_data="9.99").strip()
+
+  def get_supported_chrome_versions(self):
+    """Get the minimum and maximum supported Chrome versions.
+
+    Returns:
+      A tuple of the form (min_version, max_version).
+    """
+    # Minimum supported Chrome version is embedded as:
+    # const int kMinimumSupportedChromeVersion[] = {27, 0, 1453, 0};
+    version_cc_file_name = 'version.cc'
+    version_cc_file_data = self.m.file.read(
+        name='read %s' % version_cc_file_name,
+        path=self.chromedriver_dir.join('chrome', version_cc_file_name),
+        test_data='int kMinimumSupportedChromeVersion[] = {42, 0, 2311, 0};')
+    chrome_min_version_line = [x for x in version_cc_file_data.splitlines()
+                               if 'kMinimumSupportedChromeVersion' in x]
+    chrome_min_version = chrome_min_version_line[0].split('{')[1].split(',')[0]

[luqui, 2015/08/06 00:57:37]

Use a regex instead of 'in' and 'split'.  It'll be way more obvious.

[mikecase (-- gone --), 2016/04/04 21:46:41]

Switching to regex.

+
+    version_file_name = 'VERSION'
+    version_file_data = self.m.file.read(

[luqui, 2015/08/06 00:57:36]

BTW we have support for nested steps now if this gets to be a bit visually
overwhelming.  See recipes/example/nested.py

[mikecase (-- gone --), 2016/04/04 21:46:41]

Done. Trying out the nested steps.

+        name='read %s' % version_file_name,
+        path=self.m.path['checkout'].join('chrome', version_file_name),
+        test_data='MAJOR=46\nMINOR=0\nBUILD=2469\nPATCH=0')
+    chrome_max_version_line = version_file_data.splitlines()[0]
+    chrome_max_version =chrome_max_version_line.split('=')[1].strip()
+    return (chrome_min_version, chrome_max_version)
+
+  def commit_position_state(self, test_results_log, commit_position):
+    """Check the state of tests at a given commit position.
+
+    Considers tests as having passed at a commit position if they passed at
+    revisons both before and after.
+
+    Args:
+      test_results_log: A test results log dictionary from
+          get_test_results_log().
+      commit_position: The commit position to check at.
+
+    Returns:
+      'passed', 'failed', or 'unknown'
+    """
+    assert isinstance(commit_position, int), ('The commit position must '
+        'be an int')
+
+    keys = sorted(test_results_log.keys())
+    # Return passed if the exact commit position passed on Android.
+    if commit_position in test_results_log:
+      return 'passed' if test_results_log[commit_position] else 'failed'
+    # Tests were not run on this exact commit position on Android.
+    index = bisect.bisect_right(keys, commit_position)
+    # Tests have not yet run on Android at or above this commit position.
+    if index == len(test_results_log):
+      return 'unknown'
+    # No log exists for any prior commit position, assume it failed.
+    if index == 0:
+      return 'failed'
+    # Return passed if the commit position on both sides passed.
+    if test_results_log[keys[index]] and test_results_log[keys[index - 1]]:
+      return 'passed'
+    return 'failed'
+
+  def archive_good_build(self, chromedriver_platform, commit_position):
+    """Archive chromedriver binary if the build is green."""
+    assert chromedriver_platform != 'android'
+    server_name = 'chromedriver'
+    if self.m.platform.is_win:
+      server_name += '.exe'
+
+    chromedriver_dir = self.get_build_dir(server_name)
+    output_zip_path = self.m.path.mkdtemp(
+        'zip_to_archive').join('chromedriver.zip')
+    pkg = self.m.zip.make_package(root=chromedriver_dir, output=output_zip_path)

[luqui, 2015/08/06 00:57:37]

Separate these three lines by whitespace since they're really only doing one
thing.  Or factor out into a method (this pattern is used three times in the
source).   zip_one_file or smth

[mikecase (-- gone --), 2016/04/04 21:46:42]

Done. Separated out these 3 lines w/ whitespace.

+    pkg.add_file(path=chromedriver_dir.join(server_name))
+    pkg.zip(step_name='zip chromedriver')
+    build_name = 'chromedriver_%s_%s.%s.zip' % (
+        chromedriver_platform, self.get_version(), commit_position)
+    self.m.gsutil.upload(name='archive good build',
+                         source=output_zip_path,
+                         bucket=GS_CONTINUOUS_URL,
+                         dest=build_name,
+                         link_name=build_name)
+
+    latest_build_file = self.m.path.mkdtemp(
+        'latest_build_file_dir').join('latest_build')
+    latest_url = 'latest_%s' % chromedriver_platform
+    self.m.file.write(name='write latest build version',
+                      path='latest_build_file',
+                      data=build_name)
+    self.m.gsutil.upload(name='upload latest build version',
+                         source=latest_build_file,
+                         bucket=GS_CONTINUOUS_URL,
+                         dest=latest_url,
+                         link_name='latest build version')
+
+  def was_released(self, version, chromedriver_platform):
+    """Check if the specified version is released for the given platform."""
+    try:
+      step_result = self.m.gsutil.list(
+          name='list chromedriver versions',
+          url='gs://%s/%s/chromedriver_%s.zip' % (
+              GS_CHROMEDRIVER_BUCKET, version, chromedriver_platform))
+      return True
+    except self.m.step.StepFailure:
+      return False
+
+  def maybe_release(self, chromedriver_platform):
+    """Releases a release candidate if conditions are right."""
+    assert chromedriver_platform != 'android'

[luqui, 2015/08/06 20:52:58]

Why? (answer in comment)

I'm confused, since below you are talking about getting android test results and
stuff.  I just don't understand the relationship of android to linux releases I
guess

[samuong, 2015/08/06 21:58:39]

ChromeDriver doesn't run on Android. It runs on a desktop, and via adb, it
drives a browser session that runs on the Android device. The Android bot is a
Linux machine with a Nexus device attached, and we use it to gate the release.
If the desktop tests pass, but the Android ones don't, we don't release.

The desktop bots are responsible for actually pushing out a release. So they
need to check if the Android test results are good, and if so, do a release. The
Android bot never does a release itself - it just uploads its test results to
GS, and lets the desktop bots handle the rest.

So the assertion makes sense; this function should never run on the Android bot.

+    version = self.get_version()
+    # Check if the current version has already been released.
+    if self.was_released(version, chromedriver_platform):
+      return
+    # Fetch Android test results.
+    android_test_results = self.get_test_results_log('android')
+    # Fetch release candidates.
+    step_result = self.m.gsutil.list(
+        name='list release candidates',
+        url='gs://%s/chromedriver_%s_%s*' % (
+            GS_CONTINUOUS_URL, chromedriver_platform, version),
+        stdout=self.m.raw_io.output(),
+        step_test_data=lambda: self.m.raw_io.test_api.stream_output(
+            'test/dir/chromedriver_linux64_9.99.222222.zip', stream='stdout'))
+    output = step_result.stdout
+
+    candidate_pattern = re.compile(r'.*/chromedriver_%s_%s\.(\d+)\.zip$' % (
+        chromedriver_platform, version))
+    candidates = []
+    for line in output.strip().split('\n'):
+      result = candidate_pattern.match(line)
+      if not result:
+        logging.warning('Ignored line "%s"' % line)
+        continue
+      candidates.append(int(result.group(1)))
+
+    # Release the latest candidate build that passed Android, if any.
+    # In this way, if a hot fix is needed, we can delete the release from
+    # the chromedriver bucket instead of bumping up the release version number.
+    candidates.sort(reverse=True)
+    for commit_position in candidates:
+      android_result = self.commit_position_state(android_test_results,
+                                                  commit_position)
+      if android_result == 'failed':
+        logging.warning('Android tests did not pass at commit position %s' %
+            commit_position)
+      elif android_result == 'passed':
+        logging.warning('Android tests passed at commit position %s' %
+            commit_position)
+        candidate = 'chromedriver_%s_%s.%s.zip' % (
+            chromedriver_platform, version, commit_position)
+        self.release(GS_CONTINUOUS_URL, candidate,
+                     version, chromedriver_platform)
+        break
+      else:
+        logging.warning('Android tests have not run at a commit position '
+            'as recent as %s' % commit_position) # pragma: no cover
+
+  def release(self, bucket, build, version, chromedriver_platform):
+    """Releases the given candidate build."""
+    release_name = 'chromedriver_%s.zip' % chromedriver_platform
+    temp_dir = self.m.path.mkdtemp('temp_dir')
+    zip_path = temp_dir.join(build)
+    self.m.gsutil.download(name='download release build',
+                           source=build,
+                           bucket=bucket,
+                           dest=zip_path)
+    if self.m.platform.is_linux:
+      unzip_dir = temp_dir.join('unzipped')
+      self.m.zip.unzip(step_name='unzip prebuilt',
+                       zip_file=zip_path,
+                       output=unzip_dir)
+      server_path = unzip_dir.join('chromedriver')
+      zip_path = unzip_dir.join('chromedriver.zip')
+      self.m.step(name='strip', cmd=['strip', server_path])
+      pkg = self.m.zip.make_package(root=unzip_dir, output=zip_path)
+      pkg.add_file(path=server_path)
+      pkg.zip(step_name='zip chromedriver')
+
+    self.m.gsutil.upload(
+        name='upload release build',
+        source=zip_path,
+        bucket=GS_CHROMEDRIVER_BUCKET,
+        dest='%s/%s' % (version, release_name))
+
+    self.maybe_upload_release_notes(version)
+    self.maybe_update_latest_release(version)
+
+  def get_web_page_content(self, url):
+    """Return the content of the web page specified by the given url."""
+    return urllib2.urlopen(url).read()
+
+  def maybe_upload_release_notes(self, version):
+    """Upload release notes if conditions are right."""
+    # Check if the current version has already been released.
+    notes_name = 'notes.txt'
+    notes_bucket = GS_CHROMEDRIVER_BUCKET
+    notes_url = '%s/%s' % (version, notes_name)
+    prev_version = '.'.join([version.split('.')[0],
+                             str(int(version.split('.')[1]) - 1)])
+    prev_notes_url = '%s/%s' % (prev_version, notes_name)
+
+    try:
+      step_result = self.m.gsutil.list(
+          name='list version notes',
+          url='gs://%s/%s' % (notes_bucket, notes_url))
+      return
+    except self.m.step.StepFailure:
+      pass
+
+    fixed_issues = []
+    query = ('https://code.google.com/p/chromedriver/issues/csv?'
+             'can=1&q=label%%3AChromeDriver-%s&colspec=ID%%20Summary' % version)
+    issues = self.get_web_page_content(query)
+    for issue in csv.reader(issues.split('\n')[1:]):
+      if not issue:
+        continue
+      else: # pragma: no cover
+        issue_id = issue[0] 
+        desc = issue[1]
+        labels = issue[2]
+        labels.remove('ChromeDriver-%s' % version)
+        fixed_issues += ['Resolved issue %s: %s [%s]' %
+            (issue_id, desc, labels)]
+
+    
+    temp_notes_fname = self.m.path.mkdtemp('temp_notes').join('temp_notes')
+
+    try:
+      self.m.gsutil.download(name='download prev notes',
+                             source=prev_notes_url,
+                             bucket=notes_bucket,
+                             dest=temp_notes_fname)
+      old_notes = self.m.file.read(name='read old notes',
+                                   path=temp_notes_fname)
+    except self.m.step.StepFailure:
+      old_notes = ''
+
+    new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
+        version, datetime.date.today().isoformat(),
+        'Supports Chrome v%s-%s' % self.get_supported_chrome_versions(),
+        '\n'.join(fixed_issues),
+        old_notes)
+    self.m.file.write(name='write new notes',
+                      path=temp_notes_fname,
+                      data=new_notes)
+    self.m.gsutil.upload(name='upload new notes',
+                         source=temp_notes_fname,
+                         bucket=notes_bucket,
+                         dest=notes_url)
+
+  def maybe_update_latest_release(self, version):
+    """Update the file LATEST_RELEASE with the latest release version number."""
+    latest_release_fname = 'LATEST_RELEASE'
+    latest_release_url = 'gs://%s/%s' % (
+        GS_CHROMEDRIVER_BUCKET, latest_release_fname)
+
+    # Check if LATEST_RELEASE is up-to-date.
+    step_result = self.m.gsutil.cat(
+        name='check latest release',
+        url=latest_release_url,
+        stdout=self.m.raw_io.output(),
+        step_test_data=lambda: self.m.raw_io.test_api.stream_output(
+            '9.98', stream='stdout'))
+    latest_released_version = step_result.stdout
+
+    if version == latest_released_version:
+      return
+
+    # Check if chromedriver was released on all supported platforms.
+    supported_platforms = ['linux32', 'linux64', 'mac32', 'win32']
+    for platform in supported_platforms:
+      if not self.was_released(version, platform):
+        return # pragma: no cover
+
+    temp_latest_release_fname = self.m.path.mkdtemp(
+        'temp_notes').join('temp_notes')
+    self.m.file.write(name='write latest release number',
+                      path=temp_latest_release_fname,
+                      data=version)
+    self.m.gsutil.upload(name='upload latest release number',
+                         source=temp_latest_release_fname,
+                         bucket=GS_CHROMEDRIVER_BUCKET,
+                         dest=latest_release_fname)
+
+  def wait_for_latest_snapshot(self, commit_position):

[luqui, 2015/08/06 20:52:58]

This will tie up a builder just sleeping.  How long do you expect to wait here?

If this isn't just a race-condition-breaking sort of sleep, I would recommend
cron scheduling this builder and having it abort the build if the snapshot is
not available.  If it is a race-condition-breaker, consider what happens in the
case of an error where the snapshot did not get uploaded.

[mikecase (-- gone --), 2016/04/04 21:46:41]

Yeah, I think that is a good idea. I'll look to see if I can find what exactly
is building the snapshots this bot needs, and trigger it from there.

+    while True:
+      snapshot_position = self.archive.get_latest_snapshot_version()
+      if commit_position is not None and snapshot_position is not None:
+        if int(snapshot_position) >= int(commit_position):
+          break
+        logging.debug('Waiting for snapshot >= %s, found %s' %
+                      (commit_position, snapshot_position)) # pragma: no cover
+      time.sleep(60) # pragma: no cover
+    logging.debug('Got snapshot commit position %s' % snapshot_position)
+
+  def add_tools_to_path(self, env):
+    """Add some tools like Ant and Java to PATH for testing steps to use."""
+    paths = []
+    error_message = ''
+    if self.m.platform.is_win:
+      paths = [
+          # Path to Ant and Java, required for the java acceptance tests.
+          'C:\\Program Files (x86)\\Java\\ant\\bin',

[luqui, 2015/08/06 20:52:58]

Ack hardcoded paths!

I do not have a suggestion here.  Eventually we'll isolate these maybe?

[samuong, 2015/08/06 21:58:39]

I'd love to get rid of this too, but we'd need to somehow get ant/java installed
on our Windows bot first. I'm not sure why it wasn't done in the first place (I
wasn't around when it was set up) but what do we need to do to get this set up?
Right now we have to rdp into the bot and install it manually...

+          'C:\\Program Files (x86)\\Java\\jre\\bin',
+      ]
+      error_message = ('Java test steps will fail as expected and '
+                       'they can be ignored.\n'
+                       'Ant, Java or others might not be installed on bot.\n'
+                       'Please refer to page "WATERFALL" on site '
+                       'go/chromedriver.')
+    if paths:
+      path_missing = False
+      for path in paths:
+        if (not self.m.path.exists(path) or
+            not self.m.file.listdir(name='tools', path=path)):
+          logging.warning('Directory "%s" is not found or empty.' % path)
+          path_missing = True
+      if path_missing:
+        logging.error(error_message)
+        return
+      env['PATH'] = self.m.path.pathsep.join(
+          [env['PATH']] + paths) # pragma: no cover
+
+  def generate_test_command(self, script, chromedriver, ref_chromedriver=None,
+                            chrome=None, chrome_version=None,
+                            android_package=None, build_type=None, verbose=None):
+    log_path = self.m.path.mkdtemp('chromedriver_').join('test_log')
+    cmd = [script, '--chromedriver', chromedriver, '--log-path', log_path]
+    if ref_chromedriver:
+      cmd.extend(['--reference-chromedriver', ref_chromedriver])
+    if chrome:
+      cmd.extend(['--chrome', chrome])
+    if chrome_version:
+      cmd.extend(['--chrome-version', chrome_version])
+    if build_type:
+      cmd.extend(['--build-type', build_type])
+    if verbose:
+      cmd.extend(['--verbose'])
+    if android_package:
+      cmd = ['xvfb-run', '-a'] + cmd
+      cmd.extend(['--android-package', android_package])
+    return cmd
+
+  def run_python_tests(self, chromedriver, ref_chromedriver,
+                       chrome=None, chrome_version=None,
+                       chrome_version_name=None, android_package=None,
+                       build_type=None, **kwargs):
+    version_info = ''
+    if chrome_version_name:
+      version_info = '(%s)' % chrome_version_name
+    self.m.step('python_tests%s' % version_info,
+                self.generate_test_command('run_py_tests.py', chromedriver,
+                                           ref_chromedriver=ref_chromedriver,
+                                           chrome=chrome,
+                                           chrome_version=chrome_version,
+                                           android_package=android_package,
+                                           build_type=build_type),
+                **kwargs)
+
+  def run_java_tests(self, chromedriver, chrome=None, chrome_version=None,
+                     chrome_version_name=None, android_package=None,
+                     build_type=None, verbose=False, **kwargs):
+    version_info = ''
+    if chrome_version_name:
+      version_info = '(%s)' % chrome_version_name
+    self.m.step('java_tests%s' % version_info,
+                self.generate_test_command('run_java_tests.py',
+                                           chromedriver,
+                                           ref_chromedriver=None,
+                                           chrome=chrome,
+                                           chrome_version=chrome_version,
+                                           android_package=android_package,
+                                           build_type=build_type,
+                                           verbose=verbose),
+                **kwargs)
+
+  def run_cpp_tests(self, cpp_tests):
+    self.m.step('chromedriver_tests', [cpp_tests])
+
+  def download_chrome(self, version_name, revision, download_site):
+    try:
+      temp_dir = self.m.path.mkdtemp('chrome')
+      return (temp_dir, self.archive.download_chrome(
+          revision, temp_dir, download_site))
+    except self.m.step.StepFailure:
+      return (None, None)
+
+  def kill_chromes(self):
+    chrome_map = {
+        'win': 'chrome.exe',
+        'mac': 'Chromium',
+        'linux': 'chrome',
+    }
+    if self.m.platform.is_win:
+      cmd = ['taskkill', '/F', '/IM']
+    else:
+      cmd = ['killall', '-9']
+    cmd.append(chrome_map[self.m.platform.name])
+    self.m.step('kill chromes', cmd)
+
+  def run_all_tests(self, android_packages=None):
+    exe_postfix = ''
+    if self.m.platform.is_win:
+      exe_postfix = '.exe'
+    cpp_tests_name = 'chromedriver_tests' + exe_postfix
+    server_name = 'chromedriver' + exe_postfix
+
+    required_build_outputs = [server_name]
+    if not android_packages:
+      required_build_outputs += [cpp_tests_name]
+    build_dir = self.get_build_dir(required_build_outputs)
+    build_type = self.m.path.basename(build_dir)
+    logging.info('Using build outputs from %s' % build_dir)
+
+    test_env = {'PATH': '%(PATH)s'}
+    self.add_tools_to_path(test_env)
+
+    chromedriver = self.m.path.join(build_dir, server_name)
+    platform_name = self.m.platform.name
+    if self.m.platform.is_linux and self.m.platform.bits == 64:
+      platform_name = 'linux64'
+    ref_chromedriver = self.m.path.join(
+        self.m.path['checkout'],
+        'chrome', 'test', 'chromedriver', 'third_party', 'java_tests',
+        'reference_builds', 'chromedriver_%s%s' % (platform_name, exe_postfix))
+
+    failure = False
+    if android_packages:
+      test_env['PATH'] = self.m.path.pathsep.join([
+          test_env['PATH'],
+          str(self.m.path['checkout'].join(
+              'chrome', 'test', 'chromedriver', 'chrome'))])
+      for package in android_packages:
+        try:
+          with self.m.step.defer_results():
+            self.run_python_tests(chromedriver,
+                                  ref_chromedriver,
+                                  chrome_version_name=package,
+                                  android_package=package,
+                                  build_type=build_type,
+                                  env=test_env)
+            self.run_java_tests(chromedriver,
+                                chrome_version_name=package,
+                                android_package=package,
+                                build_type=build_type,
+                                verbose=True,
+                                env=test_env)
+        except recipe_api.AggregatedStepFailure:
+          failure = True
+    else:
+      latest_snapshot_revision = self.archive.get_latest_snapshot_version()
+
+      versions = [
+          ['HEAD', latest_snapshot_revision],
+          ['44', self.archive.CHROME_44_REVISION],
+          ['43', self.archive.CHROME_43_REVISION],
+          ['42', self.archive.CHROME_42_REVISION],
+      ]
+      for version in versions:
+        download_site = chromium_archive.Url.GS_CHROMIUM_CONTINUOUS_URL
+        version_name = version[0]
+        if version_name == 'HEAD':
+          version_name = version[1]
+          download_site = self.archive.get_snapshot_download_site()
+        temp_dir, chrome_path = self.download_chrome(
+            version_name, version[1], download_site)
+        try:
+          with self.m.step.defer_results():
+            self.run_python_tests(chromedriver,
+                                  ref_chromedriver,
+                                  chrome=chrome_path,
+                                  chrome_version=version[0],
+                                  chrome_version_name='v%s' % version_name,
+                                  build_type=build_type)
+            self.run_java_tests(chromedriver, chrome=chrome_path,
+                                chrome_version=version[0],
+                                chrome_version_name='v%s' % version_name,
+                                build_type=build_type)
+            self.kill_chromes()
+        except recipe_api.AggregatedStepFailure:
+          failure = True
+
+      try:
+        cpp_tests = self.m.path.join(build_dir, cpp_tests_name)
+        self.run_cpp_tests(cpp_tests)
+      except recipe_api.StepFailure:
+        failure = True
+    return not failure