Converted Android Chromedriver buildbot scripts to recipes.

scripts/slave/recipe_modules/chromedriver/api.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import bisect
+import csv
+import datetime
+import re
+import tempfile
+import time
+
+from recipe_engine import recipe_api
+from . import archive as chromium_archive

[jbudorick, 2016/05/04 18:10:42]

If you're going to alias it in the same recipe_module, just rename the module to
chromium_archive.

[mikecase (-- gone --), 2016/06/16 17:55:08]

Done

+
+GS_CHROMEDRIVER_BUCKET = 'chromedriver'
+GS_CHROMEDRIVER_DATA_BUCKET = 'chromedriver-data'
+GS_CHROMEDRIVER_RELEASE_URL = 'chromedriver'
+GS_CONTINUOUS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/continuous'
+GS_PREBUILTS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/prebuilts'
+GS_SERVER_LOGS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/server_logs'
+
+TEST_LOG_FORMAT = '%s_log.json'
+
+
+class ChromedriverApi(recipe_api.RecipeApi):
+
+  def __init__(self, *args, **kwargs):
+    super(ChromedriverApi, self).__init__(*args, **kwargs)
+    self.archive = chromium_archive.Archive(self.m)

[jbudorick, 2016/05/04 18:10:42]

Why is this public? If you have to provide public access to it, it should be
through a property.

[mikecase (-- gone --), 2016/06/16 17:55:08]

Done

+    self._chromedriver_log_dir = None
+
+  @property
+  def chromedriver_dir(self):
+    """Gets the directory of the chromedriver code in Chromium."""
+    return self.m.path['checkout'].join('chrome', 'test', 'chromedriver')
+
+  @property
+  def chromedriver_log_dir(self):
+    """Gets the directory of where the chromedriver logs are saved."""
+    if not self._chromedriver_log_dir:
+      self._chromedriver_log_dir = self.m.path.mkdtemp('chromedriver_logs')

[jbudorick, 2016/05/04 18:10:43]

Does this get cleaned up somewhere? Should it?

+    return self._chromedriver_log_dir
+
+  def get_chromedriver_platform(self, is_android):
+    """Gets the platform name for chromedriver."""
+    bitness = '32'

[jbudorick, 2016/05/04 18:10:42]

nit: 'bitness'? Why not just 'bits'?

[mikecase (-- gone --), 2016/06/16 17:55:08]

The reason is because that is what the scripts I converted had. Agree 'bits' is
a better name however.

+    if self.m.platform.is_linux and self.m.platform.bits == 64:
+      bitness = '64'
+    platform = '%s%s' % (self.m.platform.name, bitness)
+    if is_android:

[jbudorick, 2016/05/04 18:10:42]

Can you just do this at the beginning?

[mikecase (-- gone --), 2016/06/16 17:55:08]

Done

+      platform = 'android'
+    return platform
+
+  def archive_prebuilts(self, commit_position):
+    """Uploads the prebuilts to google storage."""
+    output_zip_path = self.m.path.mkdtemp('zip_to_archive').join(

[jbudorick, 2016/05/04 18:10:43]

Same question w.r.t. cleaning up the temporary directory.

[mikecase (-- gone --), 2016/06/16 17:55:08]

I made sure I clean up all temp dirs now.

+        'chromedriver.zip')
+
+    pkg = self.m.zip.make_package(
+        root=self.m.chromium.output_dir, output=output_zip_path)
+    pkg.add_file(path=self.m.chromium.output_dir.join('chromedriver'))
+    pkg.zip(step_name='zip chromedriver')
+
+    self.m.gsutil.upload(
+        name='archive prebuilt',
+        source=output_zip_path,
+        bucket=GS_PREBUILTS_URL,
+        dest='r%s.zip' % commit_position,
+        link_name='Prebuilt chromedriver r%s.zip' % commit_position)
+
+  def download_prebuilts(self):
+    """Downloads the most recent prebuilts from google storage."""
+    prebuilt_dir = self.m.path.mkdtemp('prebuilt')
+    zipfile = prebuilt_dir.join('build.zip')
+    unzip_dir = prebuilt_dir.join('unzipped')
+
+    self.m.gsutil.download_latest_file(base_url=GS_PREBUILTS_URL,
+                                       partial_name=GS_PREBUILTS_URL + '/r',
+                                       destination=zipfile,
+                                       name='download latest prebuilt')
+    self.m.zip.unzip(step_name='unzip prebuilt',
+                     zip_file=zipfile,
+                     output=unzip_dir)
+    self.m.file.move(name='move prebuilt',
+                     source=unzip_dir.join('chromedriver'),
+                     dest=self.m.chromium.output_dir,
+                     infra_step=False)
+
+  def archive_server_logs(self):
+    """Uploads chromedriver server logs to google storage."""
+    server_logs = self.m.file.glob(
+        name='glob search for server logs',
+        pattern=self.chromedriver_log_dir.join('*'),
+        test_data=[self.chromedriver_log_dir.join('chromedriver_123'),
+                   self.chromedriver_log_dir.join('chromedriver_456'),
+                   self.chromedriver_log_dir.join('chromedriver_789')])
+    with self.m.step.nest('upload chromedriver server logs'):
+      for server_log in server_logs:
+        self.m.gsutil.upload(name='uploading %s' % server_log,
+                             source=server_log,
+                             bucket=GS_SERVER_LOGS_URL,
+                             dest=self.m.path.basename(server_log),
+                             link_name='server log %s' % server_log)
+
+  def get_test_results_log(self, chromedriver_platform):
+    """Gets the test results log for the given chromedriver platform.
+
+    Args:
+      chromedriver_platform: The platform that the test results log is for.
+
+    Returns:
+      A dictionary where the keys are commit positions and the values are
+      booleans indicating whether the tests passed.
+    """
+    log_name = TEST_LOG_FORMAT % chromedriver_platform
+    temp_log = self.m.path.mkdtemp('results_log').join(log_name)
+    try:
+      self.m.gsutil.download(name='download results log',
+                             source=log_name,
+                             bucket=GS_CHROMEDRIVER_DATA_BUCKET,
+                             dest=temp_log)
+      json_data = self.m.file.read(name='read results log file',
+                                   path=temp_log,
+                                   test_data='{"2222222222": true}')
+      json_dict = self.m.json.loads(json_data)
+    except self.m.step.StepFailure:
+      json_dict = {}
+    return { int(k): v for k, v in json_dict.items() }

[jbudorick, 2016/05/04 18:10:43]

nit: no spaces after { & before }

[mikecase (-- gone --), 2016/06/16 17:55:08]

done

+
+  def put_test_results_log(self, chromedriver_platform, test_results_log):
+    """Pushes the given test results log to google storage."""
+    temp_dir = self.m.path.mkdtemp('results_log')
+    log_name = TEST_LOG_FORMAT % chromedriver_platform
+    log_path = temp_dir.join(log_name)
+    self.m.file.write(name='write results log to file %s' % log_name,
+                      path=log_path,
+                      data=self.m.json.dumps(test_results_log))
+    self.m.gsutil.upload(name='upload results log %s' % log_name,
+                         source=log_path,
+                         bucket=GS_CHROMEDRIVER_DATA_BUCKET,
+                         dest=log_name,
+                         link_name='results log')
+
+  def update_test_results_log(self, chromedriver_platform,
+                              commit_position, passed):
+    """Updates the test results log stored in GS for the given platform.
+
+    Args:
+      chromedriver_platform: The platform name.
+      commit_position: The commit position number.
+      passed: Boolean indicating whether the tests passed at this
+          commit position.
+    """
+    log = self.get_test_results_log(chromedriver_platform)
+    while len(log) > 500: # pragma: no cover

[jbudorick, 2016/05/04 18:10:43]

yikes

[mikecase (-- gone --), 2016/06/16 17:55:08]

haha. This was ported from the chromedriver scripts. I can change implementation
if you want. I did at least factor out that 500 constant to be
TEST_LOG_MAX_LENGTH

+      del log[min(log.keys())]
+    if commit_position not in log:
+      log[commit_position] = bool(passed)
+      self.put_test_results_log(chromedriver_platform, log)
+    else:
+      raise self.m.step.StepFailure(
+          'Results already exist for commit position %s' % commit_position)
+
+  def get_version(self):
+    """Get the current chromedriver version."""
+    version_file = self.chromedriver_dir.join('VERSION')

[jbudorick, 2016/05/04 18:10:43]

Is VERSION just a text file with a string version in it?

[mikecase (-- gone --), 2016/06/16 17:55:08]

yes. But this function is now gone as I am only adding Android support.

+    return self.m.file.read(name='read version file',
+                            path=version_file,
+                            test_data="9.99").strip()
+
+  def get_supported_chrome_versions(self):
+    """Get the minimum and maximum supported Chrome versions.
+
+    Returns:
+      A tuple of the form (min_version, max_version).
+    """
+    # Minimum supported Chrome version is embedded as:
+    # const int kMinimumSupportedChromeVersion[] = {27, 0, 1453, 0};
+    with self.m.step.nest('Getting supported chrome versions'):
+      version_cc_file_name = 'version.cc'
+      version_cc_file_data = self.m.file.read(
+          name='read %s for min supported chrome' % version_cc_file_name,
+          path=self.chromedriver_dir.join('chrome', version_cc_file_name),
+          test_data='int kMinimumSupportedChromeVersion[] = {42, 0, 2311, 0};')
+      min_version_re = re.compile(
+          r'int kMinimumSupportedChromeVersion\[\] = {(\d+), \d+, \d+, \d+};')

[jbudorick, 2016/05/04 18:10:43]

- module-scope constant, plz
- I'm not crazy about regex searching a c++ file.
- what if the line starts with spaces?

[mikecase (-- gone --), 2016/06/16 17:55:08]

This is a copy of the implementation from the Chromedriver buildbot scripts.
BUt, this function is now gone as I am only adding Android support. Will review
this comment when going back to add Linux/Windows.

+      for line in version_cc_file_data.splitlines():
+        m = min_version_re.match(line)
+        if m:
+          chrome_min_version = m.group(1)
+          break
+      else: # pragma: no cover

[jbudorick, 2016/05/04 18:10:42]

This should be coverable in example.py.

+        chrome_min_version = None
+        self.m.step.active_result.presentation.step_text = (
+            'Error parsing chrome min version from %s file.'
+            % version_cc_file_name)
+        self.m.step.active_result.presentation.status = self.m.step.FAILURE
+
+      version_file_name = 'VERSION'
+      version_file_data = self.m.file.read(
+          name='read %s for max supported chrome' % version_file_name,
+          path=self.m.path['checkout'].join('chrome', version_file_name),
+          test_data='MAJOR=46\nMINOR=0\nBUILD=2469\nPATCH=0')
+      max_version_re = re.compile(r'MAJOR=(\d+)')

[jbudorick, 2016/05/04 18:10:42]

module-scope constant here too, plz

+      for line in version_file_data.splitlines():
+        m = max_version_re.match(line)
+        if m:
+          chrome_max_version = m.group(1)
+          break
+      else: # pragma: no cover

[jbudorick, 2016/05/04 18:10:42]

This should also be coverable in example.py.

+        chrome_max_version = None
+        self.m.step.active_result.presentation.step_text = (
+            'Error parsing chrome max version from %s file.'
+            % version_file_name)
+        self.m.step.active_result.presentation.status = self.m.step.FAILURE
+
+      return (chrome_min_version, chrome_max_version)
+
+  def commit_position_state(self, test_results_log, commit_position):
+    """Check the state of tests at a given commit position.
+
+    Considers tests as having passed at a commit position if they passed at
+    revisons both before and after.
+
+    Args:
+      test_results_log: A test results log dictionary from
+          |get_test_results_log|.
+      commit_position: The commit position to check at.
+
+    Returns:
+      'passed', 'failed', or 'unknown'
+    """
+    assert isinstance(commit_position, int), (
+        'The commit position must be an int')
+
+    keys = sorted(test_results_log.keys())

[jbudorick, 2016/05/04 18:10:43]

nit: a few blank lines in here would make this more readable.

+    # Return passed if the exact commit position passed on Android.
+    if commit_position in test_results_log:
+      return 'passed' if test_results_log[commit_position] else 'failed'

[jbudorick, 2016/05/04 18:10:42]

You have 'failed' three times in this function. As it's part of the interface,
perhaps it'd be better to constify it at class scope.

(similarly for 'passed', and maybe 'unknown')

+    # Tests were not run on this exact commit position on Android.
+    index = bisect.bisect_right(keys, commit_position)
+    # Tests have not yet run on Android at or above this commit position.
+    if index == len(test_results_log):
+      return 'unknown'
+    # No log exists for any prior commit position, assume it failed.
+    if index == 0:
+      return 'failed' # pragma: no cover
+    # Return passed if the commit position on both sides passed.
+    if test_results_log[keys[index]] and test_results_log[keys[index - 1]]:
+      return 'passed' # pragma: no cover
+    return 'failed'
+
+  def archive_good_build(self, chromedriver_platform, commit_position):
+    """Archive chromedriver binary if the build is green."""
+    assert chromedriver_platform != 'android'

[jbudorick, 2016/05/04 18:10:42]

?

+    server_name = 'chromedriver'
+    if self.m.platform.is_win:
+      server_name += '.exe'
+
+    output_zip_path = self.m.path.mkdtemp(

[jbudorick, 2016/05/04 18:10:43]

temp directory cleanup, etc

+        'zip_to_archive').join('chromedriver.zip')
+
+    pkg = self.m.zip.make_package(root=self.m.chromium.output_dir,
+                                  output=output_zip_path)
+    pkg.add_file(path=self.m.chromium.output_dir.join(server_name))
+    pkg.zip(step_name='zip chromedriver')
+
+    build_name = 'chromedriver_%s_%s.%s.zip' % (
+        chromedriver_platform, self.get_version(), commit_position)
+    self.m.gsutil.upload(name='archive good build',
+                         source=output_zip_path,
+                         bucket=GS_CONTINUOUS_URL,
+                         dest=build_name,
+                         link_name=build_name)
+
+    latest_build_file = self.m.path.mkdtemp(
+        'latest_build_file_dir').join('latest_build')
+    latest_url = 'latest_%s' % chromedriver_platform
+    self.m.file.write(name='write latest build version',
+                      path='latest_build_file',
+                      data=build_name)
+    self.m.gsutil.upload(name='upload latest build version',
+                         source=latest_build_file,
+                         bucket=GS_CONTINUOUS_URL,
+                         dest=latest_url,
+                         link_name='latest build version')
+
+  def was_released(self, version, chromedriver_platform):
+    """Check if the specified version is released for the given platform."""
+    try:
+      step_result = self.m.gsutil.list(
+          name='list chromedriver versions',
+          url='gs://%s/%s/chromedriver_%s.zip' % (
+              GS_CHROMEDRIVER_BUCKET, version, chromedriver_platform))
+      return True
+    except self.m.step.StepFailure:
+      return False
+
+  def maybe_release(self, chromedriver_platform):
+    """Releases a release candidate if conditions are right."""
+    assert chromedriver_platform != 'android'

[jbudorick, 2016/05/04 18:10:43]

?

+    version = self.get_version()
+    # Check if the current version has already been released.
+    if self.was_released(version, chromedriver_platform):
+      return
+    # Fetch Android test results.
+    android_test_results = self.get_test_results_log('android')
+    # Fetch release candidates.
+    step_result = self.m.gsutil.list(
+        name='list release candidates',
+        url='gs://%s/chromedriver_%s_%s*' % (
+            GS_CONTINUOUS_URL, chromedriver_platform, version),
+        stdout=self.m.raw_io.output(),
+        step_test_data=lambda: self.m.raw_io.test_api.stream_output(
+            'test/dir/chromedriver_linux64_9.99.2222222222.zip',
+            stream='stdout'))
+    output = step_result.stdout
+
+    candidate_pattern = re.compile(r'.*/chromedriver_%s_%s\.(\d+)\.zip$' % (
+        chromedriver_platform, version))
+    candidates = []
+    for line in output.strip().split('\n'):

[jbudorick, 2016/05/04 18:10:42]

splitlines?

+      result = candidate_pattern.match(line)
+      if result:
+        candidates.append(int(result.group(1)))
+
+    # Release the latest candidate build that passed Android, if any.
+    # In this way, if a hot fix is needed, we can delete the release from
+    # the chromedriver bucket instead of bumping up the release version number.
+    candidates.sort(reverse=True)
+    # TODO(mikecase): Add some form of logging to indicate which candidates
+    # failed the tests and which candidates have no test results at all.
+    for commit_position in candidates:
+      android_result = self.commit_position_state(android_test_results,
+                                                  commit_position)
+      if android_result == 'passed':
+        step_result.presentation.step_text = (
+            'Android tests passed at commit position %s' % commit_position)
+        candidate = 'chromedriver_%s_%s.%s.zip' % (
+            chromedriver_platform, version, commit_position)
+        self.release(GS_CONTINUOUS_URL, candidate,
+                     version, chromedriver_platform)
+        break

[jbudorick, 2016/05/04 18:10:43]

break at the end of a function...?

+
+  def release(self, bucket, build, version, chromedriver_platform):
+    """Releases the given candidate build."""
+    release_name = 'chromedriver_%s.zip' % chromedriver_platform
+    temp_dir = self.m.path.mkdtemp('temp_dir')
+    zip_path = temp_dir.join(build)
+    self.m.gsutil.download(name='download release build',
+                           source=build,
+                           bucket=bucket,
+                           dest=zip_path)
+    if self.m.platform.is_linux:
+      unzip_dir = temp_dir.join('unzipped')
+      self.m.zip.unzip(step_name='unzip prebuilt',
+                       zip_file=zip_path,
+                       output=unzip_dir)
+      server_path = unzip_dir.join('chromedriver')
+      zip_path = unzip_dir.join('chromedriver.zip')
+      self.m.step(name='strip', cmd=['strip', server_path])
+
+      pkg = self.m.zip.make_package(root=unzip_dir, output=zip_path)
+      pkg.add_file(path=server_path)
+      pkg.zip(step_name='zip chromedriver')
+
+    self.m.gsutil.upload(
+        name='upload release build',
+        source=zip_path,
+        bucket=GS_CHROMEDRIVER_BUCKET,
+        dest='%s/%s' % (version, release_name))
+
+    self.maybe_upload_release_notes(version)
+    self.maybe_update_latest_release(version)
+
+  def get_web_page_content(self, url, test_data=''):

[jbudorick, 2016/05/04 18:10:43]

What the?

+    """Return the content of the web page specified by the given url."""
+    step_result = self.m.python.inline('get webpage content', program="""
+        import sys
+        import urllib2
+        print urllib2.urlopen(sys.argv[1])
+        """,
+        args=[url],
+        stdout=self.m.raw_io.output(),
+        step_test_data=(lambda: self.m.raw_io.test_api.stream_output(
+            test_data, stream='stdout')))
+    return step_result.stdout
+
+  def maybe_upload_release_notes(self, version):
+    """Upload release notes if conditions are right."""
+    # Check if the current version has already been released.
+    notes_name = 'notes.txt'
+    notes_bucket = GS_CHROMEDRIVER_BUCKET
+    notes_url = '%s/%s' % (version, notes_name)
+    prev_version = '.'.join([version.split('.')[0],
+                             str(int(version.split('.')[1]) - 1)])
+    prev_notes_url = '%s/%s' % (prev_version, notes_name)
+
+    try:
+      step_result = self.m.gsutil.list(
+          name='list version notes',
+          url='gs://%s/%s' % (notes_bucket, notes_url))
+      return
+    except self.m.step.StepFailure:
+      pass
+
+    fixed_issues = []
+    query = ('https://code.google.com/p/chromedriver/issues/csv?'
+             'can=1&q=label%%3AChromeDriver-%s&colspec=ID%%20Summary' % version)
+    issues = self.get_web_page_content(
+        query, test_data=(
+            '"ID","Summary","AllLabels"\n'
+            '"1000","Bug Summary","ChromeDriver-%s, Hotlist-GoodFirstBug"\n'
+            % version))
+    for issue in csv.reader(issues.split('\n')[1:]):
+      if not issue: # pragma: no cover
+        continue
+      else:
+        issue_id = issue[0] 
+        desc = issue[1]
+        labels = issue[2].split(', ')
+        labels.remove('ChromeDriver-%s' % version)
+        if 'Hotlist-GoodFirstBug' in labels:
+          labels.remove('Hotlist-GoodFirstBug')
+        fixed_issues += ['Resolved issue %s: %s [%s]' %
+            (issue_id, desc, labels)]
+
+    temp_notes_file = self.m.path.mkdtemp('temp_notes').join('temp_notes')
+
+    try:
+      self.m.gsutil.download(name='download prev notes',
+                             source=prev_notes_url,
+                             bucket=notes_bucket,
+                             dest=temp_notes_file)
+      old_notes = self.m.file.read(name='read old notes',
+                                   path=temp_notes_file)
+    except self.m.step.StepFailure:
+      old_notes = ''
+
+    new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
+        version, datetime.date.today().isoformat(),
+        'Supports Chrome v%s-%s' % self.get_supported_chrome_versions(),
+        '\n'.join(fixed_issues),
+        old_notes)
+    self.m.file.write(name='write new notes',
+                      path=temp_notes_file,
+                      data=new_notes)
+    self.m.gsutil.upload(name='upload new notes',
+                         source=temp_notes_file,
+                         bucket=notes_bucket,
+                         dest=notes_url)
+
+  def maybe_update_latest_release(self, version):
+    """Update the file LATEST_RELEASE with the latest release version number."""
+    latest_release_filename = 'LATEST_RELEASE'
+    latest_release_url = 'gs://%s/%s' % (
+        GS_CHROMEDRIVER_BUCKET, latest_release_filename)
+
+    # Check if LATEST_RELEASE is up-to-date.
+    step_result = self.m.gsutil.cat(
+        name='check latest release',
+        url=latest_release_url,
+        stdout=self.m.raw_io.output(),
+        step_test_data=lambda: self.m.raw_io.test_api.stream_output(
+            '9.98', stream='stdout'))
+    latest_released_version = step_result.stdout
+
+    if version == latest_released_version:
+      return
+
+    # Check if chromedriver was released on all supported platforms.
+    supported_platforms = ['linux32', 'linux64', 'mac32', 'win32']
+    for platform in supported_platforms:
+      if not self.was_released(version, platform):
+        return # pragma: no cover
+
+    temp_latest_release_filename = self.m.path.mkdtemp(
+        'temp_notes').join('temp_notes')
+    self.m.file.write(name='write latest release number',
+                      path=temp_latest_release_filename,
+                      data=version)
+    self.m.gsutil.upload(name='upload latest release number',
+                         source=temp_latest_release_filename,
+                         bucket=GS_CHROMEDRIVER_BUCKET,
+                         dest=latest_release_filename)
+
+  def add_tools_to_path(self, env):
+    """Add some tools like Ant and Java to PATH for testing steps to use."""
+    paths = []
+    error_message = ''
+    if self.m.platform.is_win:
+      paths = [
+        # Path to Ant and Java, required for the java acceptance tests.
+        'C:\\Program Files (x86)\\Java\\ant\\bin',
+        'C:\\Program Files (x86)\\Java\\jre\\bin',
+      ]
+      error_message = ('Java test steps will fail as expected and '
+                       'they can be ignored.\n'
+                       'Ant, Java or others might not be installed on bot.\n'
+                       'Please refer to page "WATERFALL" on site '
+                       'go/chromedriver.\n'
+                       '\n'
+                       'Missing Directories\n'
+                       '%s')
+    if paths:
+      # Empty step to let us log errors for adding tools to path.
+      step_result = self.m.step('add tools to path', [])
+      missing_directories = []
+      for path in paths:
+        if (not self.m.path.exists(path) or
+            not self.m.file.listdir(name='tools', path=path)):
+          missing_directories.append(path)
+      if missing_directories:
+        step_result.presentation.step_text = (
+            error_message % '\n'.join(missing_directories))
+        return
+
+      env['PATH'] = self.m.path.pathsep.join(
+          [env['PATH']] + paths) # pragma: no cover
+
+  def generate_test_command(self, script, chromedriver, ref_chromedriver=None,
+                            chrome=None, chrome_version=None,
+                            android_package=None, build_type=None,
+                            verbose=None):
+    cmd = [
+        script, 
+        '--chromedriver', chromedriver,
+        '--log-dir', str(self.chromedriver_log_dir)
+    ]
+    if ref_chromedriver:
+      cmd.extend(['--reference-chromedriver', ref_chromedriver])
+    if chrome:
+      cmd.extend(['--chrome', chrome])
+    if chrome_version:
+      cmd.extend(['--chrome-version', chrome_version])
+    if build_type:
+      cmd.extend(['--build-type', build_type])
+    if verbose:
+      cmd.extend(['--verbose'])
+    if android_package:
+      cmd = ['xvfb-run', '-a'] + cmd
+      cmd.extend(['--android-package', android_package])
+    return cmd
+
+  def run_python_tests(self, chromedriver, ref_chromedriver,
+                       chrome=None, chrome_version=None,
+                       chrome_version_name=None, android_package=None,
+                       build_type=None, **kwargs):
+    version_info = ''
+    if chrome_version_name:
+      version_info = '(%s)' % chrome_version_name
+    self.m.step('python_tests%s' % version_info,
+                self.generate_test_command('run_py_tests.py', chromedriver,
+                                           ref_chromedriver=ref_chromedriver,
+                                           chrome=chrome,
+                                           chrome_version=chrome_version,
+                                           android_package=android_package,
+                                           build_type=build_type),
+                **kwargs)
+
+  def run_java_tests(self, chromedriver, chrome=None, chrome_version=None,
+                     chrome_version_name=None, android_package=None,
+                     build_type=None, verbose=False, **kwargs):
+    version_info = ''
+    if chrome_version_name:
+      version_info = '(%s)' % chrome_version_name
+    self.m.step('java_tests%s' % version_info,
+                self.generate_test_command('run_java_tests.py',
+                                           chromedriver,
+                                           ref_chromedriver=None,
+                                           chrome=chrome,
+                                           chrome_version=chrome_version,
+                                           android_package=android_package,
+                                           build_type=build_type,
+                                           verbose=verbose),
+                **kwargs)
+
+  def run_cpp_tests(self, cpp_tests):
+    self.m.step('chromedriver_tests', [cpp_tests])
+
+  def download_chrome(self, version_name, revision, download_site):
+    try:
+      temp_dir = self.m.path.mkdtemp('chrome')
+      return (temp_dir, self.archive.download_chrome(
+          revision, temp_dir, download_site))
+    except self.m.step.StepFailure:
+      return (None, None)
+
+  def kill_chromes(self):
+    chrome_map = {
+        'win': 'chrome.exe',
+        'mac': 'Chromium',
+        'linux': 'chrome',
+    }
+    if self.m.platform.is_win:
+      cmd = ['taskkill', '/F', '/IM']
+    else:
+      cmd = ['killall', '-9']
+    cmd.append(chrome_map[self.m.platform.name])
+    self.m.step('kill chromes', cmd)
+
+  def run_all_tests(self, android_packages=None):
+    exe_postfix = ''

[jbudorick, 2016/05/04 18:10:42]

I think this is cleaner if you set the affected variables directly rather than
messing with a postfix.

+    if self.m.platform.is_win:
+      exe_postfix = '.exe'
+    cpp_tests_name = 'chromedriver_tests' + exe_postfix
+    server_name = 'chromedriver' + exe_postfix
+
+    required_build_outputs = [server_name]
+    if not android_packages:
+      required_build_outputs += [cpp_tests_name]
+    build_type = self.m.path.basename(self.m.chromium.output_dir)
+
+    test_env = {'PATH': '%(PATH)s'}
+    self.add_tools_to_path(test_env)
+
+    chromedriver = self.m.chromium.output_dir.join(server_name)
+    platform_name = self.m.platform.name
+    if self.m.platform.is_linux and self.m.platform.bits == 64:
+      platform_name = 'linux64'
+    ref_chromedriver = self.m.path.join(
+        self.m.path['checkout'],
+        'chrome', 'test', 'chromedriver', 'third_party', 'java_tests',
+        'reference_builds', 'chromedriver_%s%s' % (platform_name, exe_postfix))
+
+    failure = False
+    if android_packages:
+      test_env['PATH'] = self.m.path.pathsep.join([
+          test_env['PATH'],
+          str(self.m.path['checkout'].join(
+              'chrome', 'test', 'chromedriver', 'chrome'))])
+      for package in android_packages:
+        try:
+          with self.m.step.defer_results():
+            self.run_python_tests(chromedriver,
+                                  ref_chromedriver,
+                                  chrome_version_name=package,
+                                  android_package=package,
+                                  build_type=build_type,
+                                  env=test_env)
+            self.run_java_tests(chromedriver,
+                                chrome_version_name=package,
+                                android_package=package,
+                                build_type=build_type,
+                                verbose=True,
+                                env=test_env)
+        except recipe_api.AggregatedStepFailure:
+          failure = True
+    else:
+      latest_snapshot_revision = self.archive.get_latest_snapshot_version()
+
+      versions = [
+          ['HEAD', latest_snapshot_revision],
+          ['47', self.archive.CHROME_47_REVISION],
+          ['46', self.archive.CHROME_46_REVISION],
+          ['45', self.archive.CHROME_45_REVISION],
+      ]
+      for version in versions:
+        download_site = chromium_archive.Url.GS_CHROMIUM_CONTINUOUS_URL
+        version_name = version[0]
+        if version_name == 'HEAD':
+          version_name = version[1]
+          download_site = self.archive.get_snapshot_download_site()
+        temp_dir, chrome_path = self.download_chrome(
+            version_name, version[1], download_site)
+        try:
+          with self.m.step.defer_results():
+            self.run_python_tests(chromedriver,
+                                  ref_chromedriver,
+                                  chrome=chrome_path,
+                                  chrome_version=version[0],
+                                  chrome_version_name='v%s' % version_name,
+                                  build_type=build_type)
+            self.run_java_tests(chromedriver, chrome=chrome_path,
+                                chrome_version=version[0],
+                                chrome_version_name='v%s' % version_name,
+                                build_type=build_type)
+            self.kill_chromes()
+        except recipe_api.AggregatedStepFailure:
+          failure = True
+
+      try:
+        cpp_tests = self.m.chromium.output_dir.join(cpp_tests_name)
+        self.run_cpp_tests(cpp_tests)
+      except recipe_api.StepFailure:
+        failure = True
+    return not failure