[chromedriver] Update run_buildbot_steps.py to work with git hashes

chrome/test/chromedriver/run_buildbot_steps.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all the buildbot steps for ChromeDriver except for update/compile."""
 
 import bisect
 import csv
 import datetime
@@ skipping 13 matching lines @@
 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
 GS_CHROMEDRIVER_BUCKET = 'gs://chromedriver'
 GS_CHROMEDRIVER_DATA_BUCKET = 'gs://chromedriver-data'
 GS_CHROMEDRIVER_RELEASE_URL = 'http://chromedriver.storage.googleapis.com'
 GS_CONTINUOUS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/continuous'
 GS_PREBUILTS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/prebuilts'
 GS_SERVER_LOGS_URL = GS_CHROMEDRIVER_DATA_BUCKET + '/server_logs'
 SERVER_LOGS_LINK = (
     'http://chromedriver-data.storage.googleapis.com/server_logs')
 TEST_LOG_FORMAT = '%s_log.json'
-GS_GITHASH_TO_SVN_URL = (
+GS_GIT_LOG_URL = (
     'https://chromium.googlesource.com/chromium/src/+/%s?format=json')
 GS_SEARCH_PATTERN = (
-    r'.*git-svn-id: svn://svn.chromium.org/chrome/trunk/src@(\d+) ')
+    r'Cr-Commit-Position: refs/heads/master@{#(\d+)}')
+CR_REV_URL = 'https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/%s'
 
 SCRIPT_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir, os.pardir,
                           os.pardir, os.pardir, os.pardir, 'scripts')
 SITE_CONFIG_DIR = os.path.join(_THIS_DIR, os.pardir, os.pardir, os.pardir,
                                os.pardir, os.pardir, os.pardir, os.pardir,
                                'site_config')
 sys.path.append(SCRIPT_DIR)
 sys.path.append(SITE_CONFIG_DIR)
 
 import archive
@@ skipping 329 matching lines @@
   for file_name in os.listdir(tmp_dir):
     file_path = os.path.join(tmp_dir, file_name)
     if os.path.isdir(file_path):
       print 'deleting sub-directory', file_path
       shutil.rmtree(file_path, True)
     if file_name.startswith('chromedriver_'):
       print 'deleting file', file_path
       os.remove(file_path)
 
 
-def _GetSVNRevisionFromGitHash(snapshot_hashcode):
-  json_url = GS_GITHASH_TO_SVN_URL % snapshot_hashcode
+def _GetCommitPositionFromGitHash(snapshot_hashcode):
+  json_url = GS_GIT_LOG_URL % snapshot_hashcode
   try:
     response = urllib2.urlopen(json_url)
   except urllib2.HTTPError as error:
     util.PrintAndFlush('HTTP Error %d' % error.getcode())
   except urllib2.URLError as error:
     util.PrintAndFlush('URL Error %s' % error.message)
     return None
   data = json.loads(response.read()[4:])
   if 'message' in data:
     message = data['message'].split('\n')
     message = [line for line in message if line.strip()]
     search_pattern = re.compile(GS_SEARCH_PATTERN)
     result = search_pattern.search(message[len(message)-1])
     if result:
       return result.group(1)
   util.PrintAndFlush('Failed to get svn revision number for %s' %
                      snapshot_hashcode)
   return None
 
 
+def _GetGitHashFromCommitPosition(commit_position):
+  json_url = CR_REV_URL % commit_position
+  try:
+    response = urllib2.urlopen(json_url)
+  except urllib2.HTTPError as error:
+    util.PrintAndFlush('HTTP Error %d' % error.getcode())

[stgao, 2014/08/25 20:41:27]

Why no return?

[samuong, 2014/08/25 20:51:11]

I'm not sure. I was blindly copying and pasting code from
_GetCommitPositionFromGitHash(), which didn't have a return there either. Looks
like this was an existing bug, so I've fixed both functions.

+  except urllib2.URLError as error:
+    util.PrintAndFlush('URL Error %s' % error.message)
+    return None
+  data = json.loads(response.read())
+  if 'git_sha' in data:
+    return data['git_sha']
+  util.PrintAndFlush('Failed to get git hash for %s' % commit_position)
+  return None
+
+
 def _WaitForLatestSnapshot(revision):
   util.MarkBuildStepStart('wait_for_snapshot')
   def _IsRevisionNumber(revision):
     if isinstance(revision, int):
       return True
     else:
       return revision.isdigit()
   while True:
     snapshot_revision = archive.GetLatestSnapshotVersion()
-    if not _IsRevisionNumber(revision):
-      revision = _GetSVNRevisionFromGitHash(revision)
     if not _IsRevisionNumber(snapshot_revision):
-      snapshot_revision = _GetSVNRevisionFromGitHash(snapshot_revision)
+      snapshot_revision = _GetCommitPositionFromGitHash(snapshot_revision)
     if revision is not None and snapshot_revision is not None:
       if int(snapshot_revision) >= int(revision):
         break
       util.PrintAndFlush('Waiting for snapshot >= %s, found %s' %
                          (revision, snapshot_revision))
     time.sleep(60)
   util.PrintAndFlush('Got snapshot revision %s' % snapshot_revision)
 
 
 def _AddToolsToPath(platform_name):
@@ skipping 25 matching lines @@
     os.environ['PATH'] += os.pathsep + os.pathsep.join(paths)
 
 
 def main():
   parser = optparse.OptionParser()
   parser.add_option(
       '', '--android-packages',
       help=('Comma separated list of application package names, '
             'if running tests on Android.'))
   parser.add_option(
-      '-r', '--revision', type='int', help='Chromium revision')
+      '-r', '--revision', help='Chromium revision')
   parser.add_option(
       '', '--update-log', action='store_true',
       help='Update the test results log (only applicable to Android)')
   options, _ = parser.parse_args()
 
   bitness = '32'
   if util.IsLinux() and platform_module.architecture()[0] == '64bit':
     bitness = '64'
   platform = '%s%s' % (util.GetPlatformName(), bitness)
   if options.android_packages:
     platform = 'android'
 
   _CleanTmpDir()
 
+  if not options.revision:
+    commit_position = None

[stgao, 2014/08/25 20:41:27]

In this case, I think we should fail the test. Because we don't know which
commit position we should wait for the snapshot, right?

[samuong, 2014/08/25 20:51:11]

I was trying to maintain the script's original behavior. There's a case where if
the platform is android and --update-log is not passed, we're allowed to have no
--revision.

In this case it will download the prebuilt chromedriver and test it against the
chrome build that is installed on the device (so there's no snapshot to wait
for).

If we're doing a non-android build, there are still checks to make sure that a
revision is supplied.

[stgao, 2014/08/25 23:17:12]

OK. Sound good.

+  elif options.revision.isdigit():
+    commit_position = options.revision
+  else:
+    commit_position = _GetCommitPositionFromGitHash(options.revision)
+
   if platform == 'android':
     if not options.revision and options.update_log:
       parser.error('Must supply a --revision with --update-log')
     _DownloadPrebuilts()
   else:
     if not options.revision:
       parser.error('Must supply a --revision')
     if platform == 'linux64':
-      _ArchivePrebuilts(options.revision)
-    _WaitForLatestSnapshot(options.revision)
+      _ArchivePrebuilts(commit_position)
+    _WaitForLatestSnapshot(commit_position)
 
   _AddToolsToPath(platform)
 
   cmd = [
       sys.executable,
       os.path.join(_THIS_DIR, 'test', 'run_all_tests.py'),
   ]
   if platform == 'android':
     cmd.append('--android-packages=' + options.android_packages)
 
   passed = (util.RunCommand(cmd) == 0)
 
   _ArchiveServerLogs()
 
   if platform == 'android':
     if options.update_log:
       util.MarkBuildStepStart('update test result log')
-      _UpdateTestResultsLog(platform, options.revision, passed)
+      _UpdateTestResultsLog(platform, commit_position, passed)
   elif passed:
-    _ArchiveGoodBuild(platform, options.revision)
+    _ArchiveGoodBuild(platform, commit_position)
     _MaybeRelease(platform)
 
   if not passed:
     # Make sure the build is red if there is some uncaught exception during
     # running run_all_tests.py.
     util.MarkBuildStepStart('run_all_tests.py')
     util.MarkBuildStepError()
 
   # Add a "cleanup" step so that errors from runtest.py or bb_device_steps.py
   # (which invoke this script) are kept in thier own build step.
   util.MarkBuildStepStart('cleanup')
 
 
 if __name__ == '__main__':
   main()