Add git/gerrit-on-borg utilities.

testing_support/gerrit_test_case.py

+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Test framework for code that interacts with gerrit.
+
+class GerritTestCase
+--------------------------------------------------------------------------------
+This class initializes and runs an a gerrit instance on localhost.  To use the
+framework, define a class that extends GerritTestCase, and then do standard
+python unittest development as described here:
+
+http://docs.python.org/2.7/library/unittest.html#basic-example
+
+When your test code runs, the framework will:
+
+- Download the latest stable(-ish) binary release of the gerrit code.
+- Start up a live gerrit instance running in a temp directory on the localhost.
+- Set up a single gerrit user account with admin priveleges.
+- Supply credential helpers for interacting with the gerrit instance via http
+  or ssh.
+
+Refer to depot_tools/testing_support/gerrit-init.sh for details about how the
+gerrit instance is set up, and refer to helper methods defined below
+(createProject, cloneProject, uploadChange, etc.) for ways to interact with the
+gerrit instance from your test methods.
+
+
+class RepoTestCase
+--------------------------------------------------------------------------------
+This class extends GerritTestCase, and creates a set of project repositories
+and a manifest repository that can be used in conjunction with the 'repo' tool.
+
+Each test method will initialize and sync a brand-new repo working directory.
+The 'repo' command may be invoked in a subprocess as part of your tests.
+
+One gotcha: 'repo upload' will always attempt to use the ssh interface to talk
+to gerrit.
+"""
+
+import collections
+from cStringIO import StringIO
+import errno
+import httplib
+import imp
+import json
+import netrc
+import os
+import re
+import shutil
+import signal
+import socket
+import stat
+import subprocess
+import sys
+import tempfile
+import unittest
+import urllib
+
+DEPOT_TOOLS_DIR = os.path.normpath(os.path.join(
+    os.path.realpath(__file__), '..', '..'))
+
+gerrit_util = None
+def _import_gerrit_util():

[Vadim Sh., 2013/10/11 18:25:39]

It's too complicated :( 

I looked at existing files in testing_support/, they already assume depot_tools
is in path (this one for example:
http://src.chromium.org/viewvc/chrome/trunk/tools/depot_tools/testing_support...)

Also imp.load_module reloads a module if it was already loaded. It may be
undesirable (if module's globals were modified for instance).

[szager, 2013/10/11 18:48:37]

OK, back to plan A then.

+  fp, pathname, description = imp.find_module('gerrit_util', [DEPOT_TOOLS_DIR])
+  try:
+    global gerrit_util
+    gerrit_util = imp.load_module('gerrit_util', fp, pathname, description)
+  finally:
+    if fp:
+      fp.close()
+_import_gerrit_util()
+del _import_gerrit_util
+
+
+# When debugging test code, it's sometimes helpful to leave the test gerrit
+# instance intact and running after the test code exits.  Setting TEARDOWN
+# to False will do that.
+TEARDOWN = True
+
+class GerritTestCase(unittest.TestCase):
+  """Test class for tests that interact with a gerrit server.
+
+  The class setup creates and launches a stand-alone gerrit instance running on
+  localhost, for test methods to interact with.  Class teardown stops and
+  deletes the gerrit instance.
+
+  Note that there is a single gerrit instance for ALL test methods in a
+  GerritTestCase sub-class.
+  """
+
+  COMMIT_RE = re.compile(r'^commit ([0-9a-fA-F]{40})$')
+  CHANGEID_RE = re.compile('^\s+Change-Id:\s*(\S+)$')
+  DEVNULL = open(os.devnull, 'w')
+  TEST_USERNAME = 'test-username'
+  TEST_EMAIL = 'test-username@test.org'
+
+  GerritInstance = collections.namedtuple('GerritInstance', [
+      'credential_file',
+      'gerrit_dir',
+      'gerrit_exe',
+      'gerrit_host',
+      'gerrit_pid',
+      'gerrit_url',
+      'git_dir',
+      'git_host',
+      'git_url',
+      'http_port',
+      'netrc_file',
+      'ssh_ident',
+      'ssh_port',
+  ])
+
+  @classmethod
+  def getproto(cls):

[Vadim Sh., 2013/10/11 18:25:39]

It's not used.

[szager, 2013/10/11 18:48:37]

Whoops, I put that in while testing the import code that I just ripped out.

+    return gerrit_util.GERRIT_PROTOCOL
+
+  @classmethod
+  def check_call(cls, *args, **kwargs):
+    kwargs.setdefault('stdout', cls.DEVNULL)
+    kwargs.setdefault('stderr', cls.DEVNULL)
+    subprocess.check_call(*args, **kwargs)
+
+  @classmethod
+  def check_output(cls, *args, **kwargs):
+    kwargs.setdefault('stderr', cls.DEVNULL)
+    return subprocess.check_output(*args, **kwargs)
+
+  @classmethod
+  def _create_gerrit_instance(cls, gerrit_dir):
+    gerrit_init_script = os.path.join(
+        DEPOT_TOOLS_DIR, 'testing_support', 'gerrit-init.sh')
+    http_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    http_sock.bind(('', 0))
+    http_port = str(http_sock.getsockname()[1])
+    ssh_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    ssh_sock.bind(('', 0))
+    ssh_port = str(ssh_sock.getsockname()[1])
+
+    # NOTE: this is not completely safe.  These port numbers could be
+    # re-assigned by the OS between the calls to socket.close() and gerrit
+    # starting up.  The only safe way to do this would be to pass file
+    # descriptors down to the gerrit process, which is not even remotely
+    # supported.  Alas.
+    http_sock.close()
+    ssh_sock.close()
+    cls.check_call(['bash', gerrit_init_script, '--http-port', http_port,
+                    '--ssh-port', ssh_port, gerrit_dir])
+
+    gerrit_exe = os.path.join(gerrit_dir, 'bin', 'gerrit.sh')
+    cls.check_call(['bash', gerrit_exe, 'start'])
+    with open(os.path.join(gerrit_dir, 'logs', 'gerrit.pid')) as fh:
+      gerrit_pid = int(fh.read().rstrip())
+
+    return cls.GerritInstance(
+        credential_file=os.path.join(gerrit_dir, 'tmp', '.git-credentials'),
+        gerrit_dir=gerrit_dir,
+        gerrit_exe=gerrit_exe,
+        gerrit_host='localhost:%s' % http_port,
+        gerrit_pid=gerrit_pid,
+        gerrit_url='http://localhost:%s' % http_port,
+        git_dir=os.path.join(gerrit_dir, 'git'),
+        git_host='%s/git' % gerrit_dir,
+        git_url='file://%s/git' % gerrit_dir,
+        http_port=http_port,
+        netrc_file=os.path.join(gerrit_dir, 'tmp', '.netrc'),
+        ssh_ident=os.path.join(gerrit_dir, 'tmp', 'id_rsa'),
+        ssh_port=ssh_port,)
+
+  @classmethod
+  def setUpClass(cls):
+    """Sets up the gerrit instances in a class-specific temp dir."""
+    # Create gerrit instance.
+    gerrit_dir = tempfile.mkdtemp()
+    os.chmod(gerrit_dir, 0700)
+    gi = cls.gerrit_instance = cls._create_gerrit_instance(gerrit_dir)
+
+    # Set netrc file for http authentication.
+    cls.gerrit_util_netrc_orig = gerrit_util.NETRC
+    gerrit_util.NETRC = netrc.netrc(gi.netrc_file)
+
+    # gerrit_util.py defaults to using https, but for testing, it's much
+    # simpler to use http connections.
+    cls.gerrit_util_protocol_orig = gerrit_util.GERRIT_PROTOCOL
+    gerrit_util.GERRIT_PROTOCOL = 'http'
+
+    # Because we communicate with the test server via http, rather than https,
+    # libcurl won't add authentication headers to raw git requests unless the
+    # gerrit server returns 401.  That works for pushes, but for read operations
+    # (like git-ls-remote), gerrit will simply omit any ref that requires
+    # authentication.  By default gerrit doesn't permit anonymous read access to
+    # refs/meta/config.  Override that behavior so tests can access
+    # refs/meta/config if necessary.
+    clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'All-Projects')
+    cls._CloneProject('All-Projects', clone_path)
+    project_config = os.path.join(clone_path, 'project.config')
+    cls.check_call(['git', 'config', '--file', project_config, '--add',
+                    'access.refs/meta/config.read', 'group Anonymous Users'])
+    cls.check_call(['git', 'add', project_config], cwd=clone_path)
+    cls.check_call(
+        ['git', 'commit', '-m', 'Anonyous read for refs/meta/config'],
+        cwd=clone_path)
+    cls.check_call(['git', 'push', 'origin', 'HEAD:refs/meta/config'],
+                   cwd=clone_path)
+
+  def setUp(self):
+    self.tempdir = tempfile.mkdtemp()
+    os.chmod(self.tempdir, 0700)
+
+  def tearDown(self):
+    if TEARDOWN:
+      shutil.rmtree(self.tempdir)
+
+  @classmethod
+  def createProject(cls, name, description='Test project', owners=None,
+                    submit_type='CHERRY_PICK'):
+    """Create a project on the test gerrit server."""
+    if owners is None:
+      owners = ['Administrators']
+    body = {
+        'description': description,
+        'submit_type': submit_type,
+        'owners': owners,
+    }
+    path = 'projects/%s' % urllib.quote(name, '')
+    conn = gerrit_util.CreateHttpConn(
+        cls.gerrit_instance.gerrit_host, path, reqtype='PUT', body=body)
+    jmsg = gerrit_util.ReadHttpJsonResponse(conn, expect_status=201)
+    assert jmsg['name'] == name
+
+  @classmethod
+  def _post_clone_bookkeeping(cls, clone_path):
+    config_path = os.path.join(clone_path, '.git', 'config')
+    cls.check_call(
+        ['git', 'config', '--file', config_path, 'user.email', cls.TEST_EMAIL])
+    cls.check_call(
+        ['git', 'config', '--file', config_path, 'credential.helper',
+         'store --file=%s' % cls.gerrit_instance.credential_file])
+
+  @classmethod
+  def _CloneProject(cls, name, path):
+    """Clone a project from the test gerrit server."""
+    gi = cls.gerrit_instance
+    parent_dir = os.path.dirname(path)
+    if not os.path.exists(parent_dir):
+      os.makedirs(parent_dir)
+    url = '/'.join((gi.gerrit_url, name))
+    cls.check_call(['git', 'clone', url, path])
+    cls._post_clone_bookkeeping(path)
+    # Install commit-msg hook to add Change-Id lines.
+    hook_path = os.path.join(path, '.git', 'hooks', 'commit-msg')
+    cls.check_call(['curl', '-o', hook_path,
+                    '/'.join((gi.gerrit_url, 'tools/hooks/commit-msg'))])
+    os.chmod(hook_path, stat.S_IRWXU)
+    return path
+
+  def cloneProject(self, name, path=None):
+    """Clone a project from the test gerrit server."""
+    if path is None:
+      path = os.path.basename(name)
+      if path.endswith('.git'):
+        path = path[:-4]
+    path = os.path.join(self.tempdir, path)
+    return self._CloneProject(name, path)
+
+  @classmethod
+  def _CreateCommit(cls, clone_path, fn=None, msg=None, text=None):
+    """Create a commit in the given git checkout."""
+    if not fn:
+      fn = 'test-file.txt'
+    if not msg:
+      msg = 'Test Message'
+    if not text:
+      text = 'Another day, another dollar.'
+    fpath = os.path.join(clone_path, fn)
+    with open(fpath, 'a') as fh:
+      fh.write('%s\n' % text)
+    cls.check_call(['git', 'add', fn], cwd=clone_path)
+    cls.check_call(['git', 'commit', '-m', msg], cwd=clone_path)
+    return cls._GetCommit(clone_path)
+
+  def createCommit(self, clone_path, fn=None, msg=None, text=None):
+    """Create a commit in the given git checkout."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    return self._CreateCommit(clone_path, fn, msg, text)
+
+  @classmethod
+  def _GetCommit(cls, clone_path, ref='HEAD'):
+    """Get the sha1 and change-id for a ref in the git checkout."""
+    log_proc = cls.check_output(['git', 'log', '-n', '1', ref], cwd=clone_path)
+    sha1 = None
+    change_id = None
+    for line in log_proc.splitlines():
+      match = cls.COMMIT_RE.match(line)
+      if match:
+        sha1 = match.group(1)
+        continue
+      match = cls.CHANGEID_RE.match(line)
+      if match:
+        change_id = match.group(1)
+        continue
+    assert sha1
+    assert change_id
+    return (sha1, change_id)
+
+  def getCommit(self, clone_path, ref='HEAD'):
+    """Get the sha1 and change-id for a ref in the git checkout."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    return self._GetCommit(clone_path, ref)
+
+  @classmethod
+  def _UploadChange(cls, clone_path, branch='master', remote='origin'):
+    """Create a gerrit CL from the HEAD of a git checkout."""
+    cls.check_call(
+        ['git', 'push', remote, 'HEAD:refs/for/%s' % branch], cwd=clone_path)
+
+  def uploadChange(self, clone_path, branch='master', remote='origin'):
+    """Create a gerrit CL from the HEAD of a git checkout."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    self._UploadChange(clone_path, branch, remote)
+
+  @classmethod
+  def _PushBranch(cls, clone_path, branch='master'):
+    """Push a branch directly to gerrit, bypassing code review."""
+    cls.check_call(
+        ['git', 'push', 'origin', 'HEAD:refs/heads/%s' % branch],
+        cwd=clone_path)
+
+  def pushBranch(self, clone_path, branch='master'):
+    """Push a branch directly to gerrit, bypassing code review."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    self._PushBranch(clone_path, branch)
+
+  @classmethod
+  def createAccount(cls, name='Test User', email='test-user@test.org',
+                    password=None, groups=None):
+    """Create a new user account on gerrit."""
+    username = email.partition('@')[0]
+    gerrit_cmd = 'gerrit create-account %s --full-name "%s" --email %s' % (
+        username, name, email)
+    if password:
+      gerrit_cmd += ' --http-password "%s"' % password
+    if groups:
+      gerrit_cmd += ' '.join(['--group %s' % x for x in groups])
+    ssh_cmd = ['ssh', '-p', cls.gerrit_instance.ssh_port,
+           '-i', cls.gerrit_instance.ssh_ident,
+           '-o', 'NoHostAuthenticationForLocalhost=yes',
+           '-o', 'StrictHostKeyChecking=no',
+           '%s@localhost' % cls.TEST_USERNAME, gerrit_cmd]
+    cls.check_call(ssh_cmd)
+
+  @classmethod
+  def _stop_gerrit(cls, gerrit_instance):
+    """Stops the running gerrit instance and deletes it."""
+    try:
+      # This should terminate the gerrit process.
+      cls.check_call(['bash', gerrit_instance.gerrit_exe, 'stop'])
+    finally:
+      try:
+        # cls.gerrit_pid should have already terminated.  If it did, then
+        # os.waitpid will raise OSError.
+        os.waitpid(gerrit_instance.gerrit_pid, os.WNOHANG)
+      except OSError as e:
+        if e.errno == errno.ECHILD:
+          # If gerrit shut down cleanly, os.waitpid will land here.
+          # pylint: disable=W0150
+          return
+
+      # If we get here, the gerrit process is still alive.  Send the process
+      # SIGKILL for good measure.
+      try:
+        os.kill(gerrit_instance.gerrit_pid, signal.SIGKILL)
+      except OSError:
+        if e.errno == errno.ESRCH:
+          # os.kill raised an error because the process doesn't exist.  Maybe
+          # gerrit shut down cleanly after all.
+          # pylint: disable=W0150
+          return
+
+      # Announce that gerrit didn't shut down cleanly.
+      msg = 'Test gerrit server (pid=%d) did not shut down cleanly.' % (
+          gerrit_instance.gerrit_pid)
+      print >> sys.stderr, msg
+
+  @classmethod
+  def tearDownClass(cls):
+    gerrit_util.NETRC = cls.gerrit_util_netrc_orig
+    gerrit_util.GERRIT_PROTOCOL = cls.gerrit_util_protocol_orig
+    if TEARDOWN:
+      cls._stop_gerrit(cls.gerrit_instance)
+      shutil.rmtree(cls.gerrit_instance.gerrit_dir)
+
+
+class RepoTestCase(GerritTestCase):
+  """Test class which runs in a repo checkout."""
+
+  REPO_URL = 'https://chromium.googlesource.com/external/repo'
+  MANIFEST_PROJECT = 'remotepath/manifest'
+  MANIFEST_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <remote name="remote1"
+          fetch="%(gerrit_url)s"
+          review="%(gerrit_host)s" />
+  <remote name="remote2"
+          fetch="%(gerrit_url)s"
+          review="%(gerrit_host)s" />
+  <default revision="refs/heads/master" remote="remote1" sync-j="1" />
+  <project remote="remote1" path="localpath/testproj1" name="remotepath/testproj1" />
+  <project remote="remote1" path="localpath/testproj2" name="remotepath/testproj2" />
+  <project remote="remote2" path="localpath/testproj3" name="remotepath/testproj3" />
+  <project remote="remote2" path="localpath/testproj4" name="remotepath/testproj4" />
+</manifest>
+"""
+
+  @classmethod
+  def setUpClass(cls):
+    GerritTestCase.setUpClass()
+    gi = cls.gerrit_instance
+
+    # Create local mirror of repo tool repository.
+    repo_mirror_path = os.path.join(gi.git_dir, 'repo.git')
+    cls.check_call(
+        ['git', 'clone', '--mirror', cls.REPO_URL, repo_mirror_path])
+
+    # Check out the top-level repo script; it will be used for invocation.
+    repo_clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'repo')
+    cls.check_call(['git', 'clone', '-n', repo_mirror_path, repo_clone_path])
+    cls.check_call(
+        ['git', 'checkout', 'origin/stable', 'repo'], cwd=repo_clone_path)
+    shutil.rmtree(os.path.join(repo_clone_path, '.git'))
+    cls.repo_exe = os.path.join(repo_clone_path, 'repo')
+
+    # Create manifest repository.
+    cls.createProject(cls.MANIFEST_PROJECT)
+    clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'manifest')
+    cls._CloneProject(cls.MANIFEST_PROJECT, clone_path)
+    manifest_path = os.path.join(clone_path, 'default.xml')
+    with open(manifest_path, 'w') as fh:
+      fh.write(cls.MANIFEST_TEMPLATE % gi.__dict__)
+    cls.check_call(['git', 'add', 'default.xml'], cwd=clone_path)
+    cls.check_call(['git', 'commit', '-m', 'Test manifest.'], cwd=clone_path)
+    cls._PushBranch(clone_path)
+
+    # Create project repositories.
+    for i in xrange(1, 5):
+      proj = 'testproj%d' % i
+      cls.createProject('remotepath/%s' % proj)
+      clone_path = os.path.join(gi.gerrit_dir, 'tmp', proj)
+      cls._CloneProject('remotepath/%s' % proj, clone_path)
+      cls._CreateCommit(clone_path)
+      cls._PushBranch(clone_path, 'master')
+
+  def setUp(self):
+    super(RepoTestCase, self).setUp()
+    manifest_url = '/'.join((self.gerrit_instance.gerrit_url,
+                            self.MANIFEST_PROJECT))
+    repo_url = '/'.join((self.gerrit_instance.gerrit_url, 'repo'))
+    self.check_call(
+        [self.repo_exe, 'init', '-u', manifest_url, '--repo-url',
+         repo_url, '--no-repo-verify'], cwd=self.tempdir)
+    self.check_call([self.repo_exe, 'sync'], cwd=self.tempdir)
+    for i in xrange(1, 5):
+      clone_path = os.path.join(self.tempdir, 'localpath', 'testproj%d' % i)
+      self._post_clone_bookkeeping(clone_path)
+      # Tell 'repo upload' to upload this project without prompting.
+      config_path = os.path.join(clone_path, '.git', 'config')
+      self.check_call(
+          ['git', 'config', '--file', config_path, 'review.%s.upload' %
+           self.gerrit_instance.gerrit_host, 'true'])
+
+  @classmethod
+  def runRepo(cls, *args, **kwargs):
+    # Unfortunately, munging $HOME appears to be the only way to control the
+    # netrc file used by repo.
+    munged_home = os.path.join(cls.gerrit_instance.gerrit_dir, 'tmp')
+    if 'env' not in kwargs:
+      env = kwargs['env'] = os.environ.copy()
+      env['HOME'] = munged_home
+    else:
+      env.setdefault('HOME', munged_home)
+    args[0].insert(0, cls.repo_exe)
+    cls.check_call(*args, **kwargs)
+
+  def uploadChange(self, clone_path, branch='master', remote='origin'):
+    review_host = self.check_output(
+        ['git', 'config', 'remote.%s.review' % remote],
+        cwd=clone_path).strip()
+    assert(review_host)
+    projectname = self.check_output(
+        ['git', 'config', 'remote.%s.projectname' % remote],
+        cwd=clone_path).strip()
+    assert(projectname)
+    GerritTestCase._UploadChange(
+        clone_path, branch=branch, remote='%s://%s/%s' % (
+            gerrit_util.GERRIT_PROTOCOL, review_host, projectname))