Add git/gerrit-on-borg utilities.

testing_support/gerrit_test_case.py

+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Test framework for code that interacts with gerrit.
+
+class GerritTestCase
+--------------------------------------------------------------------------------
+This class initializes and runs an a gerrit instance on localhost.  To use the
+framework, define a class that extends GerritTestCase, and then do standard
+python unittest development as described here:
+
+http://docs.python.org/2.7/library/unittest.html#basic-example
+
+When your test code runs, the framework will:
+
+- Download the latest stable(-ish) binary release of the gerrit code.
+- Start up a live gerrit instance running in a temp directory on the localhost.
+- Set up a single gerrit user account with admin priveleges.
+- Supply credential helpers for interacting with the gerrit instance via http
+  or ssh.
+
+Refer to depot_tools/testing_support/gerrit-init.sh for details about how the
+gerrit instance is set up, and refer to helper methods defined below
+(createProject, cloneProject, uploadChange, etc.) for ways to interact with the
+gerrit instance from your test methods.
+
+
+class RepoTestCase
+--------------------------------------------------------------------------------
+This class extends GerritTestCase, and creates a set of project repositories
+and a manifest repository that can be used in conjunction with the 'repo' tool.
+
+Each test method will initialize and sync a brand-new repo working directory.
+The 'repo' command may be invoked in a subprocess as part of your tests.
+
+One gotcha: 'repo upload' will always attempt to use the ssh interface to talk
+to gerrit.
+"""
+
+import collections
+from cStringIO import StringIO
+import errno
+import httplib
+import json
+import mock

[Vadim Sh., 2013/10/10 21:55:47]

Is it always available? Is it a requirement for gerrit_test_case.py to work?

[szager, 2013/10/10 23:58:07]

Fair point; I got rid of all the mock stuff.

+import netrc
+import os
+import re
+import shutil
+import signal
+import socket
+import stat
+import subprocess
+import sys
+import tempfile
+import unittest
+import urllib
+
+import gerrit_util

[Vadim Sh., 2013/10/10 21:55:47]

I'm assume depot_tools/ is in sys.path? gerrit_test_case.py and gerrit_util.py
are in different directories.

[szager, 2013/10/10 23:58:07]

Just to be safe, derive path to gerrit_util.py from __file__.

+
+
+# When debugging test code, it's sometimes helpful to leave the test gerrit
+# instance intact and running after the test code exits.  Setting TEARDOWN
+# to False will do that.
+TEARDOWN = True

[Vadim Sh., 2013/10/10 21:55:47]

I also find auth.type = DEVELOPMENT_BECOME_ANY_ACCOUNT in gerrit.config very
helpful. It allows to login into Gerrit UI under whatever account I want.

[szager, 2013/10/10 23:58:07]

Added this to gerrit-init.sh

+
+class GerritTestCase(unittest.TestCase):
+  """Test class for tests that interact with a gerrit server.
+
+  The class setup creates and launches a stand-alone gerrit instance running on
+  localhost, for test methods to interact with.  Class teardown stops and
+  deletes the gerrit instance.
+
+  Note that there is a single gerrit instance for ALL test methods in a
+  GerritTestCase sub-class.
+  """
+
+  COMMIT_RE = re.compile(r'^commit ([0-9a-fA-F]{40})$')
+  CHANGEID_RE = re.compile('^\s+Change-Id:\s*(\S+)$')
+  DEVNULL = open(os.devnull, 'w')
+  TEST_USERNAME = 'test-username'
+  TEST_EMAIL = 'test-username@test.org'
+
+  GerritInstance = collections.namedtuple('GerritInstance', [
+      'credential_file',
+      'gerrit_dir',
+      'gerrit_exe',
+      'gerrit_host',
+      'gerrit_pid',
+      'gerrit_url',
+      'git_dir',
+      'git_host',
+      'git_url',
+      'http_port',
+      'netrc_file',
+      'ssh_ident',
+      'ssh_port',
+  ])
+
+  @classmethod
+  def check_call(cls, *args, **kwargs):
+    kwargs.setdefault('stdout', cls.DEVNULL)
+    kwargs.setdefault('stderr', cls.DEVNULL)
+    subprocess.check_call(*args, **kwargs)
+
+  @classmethod
+  def check_output(cls, *args, **kwargs):
+    kwargs.setdefault('stderr', cls.DEVNULL)
+    return subprocess.check_output(*args, **kwargs)
+
+  @classmethod
+  def _create_gerrit_instance(cls, gerrit_dir):
+    gerrit_init_script = os.path.join(
+        os.path.dirname(__file__), 'gerrit-init.sh')

[Vadim Sh., 2013/10/10 21:55:47]

__file__ can be a path relative to a cwd when python loaded this module. If cwd
changed since then it won't work.

Convert __file__ into absolute path when loading a module, i.e. on top of this
module:

THIS_DIR = os.path.dirname(os.path.abspath(__file__))

[szager, 2013/10/10 23:58:07]

Done.

+    http_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    http_sock.bind(('', 0))
+    http_port = str(http_sock.getsockname()[1])
+    ssh_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    ssh_sock.bind(('', 0))
+    ssh_port = str(ssh_sock.getsockname()[1])
+
+    # NOTE: this is not completely safe.  These port numbers could be
+    # re-assigned by the OS between the calls to socket.close() and gerrit
+    # starting up.  The only safe way to do this would be to pass file
+    # descriptors down to the gerrit process, which is not even remotely
+    # supported.  Alas.
+    http_sock.close()
+    ssh_sock.close()
+    cls.check_call(['bash', gerrit_init_script, '--http-port', http_port,
+                    '--ssh-port', ssh_port, gerrit_dir])
+
+    gerrit_exe = os.path.join(gerrit_dir, 'bin', 'gerrit.sh')
+    cls.check_call(['bash', gerrit_exe, 'start'])
+    with open(os.path.join(gerrit_dir, 'logs', 'gerrit.pid')) as fh:
+      gerrit_pid = int(fh.read().rstrip())
+
+    return cls.GerritInstance(
+        credential_file=os.path.join(gerrit_dir, 'tmp', '.git-credentials'),
+        gerrit_dir=gerrit_dir,
+        gerrit_exe=gerrit_exe,
+        gerrit_host='localhost:%s' % http_port,
+        gerrit_pid=gerrit_pid,
+        gerrit_url='http://localhost:%s' % http_port,
+        git_dir=os.path.join(gerrit_dir, 'git'),
+        git_host='%s/git' % gerrit_dir,
+        git_url='file://%s/git' % gerrit_dir,
+        http_port=http_port,
+        netrc_file=os.path.join(gerrit_dir, 'tmp', '.netrc'),
+        ssh_ident=os.path.join(gerrit_dir, 'tmp', 'id_rsa'),
+        ssh_port=ssh_port,)
+
+  @classmethod
+  def setUpClass(cls):
+    """Sets up the gerrit instances in a class-specific temp dir."""
+    # Create gerrit instance.
+    gerrit_dir = tempfile.mkdtemp()
+    os.chmod(gerrit_dir, 0700)
+    gi = cls.gerrit_instance = cls._create_gerrit_instance(gerrit_dir)
+
+    # Set netrc file for http authentication.
+    cls.netrc_patcher = mock.patch.object(
+        gerrit_util, 'NETRC', netrc.netrc(gi.netrc_file))
+    cls.netrc_patcher.start()
+
+    # gerrit_util only knows about https connections, and that's a good thing.
+    # But for testing, it's much simpler to use http connections.
+    cls.httplib_patcher = mock.patch.object(
+        httplib, 'HTTPSConnection', httplib.HTTPConnection)
+    cls.httplib_patcher.start()
+    cls.protocol_patcher = mock.patch.object(
+        gerrit_util, 'GERRIT_PROTOCOL', 'http')
+    cls.protocol_patcher.start()
+
+    # Because we communicate with the test server via http, rather than https,
+    # libcurl won't add authentication headers to raw git requests unless the
+    # gerrit server returns 401.  That works for pushes, but for read operations
+    # (like git-ls-remote), gerrit will simply omit any ref that requires
+    # authentication.  By default gerrit doesn't permit anonymous read access to
+    # refs/meta/config.  Override that behavior so tests can access
+    # refs/meta/config if necessary.
+    clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'All-Projects')
+    cls._CloneProject('All-Projects', clone_path)
+    project_config = os.path.join(clone_path, 'project.config')
+    cls.check_call(['git', 'config', '--file', project_config, '--add',
+                    'access.refs/meta/config.read', 'group Anonymous Users'])
+    cls.check_call(['git', 'add', project_config], cwd=clone_path)
+    cls.check_call(
+        ['git', 'commit', '-m', 'Anonyous read for refs/meta/config'],
+        cwd=clone_path)
+    cls.check_call(['git', 'push', 'origin', 'HEAD:refs/meta/config'],
+                   cwd=clone_path)
+
+  def setUp(self):
+    self.tempdir = tempfile.mkdtemp()
+    os.chmod(self.tempdir, 0700)
+
+  def tearDown(self):
+    if TEARDOWN:
+      shutil.rmtree(self.tempdir)
+
+  @classmethod
+  def createProject(cls, name, description='Test project', owners=None,
+                    submit_type='CHERRY_PICK'):
+    """Create a project on the test gerrit server."""
+    if owners is None:
+      owners = ['Administrators']
+    body = {
+        'description': description,
+        'submit_type': submit_type,
+        'owners': owners,
+    }
+    path = 'projects/%s' % urllib.quote(name, '')
+    conn = gerrit_util.CreateHttpConn(
+        cls.gerrit_instance.gerrit_host, path, reqtype='PUT', body=body)
+    response = conn.getresponse()
+    assert response.status == 201
+    s = StringIO(response.read())

[Vadim Sh., 2013/10/10 21:55:47]

Can it use ReadHttpJsonResponse?

[szager, 2013/10/10 23:58:07]

The only problem is that ReadHttpJsonResponse demands an http status of 200 (or
404 in some cases).  This is one of those weird cases where the status is code
is something other than 200 on success.

I updated ReadHttpResponse and ReadHttpJsonResponse to add an
'expect_status=200' parameter, to handle this more gracefully.

+    assert s.readline().rstrip() == ")]}'"
+    jmsg = json.load(s)
+    assert jmsg['name'] == name
+
+  @classmethod
+  def _CloneProject(cls, name, path):

[Vadim Sh., 2013/10/10 21:55:47]

I see 3 different naming styles:
1) createProject
2) _CloneProject
3) _stop_gerrit

I personally don't like this. It's probably fine as long as public API is using
same style (it does).

[szager, 2013/10/10 23:58:07]

Yeah, I'm open to suggestions here.  Basically:

_underscore_linux_hacker_style is used for purely internal methods that should
not be public.

_underscoreCamelCase is used for classmethods that shadow an instance method
without reference to self.  These methods could conceivably be called from
elsewhere, but it would be unusual to do so.  I could see changing these to
UpperCamelCase if you think that would be cleaner or clearer.

regularCamelCase is used for public methods that can be invoked via
self.method().  Sometimes, that's a true instance method; other times, it's a
classmethod.

+    """Clone a project from the test gerrit server."""
+    gi = cls.gerrit_instance
+    parent_dir = os.path.dirname(path)
+    if not os.path.exists(parent_dir):
+      os.makedirs(parent_dir)
+    url = '/'.join((gi.gerrit_url, name))
+    cls.check_call(['git', 'clone', url, path])
+
+    # Install commit-msg hook to add Change-Id lines.
+    hook_path = os.path.join(path, '.git', 'hooks', 'commit-msg')
+    cls.check_call(['curl', '-o', hook_path,
+                    '/'.join((gi.gerrit_url, 'tools/hooks/commit-msg'))])
+    os.chmod(hook_path, stat.S_IRWXU)
+
+    config_path = os.path.join(path, '.git', 'config')
+    # Set git identity to test account
+    cls.check_call(['git', 'config', '--file', config_path, 'user.email',
+                    cls.TEST_EMAIL])
+    # Configure non-interactive credentials for git operations.
+    cls.check_call(['git', 'config', '--file', config_path, 'credential.helper',
+                    'store --file=%s' % gi.credential_file])
+    return path
+
+  def cloneProject(self, name, path=None):
+    """Clone a project from the test gerrit server."""
+    if path is None:
+      path = os.path.basename(name)
+      if path.endswith('.git'):
+        path = path[:-4]
+    path = os.path.join(self.tempdir, path)
+    return self._CloneProject(name, path)
+
+  @classmethod
+  def _CreateCommit(cls, clone_path, fn=None, msg=None, text=None):
+    """Create a commit in the given git checkout."""
+    if not fn:
+      fn = 'test-file.txt'
+    if not msg:
+      msg = 'Test Message'
+    if not text:
+      text = 'Another day, another dollar.'
+    fpath = os.path.join(clone_path, fn)
+    with open(fpath, 'a') as fh:
+      fh.write('%s\n' % text)
+    cls.check_call(['git', 'add', fn], cwd=clone_path)
+    cls.check_call(['git', 'commit', '-m', msg], cwd=clone_path)
+    return cls._GetCommit(clone_path)
+
+  def createCommit(self, clone_path, fn=None, msg=None, text=None):
+    """Create a commit in the given git checkout."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    return self._CreateCommit(clone_path, fn, msg, text)
+
+  @classmethod
+  def _GetCommit(cls, clone_path, ref='HEAD'):
+    """Get the sha1 and change-id for a ref in the git checkout."""
+    log_proc = cls.check_output(['git', 'log', '-n', '1', ref], cwd=clone_path)
+    sha1 = None
+    change_id = None
+    for line in log_proc.splitlines():
+

[Vadim Sh., 2013/10/10 21:55:47]

why is this new line here?

[szager, 2013/10/10 23:58:07]

Removed.

+      match = cls.COMMIT_RE.match(line)
+      if match:
+        sha1 = match.group(1)
+        continue
+      match = cls.CHANGEID_RE.match(line)
+      if match:
+        change_id = match.group(1)
+        continue
+    assert sha1
+    assert change_id
+    return (sha1, change_id)
+
+  def getCommit(self, clone_path, ref='HEAD'):
+    """Get the sha1 and change-id for a ref in the git checkout."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    return self._GetCommit(clone_path, ref)
+
+  @classmethod
+  def _UploadChange(cls, clone_path, branch='master', remote='origin'):
+    """Create a gerrit CL from the HEAD of a git checkout."""
+    cls.check_call(
+        ['git', 'push', remote, 'HEAD:refs/for/%s' % branch], cwd=clone_path)
+
+  def uploadChange(self, clone_path, branch='master', remote='origin'):
+    """Create a gerrit CL from the HEAD of a git checkout."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    self._UploadChange(clone_path, branch, remote)
+
+  @classmethod
+  def _PushBranch(cls, clone_path, branch='master'):
+    """Push a branch directly to gerrit, bypassing code review."""
+    cls.check_call(
+        ['git', 'push', 'origin', 'HEAD:refs/heads/%s' % branch],
+        cwd=clone_path)
+
+  def pushBranch(self, clone_path, branch='master'):
+    """Push a branch directly to gerrit, bypassing code review."""
+    clone_path = os.path.join(self.tempdir, clone_path)
+    self._PushBranch(clone_path, branch)
+
+  @classmethod
+  def createAccount(cls, name='Test User', email='test-user@test.org',
+                    password=None, groups=None):
+    """Create a new user account on gerrit."""
+    username = email.partition('@')[0]
+    gerrit_cmd = 'gerrit create-account %s --full-name "%s" --email %s' % (
+        username, name, email)
+    if password:
+      gerrit_cmd += ' --http-password "%s"' % password
+    if groups:
+      gerrit_cmd += ' '.join(['--group %s' % x for x in groups])
+    ssh_cmd = ['ssh', '-p', cls.gerrit_instance.ssh_port,
+           '-i', cls.gerrit_instance.ssh_ident,
+           '-o', 'NoHostAuthenticationForLocalhost=yes',
+           '-o', 'StrictHostKeyChecking=no',
+           '%s@localhost' % cls.TEST_USERNAME, gerrit_cmd]
+    cls.check_call(ssh_cmd)
+
+  @classmethod
+  def _stop_gerrit(cls, gerrit_instance):
+    """Stops the running gerrit instance and deletes it."""
+    try:
+      # This should terminate the gerrit process.
+      cls.check_call(['bash', gerrit_instance.gerrit_exe, 'stop'])
+    finally:
+      try:
+        # cls.gerrit_pid should have already terminated.  If it did, then
+        # os.waitpid will raise OSError.
+        os.waitpid(gerrit_instance.gerrit_pid, os.WNOHANG)
+      except OSError as e:
+        if e.errno == errno.ECHILD:
+          # If gerrit shut down cleanly, os.waitpid will land here.
+          # pylint: disable=W0150
+          return
+
+      # If we get here, the gerrit process is still alive.  Send the process
+      # SIGKILL for good measure.
+      try:
+        os.kill(gerrit_instance.gerrit_pid, signal.SIGKILL)
+      except OSError:
+        if e.errno == errno.ESRCH:
+          # os.kill raised an error because the process doesn't exist.  Maybe
+          # gerrit shut down cleanly after all.
+          # pylint: disable=W0150
+          return
+
+      # Announce that gerrit didn't shut down cleanly.
+      msg = 'Test gerrit server (pid=%d) did not shut down cleanly.' % (
+          gerrit_instance.gerrit_pid)
+      print >> sys.stderr, msg
+
+  @classmethod
+  def tearDownClass(cls):
+    cls.httplib_patcher.stop()
+    cls.protocol_patcher.stop()
+    if TEARDOWN:
+      cls._stop_gerrit(cls.gerrit_instance)
+      shutil.rmtree(cls.gerrit_instance.gerrit_dir)
+
+
+class RepoTestCase(GerritTestCase):
+  """Test class which runs in a repo checkout."""
+
+  REPO_URL = 'https://chromium.googlesource.com/external/repo'
+  MANIFEST_PROJECT = 'remotepath/manifest'
+  MANIFEST_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <remote name="remote1"
+          fetch="%(gerrit_url)s"
+          review="%(gerrit_host)s" />
+  <remote name="remote2"
+          fetch="%(gerrit_url)s"
+          review="%(gerrit_host)s" />
+  <default revision="refs/heads/master" remote="remote1" sync-j="1" />
+  <project remote="remote1" path="localpath/testproj1" name="remotepath/testproj1" />
+  <project remote="remote1" path="localpath/testproj2" name="remotepath/testproj2" />
+  <project remote="remote2" path="localpath/testproj3" name="remotepath/testproj3" />
+  <project remote="remote2" path="localpath/testproj4" name="remotepath/testproj4" />
+</manifest>
+"""
+
+  @classmethod
+  def setUpClass(cls):
+    GerritTestCase.setUpClass()
+    gi = cls.gerrit_instance
+
+    # Create local mirror of repo tool repository.
+    mirror_path = os.path.join(gi.git_dir, 'repo.git')
+    cls.check_call(

[Vadim Sh., 2013/10/10 21:55:47]

Are we OK with touching the network?

We can probably find repo repository locally in .repo directory.

[szager, 2013/10/10 23:58:07]

I also prefer that unit tests don't touch the network.  However, we've already
decided that gerrit-init.sh will download gerrit, so that principle is already
abandoned.

There is no reason to assume that this test code will be invoked from an
existing repo working directory.  Indeed, it's probably better if it's *not*
called from an existing repo checkout.

This could be improved very slightly by first invoking depot_tools/repo
(because, of course, depot_tools must be present); and then mirroring the
REPO_URL defined there.  But it hardly seems worth the effort to do that.

+        ['git', 'clone', '--mirror', cls.REPO_URL, mirror_path])
+
+    # Check out the top-level repo script; it will be used for invocation.
+    repo_clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'repo')
+    cls.check_call(['git', 'clone', '-n', cls.REPO_URL, repo_clone_path])

[Vadim Sh., 2013/10/10 21:55:47]

Maybe clone from mirror created above?

[szager, 2013/10/10 23:58:07]

Ah yes, that was the intention.  Fixed.

+    cls.check_call(
+        ['git', 'checkout', 'origin/stable', 'repo'], cwd=repo_clone_path)
+    shutil.rmtree(os.path.join(repo_clone_path, '.git'))
+    cls.repo_exe = os.path.join(repo_clone_path, 'repo')
+
+    # Create manifest repository.
+    cls.createProject(cls.MANIFEST_PROJECT)
+    clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'manifest')
+    cls._CloneProject(cls.MANIFEST_PROJECT, clone_path)
+    manifest_path = os.path.join(clone_path, 'default.xml')
+    with open(manifest_path, 'w') as fh:
+      fh.write(cls.MANIFEST_TEMPLATE % gi.__dict__)
+    cls.check_call(['git', 'add', 'default.xml'], cwd=clone_path)
+    cls.check_call(['git', 'commit', '-m', 'Test manifest.'], cwd=clone_path)
+    cls._PushBranch(clone_path)
+
+    # Create project repositories.
+    for i in xrange(1, 5):
+      proj = 'testproj%d' % i
+      cls.createProject('remotepath/%s' % proj)
+      clone_path = os.path.join(gi.gerrit_dir, 'tmp', proj)
+      cls._CloneProject('remotepath/%s' % proj, clone_path)
+      cls._CreateCommit(clone_path)
+      cls._PushBranch(clone_path, 'master')
+
+  def setUp(self):
+    super(RepoTestCase, self).setUp()
+    manifest_url = '/'.join((self.gerrit_instance.gerrit_url,
+                            self.MANIFEST_PROJECT))
+    repo_url = '/'.join((self.gerrit_instance.gerrit_url, 'repo'))
+    self.check_call(
+        [self.repo_exe, 'init', '-u', manifest_url, '--repo-url',
+         repo_url, '--no-repo-verify'], cwd=self.tempdir)
+    self.check_call([self.repo_exe, 'sync'], cwd=self.tempdir)
+    for i in xrange(1, 5):
+      # Do the extra bookkeeping that _CloneProject does.
+      proj = 'testproj%d' % i

[Vadim Sh., 2013/10/10 21:55:47]

Can this bookkeeping be extracted into separate function used by both
_CloneProject and this code?

[szager, 2013/10/10 23:58:07]

Done.

+      config_path = os.path.join(
+          self.tempdir, 'localpath', proj, '.git', 'config')
+      self.check_call(['git', 'config', '--file', config_path, 'user.email',
+                    self.TEST_EMAIL])
+      self.check_call(
+          ['git', 'config', '--file', config_path, 'credential.helper',
+           'store --file=%s' % self.gerrit_instance.credential_file])
+      # Tell 'repo upload' to upload this project without prompting.
+      self.check_call(
+          ['git', 'config', '--file', config_path, 'review.%s.upload' %
+           self.gerrit_instance.gerrit_host, 'true'])
+
+  @classmethod
+  def runRepo(cls, *args, **kwargs):
+    # Unfortunately, munging $HOME appears to be the only way to control the
+    # netrc file used by repo.
+    munged_home = os.path.join(cls.gerrit_instance.gerrit_dir, 'tmp')
+    if 'env' not in kwargs:
+      env = kwargs['env'] = os.environ.copy()
+      env['HOME'] = munged_home
+    else:
+      env.setdefault('HOME', munged_home)
+    args[0].insert(0, cls.repo_exe)
+    cls.check_call(*args, **kwargs)
+
+  def uploadChange(self, clone_path, branch='master', remote='origin'):
+    review_host = self.check_output(
+        ['git', 'config', 'remote.%s.review' % remote],
+        cwd=clone_path).strip()
+    assert(review_host)
+    projectname = self.check_output(
+        ['git', 'config', 'remote.%s.projectname' % remote],
+        cwd=clone_path).strip()
+    assert(projectname)
+    GerritTestCase._UploadChange(
+        clone_path, branch=branch, remote='%s://%s/%s' % (
+            gerrit_util.GERRIT_PROTOCOL, review_host, projectname))