Cross-repo recipe package system.

third_party/recipe_engine/unittests/multi_repo_test.py

+#!/usr/bin/env python
+
+import contextlib
+import copy
+import datetime
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+RECIPE_ENGINE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+THIRD_PARTY = os.path.dirname(RECIPE_ENGINE)
+sys.path.insert(0, THIRD_PARTY)
+
+from recipe_engine import package
+
+@contextlib.contextmanager
+def _in_directory(target_dir):
+  old_dir = os.getcwd()
+  os.chdir(target_dir)
+  try:
+    yield
+  finally:
+    os.chdir(old_dir)
+
+
+def _with_updates(inp, updates):
+  if inp is None:
+    return updates
+
+  outp = copy.copy(inp)
+  for k, v in updates.iteritems():
+    if isinstance(v, dict):
+      outp[k] = _with_updates(inp[k], v)
+    else:
+      outp[k] = v
+  return outp
+
+
+def _recstrify(thing):
+  if isinstance(thing, basestring):
+    return str(thing)
+  elif isinstance(thing, dict):
+    out = {}
+    for k,v in thing.iteritems():
+      out[str(k)] = _recstrify(v)
+    return out
+  elif isinstance(thing, list):
+    return map(_recstrify, thing)
+  else:
+    return thing
+
+
+class MultiRepoTest(unittest.TestCase):
+  def _run_cmd(self, cmd, env=None):
+    subprocess.call(cmd, env=env)
+
+  def _create_repo(self, name, spec):
+    repo_dir = os.path.join(self._root_dir, name)
+    os.mkdir(repo_dir)
+    with _in_directory(repo_dir):
+      self._run_cmd(['git', 'init'])
+      with open('recipe_package.pyl', 'w') as fh:
+        json.dump(spec, fh, indent=2, separators=(',', ': '))
+      self._run_cmd(['git', 'add', 'recipe_package.pyl'])
+      self._run_cmd(['git', 'commit', '-m', 'New recipe package'])
+      rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+    return {
+        'root': repo_dir,
+        'revision': rev,
+        'spec': spec,
+    }
+
+  def _commit_in_repo(self, repo, message='Empty commit', commit_date=None):
+    with _in_directory(repo['root']):
+      env = dict(os.environ)
+      if commit_date:
+        env.update({ 'GIT_COMMITTER_DATE': commit_date.isoformat() })
+        env.update({ 'GIT_AUTHOR_DATE': commit_date.isoformat() })
+      self._run_cmd(['git', 'commit', '-a', '--allow-empty', '-m', message],
+                    env=env)
+      rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+    return {
+        'root': repo['root'],
+        'revision': rev,
+        'spec': repo['spec'],
+    }
+
+  def setUp(self):
+    self.maxDiff = None
+
+    self._root_dir = tempfile.mkdtemp()
+    self._recipe_tool = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+        'recipes.py')
+
+  def tearDown(self):
+    shutil.rmtree(self._root_dir)
+
+  def _repo_setup(self, repo_deps):
+    # In order to avoid a topsort, we require that repo names are in
+    # alphebetical dependency order -- i.e. later names depend on earlier
+    # ones.
+    repos = {}
+    for k in sorted(repo_deps):
+      repos[k] = self._create_repo(k, {
+        'api_version': 0,
+        'id': k,
+        'deps': {
+            d: {
+                'repo': repos[d]['root'],
+                'branch': 'master',
+                'revision': repos[d]['revision'],
+                'path': '',
+            } for d in repo_deps[k]
+        },
+      })
+    return repos
+
+  def _run_roll(self, repo, expect_updates, commit=False):
+    with _in_directory(repo['root']):
+      stdout, _ = (
+          subprocess.Popen(['python', self._recipe_tool, 'roll'],
+                           stdout=subprocess.PIPE)
+                    .communicate())
+      if expect_updates:
+        self.assertRegexpMatches(stdout, r'Wrote \S*recipe_package\.pyl')
+      else:
+        self.assertRegexpMatches(stdout, r'No consistent rolls found')
+
+      if commit:
+        assert expect_updates, 'Cannot commit when not expecting updates'
+        git_match = re.search(r'^git commit .*', stdout, re.MULTILINE)
+        self.assertTrue(git_match)
+        git_command = git_match.group(0)
+        subprocess.call(git_command, shell=True)
+        rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+        return {
+            'root': repo['root'],
+            'revision': rev,
+            'spec': repo['spec'],
+        }
+
+
+  def _get_spec(self, repo):
+    with open(os.path.join(repo['root'], 'recipe_package.pyl')) as fp:
+      return _recstrify(json.load(fp))
+
+  def _get_commit_date(self, repo):
+    with _in_directory(repo['root']):
+      commit_date = subprocess.check_output(
+          ['git', 'log', '--format=%cI',
+           '%s~..%s' % (repo['revision'], repo['revision'])]).strip()
+    return package._parse_date(commit_date)
+
+  def test_empty_roll(self):
+    repos = self._repo_setup({
+      'a': [],
+      'b': [ 'a' ],
+    })
+    self._run_roll(repos['b'], expect_updates=False)
+
+  def test_simple_roll(self):
+    repos = self._repo_setup({
+      'a': [],
+      'b': ['a'],
+    })
+    new_a = self._commit_in_repo(repos['a'])
+    self._run_roll(repos['b'], expect_updates=True)
+    self.assertEqual(
+        self._get_spec(repos['b']),
+        _with_updates(repos['b']['spec'], {
+            'deps': {
+                'a': {
+                    'revision': new_a['revision'],
+                },
+            },
+        }))
+    self._run_roll(repos['b'], expect_updates=False)
+
+  def test_indepdendent_roll(self):
+    repos = self._repo_setup({
+        'b': [],
+        'c': [],
+        'd': ['b', 'c'],
+    })
+    new_b = self._commit_in_repo(repos['b'])
+    new_c = self._commit_in_repo(repos['c'])
+    self._run_roll(repos['d'], expect_updates=True)
+    # There is no guarantee on the order the two updates come in.
+    # (Usually we sort by date but these commits are within 1 second)
+    # However after one roll we expect only one of the two updates to
+    # have come in.
+    d_spec = self._get_spec(repos['d'])
+    self.assertTrue(
+         (d_spec['deps']['b']['revision'] == new_b['revision'])
+      != (d_spec['deps']['c']['revision'] == new_c['revision']))
+    self._run_roll(repos['d'], expect_updates=True)
+    self.assertEqual(
+        self._get_spec(repos['d']),
+        _with_updates(repos['d']['spec'], {
+            'deps': {
+                'b': {
+                    'revision': new_b['revision'],
+                },
+                'c': {
+                    'revision': new_c['revision'],
+                },
+            }
+        }))
+    self._run_roll(repos['d'], expect_updates=False)
+
+  def test_dependent_roll(self):
+    repos = self._repo_setup({
+        'a': [],
+        'b': ['a'],
+        'c': ['a'],
+        'd': ['b', 'c'],
+    })
+    new_a = self._commit_in_repo(repos['a'])
+    new_b = self._run_roll(repos['b'], expect_updates=True, commit=True)
+    new_c = self._run_roll(repos['c'], expect_updates=True, commit=True)
+
+    # We only expect one roll here because to roll b without c would
+    # result in an inconsistent revision of a, so we should skip it.
+    self._run_roll(repos['d'], expect_updates=True)
+    d_spec = self._get_spec(repos['d'])
+    self.assertEqual(
+        self._get_spec(repos['d']),
+        _with_updates(repos['d']['spec'], {
+            'deps': {
+                'b': {
+                    'revision': new_b['revision'],
+                },
+                'c': {
+                    'revision': new_c['revision'],
+                },
+            }
+        }))
+    self._run_roll(repos['d'], expect_updates=False)
+
+  def test_roll_coherence(self):
+    repos = self._repo_setup({
+        'a': [],
+        'b': [],
+        'c': ['a','b'],
+        'd': ['a','b'],
+        'e': ['c','d'],
+    })
+    # Add new commits in b and then a at a later time.  Dependent repos
+    # should always roll a first, so we get as many consistent rolls of
+    # e as possible (this is "coherence").
+    base_date = datetime.datetime(2015, 10, 21, 4, 29, 00)
+
+    new_b = self._commit_in_repo(repos['b'], commit_date=base_date)
+    new_a = self._commit_in_repo(
+        repos['a'], commit_date=base_date+datetime.timedelta(seconds=2))
+
+    new_c_with_b_rolled = self._run_roll(
+        repos['c'], expect_updates=True, commit=True)
+    self.assertEqual(
+        self._get_spec(repos['c']),
+        _with_updates(repos['c']['spec'], {
+            'deps': {
+                'b': {
+                    'revision': new_b['revision'],
+                },
+            },
+        }))
+    new_c_with_both_rolled = self._run_roll(
+        repos['c'], expect_updates=True, commit=True)
+    self._run_roll(repos['c'], expect_updates=False)
+
+    new_d_with_b_rolled = self._run_roll(
+        repos['d'], expect_updates=True, commit=True)
+    self.assertEqual(
+        self._get_spec(repos['d']),
+        _with_updates(repos['d']['spec'], {
+            'deps': {
+                'b': {
+                    'revision': new_b['revision'],
+                },
+            },
+        }))
+    new_d_with_both_rolled = self._run_roll(
+        repos['d'], expect_updates=True, commit=True)
+    self._run_roll(repos['d'], expect_updates=False)
+
+    self._run_roll(repos['e'], expect_updates=True)
+    self.assertEqual(
+        self._get_spec(repos['e']),
+        _with_updates(repos['e']['spec'], {
+            'deps': {
+                'c': {
+                    'revision': new_c_with_b_rolled['revision'],
+                },
+                'd': {
+                    'revision': new_d_with_b_rolled['revision'],
+                },
+            },
+        }))
+    self._run_roll(repos['e'], expect_updates=True)
+    self.assertEqual(
+        self._get_spec(repos['e']),
+        _with_updates(repos['e']['spec'], {
+            'deps': {
+                'c': {
+                    'revision': new_c_with_both_rolled['revision'],
+                },
+                'd': {
+                    'revision': new_d_with_both_rolled['revision'],
+                },
+            },
+        }))
+    self._run_roll(repos['e'], expect_updates=False)
+
+if __name__ == '__main__':
+  unittest.main()