Add a virtualenv-based python bootstrapping service to infra.

bootstrap/build_deps.py

Index: bootstrap/build_deps.py
diff --git a/bootstrap/build_deps.py b/bootstrap/build_deps.py
new file mode 100755
index 0000000000000000000000000000000000000000..1ded3d5508130f2522d54f247cfb64902df68ef5
--- /dev/null
+++ b/bootstrap/build_deps.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import hashlib
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+import bootstrap
+
+
+GIT_REPO = 'git+https://chromium.googlesource.com/infra/third_party/{}'
+SOURCE_URL = 'gs://{}/sources/{{}}'.format(bootstrap.BUCKET)
+WHEELS_URL = 'gs://{}/wheels/'.format(bootstrap.BUCKET)
+
+MISSING_REPOS = {
+    'coverage': 'git+https://github.com/nedbat/coveragepy/',

[agable, 2014/07/10 17:12:57]

Why aren't we mirroring this yet? We should mirror this.

[iannucci, 2014/07/11 02:14:46]

Filed a bug: crbug.com/393015

+    'python_dateutil':
+      'git+https://chromium.googlesource.com/infra/third_party/dateutil/'

[agable, 2014/07/10 17:12:57]

Is this only 'missing' because its name is dateutil instead of python_dateutil?
That's kinda silly. We can rename the repository. Or rename the dependency?

[iannucci, 2014/07/11 02:14:46]

Rename the repo. The name should match what's in setup.py for that package (e.g.
the wheel name). I mention this in crbug.com/393015 :)

+}
+
+
+def has_custom_build(name):
+  return os.path.exists(os.path.join('custom_builds', name + '.py'))
+
+
+def pip(*args, **kwargs):
+  subprocess.check_call((os.path.join(sys.prefix, 'bin', 'pip'),) + args,
+                        **kwargs)
+
+
+def wheel(arg, source_sha, build):
+  tdir = tempfile.mkdtemp()
+  try:
+    pip('wheel', '--no-index', '--no-deps', '--wheel-dir', tdir, arg)
+    grab_wheel(tdir, 'wheelhouse', source_sha, build)
+  finally:
+    shutil.rmtree(tdir)
+
+
+def grab_wheel(src, dst, source_sha, build):
+  # late import lets us grab the virtualenv pip
+  from pip.wheel import Wheel
+
+  items = os.listdir(src)
+  assert len(items) == 1, 'Wrong number of files in wheel directory: %r' % items
+
+  wheelfile = items[0]
+  wheel_info = Wheel.wheel_file_re.match(wheelfile)
+
+  assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile
+
+  os.rename(os.path.join(src, wheelfile),
+            os.path.join(dst, '%s-%s_%s%s' % (
+                wheel_info.group('namever'),
+                build,
+                source_sha,
+                wheel_info.group(4)
+            )))
+
+
+def run_custom_build(name, link, sha, build):
+  from pip.index import Link
+  from pip.download import unpack_file_url, unpack_vcs_link, is_vcs_url
+
+  assert has_custom_build(name)
+
+  link = Link(link, trusted=True)
+  unpack = unpack_vcs_link if is_vcs_url(link) else unpack_file_url
+
+  tmpd = tempfile.mkdtemp()
+  wheeld = tempfile.mkdtemp()

[dnj, 2014/07/10 16:06:18]

This could fail after 'tmpd'. Since you have dual-cleanup, consider "tmpd =
wheeld = None", running 'mkdtemp' inside the 'try', and preconditioning the
'rmtree' on "if *blah*:".

[iannucci, 2014/07/11 02:14:46]

Did it moar betterz

+  try:
+    unpack(link, tmpd)
+    m = getattr(__import__('custom_builds.%s' % name), name)

[dnj, 2014/07/10 16:06:18]

'custom_builds.%s' % (name,)

[iannucci, 2014/07/11 02:14:46]

Right. Done.

+    m.Build(tmpd, wheeld)
+    grab_wheel(wheeld, 'wheelhouse', sha, build)
+  finally:
+    shutil.rmtree(tmpd)
+    shutil.rmtree(wheeld)
+
+
+def process_git(name, rev, build):
+  print
+  print 'Processing (git)', name, rev
+
+  if name not in MISSING_REPOS:
+    url = GIT_REPO.format(name)
+  else:
+    url = MISSING_REPOS[name]
+
+  url += '@' + rev
+
+  if not has_custom_build(name):
+    wheel(url, rev, build)
+  else:
+    run_custom_build(name, url, rev, build)
+
+
+def process_gs(name, sha_ext, build):
+  print
+  print 'Processing (gs)', name, sha_ext
+
+  sha, ext = sha_ext.split('.', 1)
+  tmp = tempfile.mktemp(ext)

[dnj, 2014/07/10 16:06:18]

Consider 'with tempfile.NamedTemporaryFile(mode='wb', suffix=ext) as tmp'; then
you can avoid the try/finally cleanup. You can also use "gsutil cat" w/
stdout=tmp to eliminate a race condition and also be more efficient.

[iannucci, 2014/07/11 02:14:46]

Nop. Winders. Made more better tho. 

Don't want to cp this into python memory. There's no point.

+  link = 'file://' + tmp
+  try:
+    subprocess.check_call(['gsutil', 'cp', SOURCE_URL.format(sha_ext), tmp])
+    with open(tmp, 'rb') as f:
+      assert hashlib.sha1(f.read()).hexdigest() == sha
+    if not has_custom_build(name):
+      wheel(link, sha, build)
+    else:
+      run_custom_build(name, link, sha, build)
+  finally:
+    try:
+      os.unlink(tmp)
+    except OSError:
+      pass
+
+
+def push_wheelhouse():
+  return subprocess.call(['gsutil', '-m', 'cp', 'wheelhouse/*', WHEELS_URL])
+
+
+def main(to_build):
+  os.chdir(os.path.dirname(__file__))

[dnj, 2014/07/10 16:06:18]

I'm generally avoid 'chdir' when possible. I understand this is a focused
application w/ known internal imports and stuffs so it's not so so bad, but for
the record it would be generically safer to construct relative/absolute paths
and pass "cwd" into subprocesses when needed.

[iannucci, 2014/07/11 02:14:46]

In general, I agree.

So I fixed it. :P

+  build_env = os.path.abspath('BUILD_ENV')
+
+  print 'Parsing deps.pst'
+  deps = bootstrap.read_deps('deps.pst')
+  bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')})
+
+  shutil.rmtree('wheelhouse', ignore_errors=True)
+  try:
+    os.makedirs('wheelhouse')
+  except OSError:
+    pass
+
+  print 'Finding missing deps'
+  missing_deps = {}
+  for name, entry in deps.iteritems():
+    if to_build and name not in to_build:

[dnj, 2014/07/10 16:06:18]

It would be worth doing an "argparse" w/ "to_build" as a nargs='*' argument IMO.

[iannucci, 2014/07/11 02:14:46]

Fiiiiiine.... Done.

+      continue
+    try:
+      bootstrap.get_links({name: entry})
+    except bootstrap.NoWheelException:
+      missing_deps[name] = entry
+
+  if not missing_deps:
+    print 'Nothing to process'
+    return
+
+  print 'Processing deps:'
+  bootstrap.print_deps(missing_deps)
+
+  for name, options in missing_deps.iteritems():
+    # TODO(iannucci):  skip entries which already exist in gs
+    if 'rev' in options:
+      process_git(name, options['rev'], options['build'])
+    elif 'gs' in options:
+      process_gs(name, options['gs'], options['build'])
+    else:
+      raise Exception('Invalid options %r for %r' % (options, name))
+
+  return push_wheelhouse()
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

[dnj, 2014/07/10 16:06:18]

If you are doing top-level logging initialization (see bootstrap.py), you would
need to 'basicConfig' here, too.