Add a virtualenv-based python bootstrapping service to infra.

bootstrap/bootstrap.py

Index: bootstrap/bootstrap.py
diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py
new file mode 100755
index 0000000000000000000000000000000000000000..3bd3950eb4de78dbc28cbdf452dd58111be4c9e2
--- /dev/null
+++ b/bootstrap/bootstrap.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import ast
+import contextlib
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+logging.basicConfig()

[dnj, 2014/07/10 16:06:17]

I like to put the 'basicConfig' and initial 'setLevel' in the "if __name__ ==
'__main__'" section. This allows others to import your module and use its
classes and functions without you force-configuring their logging setup.

Leaving the 'LOGGER =' would be fine in this case as long as you don't _use_ the
until the "basicConfig" is called (which is the case).

[agable, 2014/07/10 17:12:57]

+1. This also lets you only set the loglevel once you actually know what it
should be, rather than setting a default level here and then overwriting it
after parsing command line arguments.

[iannucci, 2014/07/11 02:14:45]

Done.

+LOGGER = logging.getLogger(__name__)
+LOGGER.setLevel(logging.DEBUG)
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

[agable, 2014/07/10 17:12:57]

Comment on the expected path of root (i.e. /path/to/infra).

[iannucci, 2014/07/11 02:14:45]

Done.

+
+BUCKET = 'chrome-python-wheelhouse'
+STORAGE_URL = 'https://www.googleapis.com/storage/v1/b/{}/o'.format(BUCKET)
+OBJECT_URL = 'https://storage.googleapis.com/{}/{{}}#md5={{}}'.format(BUCKET)
+
+
+class NoWheelException(Exception):
+  def __init__(self, name, version, build, source_sha):
+    super(NoWheelException, self).__init__(
+        'No matching wheel found for (%s==%s-%s_%s)' %

[agable, 2014/07/10 17:12:56]

I think this version-build_sha format (and why is the last one an underscore?)
should be a GLOBAL.

[iannucci, 2014/07/11 02:14:45]

Unfortunately I don't think I can... they're constructed differently (on
purpose) in all places.

+        (name, version, build, source_sha))
+
+
+def ls(prefix):
+  from pip._vendor import requests
+  data = requests.get(STORAGE_URL, params=dict(
+      prefix=prefix,
+      fields='items(name,md5Hash)'
+  )).json()
+  return data.get('items', ())
+
+
+def sha_for(deps_entry):
+  if 'rev' in deps_entry:
+    return deps_entry['rev']
+  else:
+    return deps_entry['gs'].split('.')[0]
+
+
+def print_deps(deps, indent=1, with_implicit=True):

[dnj, 2014/07/10 16:06:17]

If you anticipate any reuse, consider allowing a stream to be passed in and
defaulting to 'stdout'.

[agable, 2014/07/10 17:12:56]

If you do this, this might be a good opportunity for "from __future__ import
print_function" too.

[iannucci, 2014/07/11 02:14:45]

¯\_(ツ)_/¯

I'll refactor it if something needs it.

+  for dep, entry in deps.iteritems():
+    if not with_implicit and entry.get('implicit'):
+      continue
+    print '  ' * indent + '%s: %r' % (dep, entry)
+  print
+
+
+def read_deps(path):
+  if os.path.exists(path):
+    with open(path, 'rb') as f:

[dnj, 2014/07/10 16:06:17]

Maybe pass a 'local' boolean that optionally toggles the 'b' flag. For seeded
'deps' you would use binary, but for local 'deps' (manifest) you would use local
encoding to make it easier to read/debug on that platform.

[iannucci, 2014/07/11 02:14:45]

nah, it's just for newlines. It's all single-line text anyway.

+      return ast.literal_eval(f.read())
+
+
+def get_links(deps):
+  import pip.wheel

[Vadim Sh., 2014/07/10 18:18:24]

nit: from pip import wheel

+
+  links = []
+
+  for name, entry in deps.iteritems():
+    version, source_sha = entry['version'] , sha_for(entry)

[Vadim Sh., 2014/07/10 18:18:24]

nit: remove space before ','

+    prefix = 'wheels/{}-{}-{}_{}'.format(name, version, entry['build'],

[iannucci, 2014/07/11 02:14:45]

from vadim: add a platform tag for linux (ubuntu)

e.g. u10.10

+                                         source_sha)
+    link = None
+

[agable, 2014/07/10 17:12:56]

nit: no newline

[iannucci, 2014/07/11 02:14:45]

Done.

+    count = 0
+
+    for entry in ls(prefix):
+      fname = entry['name'].split('/')[-1]
+      md5hash = entry['md5Hash'].decode('base64').encode('hex')
+      wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
+      if not wheel_info:
+        LOGGER.warn('Skipping invalid wheel: %r', fname)
+        continue
+
+      if pip.wheel.Wheel(fname).supported():
+        count += 1
+        if count > 1:
+          LOGGER.error('Found more than one matching wheel? %r', entry)

[agable, 2014/07/10 17:12:56]

... matching wheel for %r.

[iannucci, 2014/07/11 02:14:45]

Done.

+          continue
+        link = OBJECT_URL.format(entry['name'], md5hash)
+
+    if link is None:
+      raise NoWheelException(name, version, entry['build'], source_sha)
+
+    links.append(link)
+
+  return links
+
+
+@contextlib.contextmanager
+def html_index(links):
+  tf = tempfile.mktemp('.html')

[dnj, 2014/07/10 16:06:17]

Consider using 'tempfile.NamedTemporaryFile', which can autodelete on 'close()',
giving you a more file-like API.

[iannucci, 2014/07/11 02:14:45]

Yep, doesn't work on winders b/c the file can't be held open by more than one
process. wah-wah.

+  try:
+    with open(tf, 'w') as f:
+      print >> f, '<html><body>'
+      for link in links:
+        print >> f, '<a href="%s">wat</a>' % link

[dnj, 2014/07/10 16:06:17]

Is "wat" going in the final version? :)

[iannucci, 2014/07/11 02:14:45]

Yep, link text is ignored. It's dumb, dumb format. :p

[agable, 2014/07/11 18:38:16]

Maybe include a comment to that effect.

+      print >> f, '</body></html>'
+    yield tf
+  finally:
+    os.unlink(tf)
+
+
+def install(deps):
+  py = os.path.join(sys.prefix, 'bin', 'python')

[Vadim Sh., 2014/07/10 18:18:24]

Does this work on windows? Also why not sys.executable?

+  pip = os.path.join(sys.prefix, 'bin', 'pip')
+
+  links = get_links(deps)
+  with html_index(links) as ipath:
+    requirements = []
+    # TODO(iannucci): Do this as a requirements.txt
+    for name, deps_entry in deps.iteritems():
+      if not deps_entry.get('implicit'):
+        requirements.append('%s==%s' % (name, deps_entry['version']))
+    subprocess.check_call(
+        [py, pip, 'install', '--no-index', '--download-cache',
+         os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements)
+
+
+def activate_env(env, deps):
+  print 'Activating environment: %r' % env

[Vadim Sh., 2014/07/10 18:18:24]

nit: use logging for this? Even logging.debug(...).

+  if isinstance(deps, basestring):

[dnj, 2014/07/10 16:06:17]

Is there a difference between 'basestring' and 'types.StringTypes'? I've always
used the latter for this sort of check.

[agable, 2014/07/10 17:12:57]

Official python docs disagree, sorry :)

[iannucci, 2014/07/11 02:14:45]

Acknowledged.

+    deps = read_deps(deps)
+  assert deps is not None

[dnj, 2014/07/10 16:06:17]

This seems useful to attach an error message to, like:
assert deps is not None, "No 'deps' found at %s" % (deps_path,)

(Of course, you'd have to also define/persist 'deps_path').

[iannucci, 2014/07/11 02:14:45]

Icing for later, this needs to land.

+
+  if hasattr(sys, 'real_prefix'):

[dnj, 2014/07/10 16:06:17]

Do this check before loading 'deps'? Since you short-circuit any usage of 'deps'
that seems reasonable.

[iannucci, 2014/07/11 02:14:45]

Done.

+    LOGGER.error('Already activated environment!')
+    return
+
+  manifest_path = os.path.join(env, 'manifest.pst')
+  cur_deps = read_deps(manifest_path)

[dnj, 2014/07/10 16:06:17]

If you use the local 'read_deps' idea, this would need "local=True" added.

[iannucci, 2014/07/11 02:14:45]

Acknowledged.

+  if cur_deps != deps:
+    print '  Removing old environment: %r' % cur_deps
+    shutil.rmtree(env, ignore_errors=True)

[dnj, 2014/07/10 16:06:17]

You may want to do a check to ensure that none of the doomed modules are
otherwise loaded in 'sys.modules' just in case someone imports those modules
explicitly/elsewhere only to have you explode them here.

[iannucci, 2014/07/11 02:14:45]

I already check at the top that we're not in a virtualenv already ('real_prefix'
is the clue), so they couldn't be, unless someone is mucking with sys.path, in
which case they're asking for it.

+    cur_deps = None
+
+  if cur_deps is None:
+    print '  Building new environment'
+    # Add in bundled virtualenv lib
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv'))
+    import virtualenv  # pylint: disable=F0401
+    virtualenv.create_environment(
+        env, search_dirs=virtualenv.file_search_dirs())
+
+  print '  Activating environment'
+  activate_this = os.path.join(env, 'bin', 'activate_this.py')
+  execfile(activate_this, dict(__file__=activate_this))
+
+  if cur_deps is None:
+    print '  Installing deps'
+    print_deps(deps, indent=2, with_implicit=False)
+    install(deps)
+    with open(manifest_path, 'wb') as f:

[dnj, 2014/07/10 16:06:17]

If you use the local 'read_deps' flag idea, this would become 'w'.

[iannucci, 2014/07/11 02:14:45]

Acknowledged.

+      f.write(repr(deps) + '\n')

[dnj, 2014/07/10 16:06:17]

Any love for 'pprint'? Also could emit the 'vim' line you use in 'deps.pst'.

[iannucci, 2014/07/11 02:14:45]

mer. no one is going to poke at the manifest. can add later.

+
+  print 'Done creating environment'
+
+
+def main(args):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--deps_file',
+                      help='Path to python deps file (default: %(default)s)')
+  parser.add_argument('env_path', nargs='?',
+                      help='Path to place environment (default: %(default)s)',
+                      default='ENV')
+  opts = parser.parse_args(args)
+
+  activate_env(opts.env_path, opts.deps_file or {})
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

[dnj, 2014/07/10 16:06:17]

Logger-friendly version would catch uncaught Exceptions and log them
(LOGGER.exception). Not much of difference, but...

Also consider an explicit return value in the case of an Exception so it's clear
what your script will return if something goes wrong.