Add CIPD binary installation to DEPS runhooks.

bootstrap/install_cipd_packages.py

+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import platform
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+import urllib
+import urllib2
+
+
+# The path to the "infra/bootstrap/" directory.
+BOOTSTRAP_DIR = os.path.dirname(os.path.abspath(__file__))
+# The path to the "infra/" directory.
+ROOT = os.path.dirname(BOOTSTRAP_DIR)
+# The path where CIPD install lists are stored.
+CIPD_LIST_DIR = os.path.join(BOOTSTRAP_DIR, 'cipd')
+# Default binary install root.
+DEFAULT_INSTALL_ROOT = os.path.join(ROOT, 'bin')
+
+# The default URL of the CIPD backend service.
+CIPD_BACKEND_URL = 'https://chrome-infra-packages.appspot.com'
+
+# Map of CIPD configuration based on the current architecture/platform. If a
+# platform is not listed here, the bootstrap will be a no-op.
+#
+# This is keyed on the platform's (system, machine).
+ARCH_CONFIG_MAP = {
+  ('Linux', 'x86_64'): {
+    'cipd_package': 'infra/tools/cipd/linux-amd64',
+    'cipd_package_version': '99439270562c887c887b7538ed634ed3a3c0dc83',

[Vadim Sh., 2015/09/24 16:50:32]

since you are using resolve_cipd_instance_id you can specify version via tag
here, it is more "readable", e.g.
"git_revision:ed808139130f0ea8fd52fc64fd9ec47d150a2e49"

Such version string will also look the same for clients for other platforms
(since they all are built from same revision).

If you prefer to keep raw instance_id, then document how you got it, e.g. 
"./cipd resolve infra/tools/cipd/linux-amd64
-version=git_revision:ed808139130f0ea8fd52fc64fd9ec47d150a2e49"

Raw instance_id only make sense if you also modify resolve_cipd_instance_id to
return it unchanged without making a backend call (since it "resolves" to
itself). Raw instance_id (and only it) matches '^[0-9a-f]{40}$'. Using raw
instance ID everywhere has an advantage that client will make no backend calls
whatsoever if it has all packages installed already.

[dnj, 2015/09/24 17:19:35]

Ah okay. I'll still use the raw instance ID to avoid the round-trip, but will
add the note and check.

+    'cipd_install_lists': [
+      'cipd_linux_amd64.txt',
+    ],
+  },
+}
+
+
+class CipdError(Exception):
+  """Raised by install_cipd_client on fatal error."""
+
+
+def get_platform_config():
+  key = (platform.system(), platform.machine())
+  return key, ARCH_CONFIG_MAP.get(key)
+
+
+def dump_json(obj):
+  """Pretty-formats object to JSON."""
+  return json.dumps(obj, indent=2, sort_keys=True, separators=(',',':'))
+
+
+def ensure_directory(path):
+  # Ensure the parent directory exists.
+  if os.path.isdir(path):
+    return
+  if os.path.exists(path):
+    raise ValueError("Target file's directory [%s] exists, but is not a "
+                     "directory." % (path,))
+  logging.debug('Creating directory: [%s]', path)
+  os.makedirs(path)
+
+
+def write_file(path, data):

[Vadim Sh., 2015/09/24 16:50:32]

this is Linux only :( On Windows os.rename when 'path' exists will fail.

[dnj, 2015/09/24 17:19:35]

Oh good point. I don't really see a need to make this an atomic write here, so
I'll just go with a regular open in 'rb' mode.

+  """Puts a file on the disk."""
+  dirname = os.path.dirname(path)
+  ensure_directory(dirname)
+
+  # Write the file.
+  temp_file = None
+  try:
+    fd, temp_file = tempfile.mkstemp(dir=dirname)
+    with os.fdopen(fd, 'w') as f:
+      f.write(data)
+    os.rename(temp_file, path)
+    temp_file = None
+  finally:
+    if temp_file is not None:
+      os.unlink(temp_file)
+
+
+def write_tag_file(path, obj):
+  with open(path, 'w') as fd:
+    json.dump(obj, fd, sort_keys=True, indent=2)
+
+
+def read_tag_file(path):
+  try:
+    with open(path, 'r') as fd:
+      return json.load(fd)
+  except (IOError, ValueError):
+    return None
+
+
+def resolve_cipd_instance_id(cipd_backend, package, version):
+  resp = cipd_backend.call_api(
+      'repo/v1/instance/resolve',
+      package_name=package,
+      version=version)
+  return resp['instance_id']
+
+
+def install_cipd_client(cipd_backend, config, root):
+  package = config['cipd_package']
+  instance_id = resolve_cipd_instance_id(
+      cipd_backend,
+      package,
+      config.get('cipd_package_version', 'latest'))
+  logging.info('Installing CIPD client [%s] ID [%s]', package, instance_id)
+
+  # Is this the version that's already installed?
+  cipd_client = CipdClient(cipd_backend, os.path.join(root, 'cipd'))
+  current = cipd_client.read_tag()
+  if current == (package, instance_id) and os.path.isfile(cipd_client.path):
+    logging.info('CIPD client already installed.')
+    return cipd_client
+
+  # Get the client binary URL.
+  client_info = cipd_backend.call_api(
+      'repo/v1/client',
+      package_name=package,
+      instance_id=instance_id)
+  logging.info('CIPD client binary info:\n%s', dump_json(client_info))
+
+  status, raw_client_data = fetch_url(client_info['client_binary']['fetch_url'])
+  if status != 200:
+    logging.error('Failed to fetch CIPD client binary (HTTP status %d)', status)
+    return None
+
+  digest = hashlib.sha1(raw_client_data).hexdigest()
+  if digest != client_info['client_binary']['sha1']:
+    logging.error('CIPD client hash mismatch (%s != %s)', digest,
+                  client_info['client_binary']['sha1'])
+    return None
+
+  write_file(cipd_client.path, raw_client_data)
+  os.chmod(cipd_client.path, 0755)
+  cipd_client.write_tag(package, instance_id)
+  return cipd_client
+
+
+def execute(*cmd):
+  if not logging.getLogger().isEnabledFor(logging.DEBUG):
+    code = subprocess.call(cmd)
+  else:
+    # Execute the process, passing STDOUT/STDERR through our logger.
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    for line in proc.stdout:
+      logging.debug('%s: %s', cmd[0], line.rstrip())
+    code = proc.wait()
+  if code:
+    logging.error('Process failed with exit code: %d', code)
+  return code
+
+
+class CipdBackend(object):
+  def __init__(self, url):
+    self.url = url
+
+  def call_api(self, endpoint, **query):
+    """Sends GET request to CIPD backend, parses JSON response."""
+    url = '%s/_ah/api/%s' % (self.url, endpoint)
+    if query:
+      url += '?' + urllib.urlencode(sorted(query.iteritems()), True)
+    status, body = fetch_url(url)
+    if status != 200:
+      raise CipdError('Server replied with HTTP %d' % status)
+    try:
+      body = json.loads(body)
+    except ValueError:
+      raise CipdError('Server returned invalid JSON')
+    status = body.get('status')
+    if status != 'SUCCESS':
+      m = body.get('error_message') or '<no error message>'
+      raise CipdError('Server replied with error %s: %s' % (status, m))
+    return body
+
+
+class CipdClient(object):
+
+  # Filename for the CIPD package/instance_id tag.
+  _TAG_NAME = '.cipd_client_version'
+
+  def __init__(self, cipd_backend, path):
+    self.cipd_backend = cipd_backend
+    self.path = path
+
+  def exists(self):
+    return os.path.isfile(self.path)
+
+  def ensure(self, list_path, root):
+    assert os.path.isfile(list_path)
+    assert os.path.isdir(root)
+    self.call(
+        'ensure',
+        '-list', list_path,
+        '-root', root,
+        '-service-url', self.cipd_backend.url)
+
+  def call(self, *args):
+    if execute(self.path, *args):
+      raise CipdError('Failed to execute CIPD client: %s', ' '.join(args))
+
+  def write_tag(self, package, instance_id):
+    write_tag_file(self._cipd_tag_path, {
+      'package': package,
+      'instance_id': instance_id,
+    })
+
+  def read_tag(self):
+    tag = read_tag_file(self._cipd_tag_path)
+    if tag is None:
+      return None, None
+    return tag.get('package'), tag.get('instance_id')
+
+  @property
+  def _cipd_tag_path(self):
+    return os.path.join(os.path.dirname(self.path), self._TAG_NAME)
+
+
+def fetch_url(url, headers=None):
+  """Sends GET request (with retries).
+  Args:
+    url: URL to fetch.
+    headers: dict with request headers.
+  Returns:
+    (200, reply body) on success.
+    (HTTP code, None) on HTTP 401, 403, or 404 reply.
+  Raises:
+    Whatever urllib2 raises.
+  """
+  req = urllib2.Request(url)
+  req.add_header('User-Agent', 'infra-install-cipd-packages')
+  for k, v in (headers or {}).iteritems():
+    req.add_header(str(k), str(v))
+  i = 0
+  while True:
+    i += 1
+    try:
+      logging.debug('GET %s', url)
+      return 200, urllib2.urlopen(req, timeout=60).read()

[Vadim Sh., 2015/09/24 16:50:32]

This thing doesn't validate SSL cert :( 

Consider using something like
https://chrome-internal.googlesource.com/infra/infra_internal/+/master/ccompu...

Or moving the bootstrap script to infra.tools.* and using 'requests' library
from venv we have there.

[dnj, 2015/09/24 17:19:35]

grosssssssssssss, forgot about that. Done.

I had considered moving it into a proper tools package, but I was concerned
because some of the other bootstrapping code is required for that package to
work and I didn't want a DEPS file order to be the only thing standing in the
way of the bootstrap breaking.

+    except Exception as e:
+      if isinstance(e, urllib2.HTTPError):
+        logging.error('Failed to fetch %s, server returned HTTP %d', url,
+                      e.code)
+        if e.code in (401, 403, 404):
+          return e.code, None
+      else:
+        logging.exception('Failed to fetch %s', url)
+      if i == 20:
+        raise
+    logging.info('Retrying in %d sec.', i)
+    time.sleep(i)
+
+
+def main(argv):
+  parser = argparse.ArgumentParser('Installs CIPD bootstrap packages.')
+  parser.add_argument('--cipd-backend-url', metavar='URL',
+      default=CIPD_BACKEND_URL,
+      help='Specify the CIPD backend URL (default is %(default)s)')
+  parser.add_argument('-d', '--cipd-root-dir', metavar='PATH',
+      default=DEFAULT_INSTALL_ROOT,
+      help='Specify the root CIPD package installation directory.')
+  parser.add_argument('-v', '--verbose', action='count', default=0,
+      help='Increase logging verbosity. Can be specified multiple times.')
+
+  args = parser.parse_args(argv)
+
+  # Setup logging verbosity.
+  if args.verbose == 0:
+    level = logging.WARNING
+  elif args.verbose == 1:
+    level = logging.INFO
+  else:
+    level = logging.DEBUG
+  logging.getLogger().setLevel(level)
+
+  # Make sure our root directory exists.
+  root = os.path.abspath(args.cipd_root_dir)
+  ensure_directory(root)
+
+  platform_key, config = get_platform_config()
+  if not config:
+    logging.info('No bootstrap configuration for platform [%s].', platform_key)
+    return 0
+
+  cipd_backend = CipdBackend(args.cipd_backend_url)
+  cipd = install_cipd_client(cipd_backend, config, root)
+  if not cipd:
+    logging.error('Failed to install CIPD client.')
+    return 1
+  assert cipd.exists()
+
+  for l in config.get('cipd_install_lists', ()):
+    cipd.ensure(os.path.join(CIPD_LIST_DIR, l), root)
+  return 0
+
+
+if __name__ == '__main__':
+  logging.basicConfig()
+  logging.getLogger().setLevel(logging.INFO)
+  sys.exit(main(sys.argv[1:]))