cipd recipe_module

scripts/slave/recipe_modules/cipd/resources/bootstrap.py

+# Copyright 2015 The Chromium Authors. All rights reserved.

[Vadim Sh., 2015/06/29 21:00:46]

this script is very GCE-specific. Please remove all GCE-specific code and code
that is not actually involved in running install_cipd_client.

[seanmccullough, 2015/06/30 17:39:39]

Done.

+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import errno
+import hashlib
+import httplib
+import json
+import logging
+import logging.handlers
+import os
+import socket
+import ssl
+import subprocess
+import sys
+import tempfile
+import time
+import urllib
+import urllib2
+
+from recipe_engine import recipe_api
+
+# Global logger to use.
+LOG = None
+# Default package repository URL.
+CIPD_BACKEND_URL = 'https://chrome-infra-packages.appspot.com'
+# Path to CA certs bundle file to use by default.
+DEFAULT_CERT_FILE = os.path.join(
+    os.path.abspath(os.path.dirname(__file__)), 'cacert.pem')
+# URL of the GCE metadata server.
+METADATA_SERVER = 'http://169.254.169.254/computeMetadata/v1'
+# Name of the attribute with definition of what to install.
+METADATA_ATTRIBUTE = 'cipd_deployments'
+
+
+class CipdBootstrapError(Exception):
+  """Raised by install_cipd_client on fatal error."""
+
+
+class Environment(object):
+  """Environment wraps global properties passed to CipdDeployment objects.
+
+  Args:
+    cipd: path to CIPD client executable.
+    service_account_json: optional path to JSON file with service account creds.
+  """
+
+  def __init__(self, root, cipd, service_account_json):
+    self.root = root
+    self.cipd = cipd
+    self.service_account_json = service_account_json
+
+
+class CipdDeployment(object):
+  """Installs given CIPD packages into a given directory.
+
+  Args:
+    site_root: path to the directory to install packages into.
+    packages: dict {package name -> version} of all packages to install.
+    env: instance of Environment object.
+  """
+
+  def __init__(self, site_root, packages, owner, group, env):
+    self.site_root = site_root
+    self.packages = packages
+    self.owner = owner
+    self.group = group
+    self.env = env
+
+  @staticmethod
+  def create(spec, env):
+    path = spec.get('path')
+    if not isinstance(path, basestring) or not path.startswith('/'):
+      raise ValueError('Not an absolute path: %s' % (path,))
+    packages = spec.get('packages') or {}
+    if not isinstance(packages, dict):
+      raise TypeError('"packages" must be dict: %r' % (packages,))
+    for k, v in packages.iteritems():
+      if not isinstance(k, basestring):
+        raise TypeError('Package name must be string: %r' % (k,))
+      if not isinstance(v, basestring):
+        raise TypeError('Package version must be string: %r' % (v,))
+    if not packages:
+      raise ValueError('At least one package must be specified for %s' % path)
+    owner = spec.get('owner')
+    if owner and not isinstance(owner, basestring):
+      raise TypeError('"owner" must be string: %r' % (owner,))
+    group = spec.get('group') or owner
+    if group and not isinstance(group, basestring):
+      raise TypeError('"group" must be string: %r' % (group,))
+    if group and not owner:
+      raise ValueError('If "group" is specified, "owner" must be specified too')
+    return CipdDeployment(
+        reroot(env.root, str(path)), packages, owner, group, env)
+
+  def apply(self):
+    print('About to install to %s:', self.site_root)
+    for k, v in sorted(self.packages.items()):
+      print('   %s => %s', k, v)
+    package_list = '\n'.join(
+        '%s %s' % (k, v) for k, v in sorted(self.packages.iteritems()))
+    package_list_path = os.path.join(self.site_root, '.cipd', 'ensure.txt')
+    ensure_directory(os.path.dirname(package_list_path))
+    with open(package_list_path, 'w') as f:
+      f.write(package_list)
+    cmd = [
+        self.env.cipd, 'ensure',
+        '-list', package_list_path,
+        '-root', self.site_root,
+        '-service-url', CIPD_BACKEND_URL,
+    ]
+    if self.env.service_account_json:
+      cmd.extend(['-service-account-json', self.env.service_account_json])
+    exit_code = execute(cmd)
+    if exit_code:
+      return exit_code
+    if self.owner and self.group:
+      return chown(self.site_root, self.owner, self.group)
+    return 0
+
+
+def run_update(source, root, service_account_json):
+  """Installs all packages specified in the instance metadata.
+
+  Args:
+    source: optional path to JSON file to read instead of GCE metadata.
+    root: base directory for all paths specified in the JSON spec.
+    service_account_json: optional path to JSON file with service account creds.
+
+  Returns:
+    Process exit code.
+  """
+  # Grab a list of packages to install from GCE metadata attribute.
+  if source:
+    with open(source, 'r') as f:
+      attr = f.read()
+  else:
+    attr = get_gce_metadata(METADATA_ATTRIBUTE)
+  if not attr:
+    print('Nothing to install, no "%s" metadata is set', METADATA_ATTRIBUTE)
+    return 0
+  spec = json.loads(attr)
+  print(
+      'Value of "%s" metadata attribute:\n%s',
+      METADATA_ATTRIBUTE, dump_json(spec))
+  if spec.get('skip'):
+    print('skip == True, exiting')
+    return 0
+
+  # Bootstrap CIPD client.
+  client_spec = spec.get('cipd_client')
+  if not isinstance(client_spec, dict):
+    raise ValueError('Missing cipd_client section')
+  cipd_client = install_cipd_client(
+      reroot(root, client_spec['path']),
+      client_spec['package'],
+      client_spec['version'])
+
+  # Parse spec into a list of deployment objects and then execute them.
+  env = Environment(root, cipd_client, service_account_json)
+  deployments = []
+  for dep in (spec.get('deployments') or []):
+    if not dep.get('skip'):
+      deployments.append(CipdDeployment.create(dep, env))
+  for dep in deployments:
+    exit_code = dep.apply()
+    if exit_code:
+      return exit_code
+  return 0
+
+
+def install_cipd_client(path, package, version):
+  """Installs CIPD client to <path>/cipd.
+
+  Args:
+    path: root directory to install CIPD client into.
+    package: cipd client package name, e.g. infra/tools/cipd/linux-amd64.
+    version: version of the package to install.
+
+  Returns:
+    Absolute path to CIPD executable.
+  """
+  print('Ensuring CIPD client is up-to-date')
+  version_file = os.path.join(path, 'VERSION')
+  bin_file = os.path.join(path, 'cipd')
+
+  # Resolve version to concrete instance ID, e.g "live" -> "abcdef0123....".
+  instance_id = call_cipd_api(
+      'repo/v1/instance/resolve',
+      {'package_name': package, 'version': version})['instance_id']
+  print('CIPD client %s => %s', version, instance_id)
+
+  # Already installed?
+  installed_instance_id = (read_file(version_file) or '').strip()
+  if installed_instance_id == instance_id and os.path.exists(bin_file):
+    return bin_file
+
+  # Resolve instance ID to an URL to fetch client binary from.
+  client_info = call_cipd_api(
+      'repo/v1/client',
+      {'package_name': package, 'instance_id': instance_id})
+  print('CIPD client binary info:\n%s', dump_json(client_info))
+
+  # Fetch the client. It is ~10 MB, so don't bother and fetch it into memory.
+  status, raw_client_bin = fetch_url(client_info['client_binary']['fetch_url'])
+  if status != 200:
+    print('Failed to fetch client binary, HTTP %d' % status)
+    raise CipdBootstrapError('Failed to fetch client binary, HTTP %d' % status)
+  digest = hashlib.sha1(raw_client_bin).hexdigest()
+  if digest != client_info['client_binary']['sha1']:
+    raise CipdBootstrapError('Client SHA1 mismatch')
+
+  # Success.
+  print('Fetched CIPD client %s:%s at %s', package, instance_id, bin_file)
+  write_file(bin_file, raw_client_bin)
+  os.chmod(bin_file, 0755)
+  write_file(version_file, instance_id + '\n')
+  return bin_file
+
+
+def call_cipd_api(endpoint, query):
+  """Sends GET request to CIPD backend, parses JSON response."""
+  url = '%s/_ah/api/%s' % (CIPD_BACKEND_URL, endpoint)
+  if query:
+    url += '?' + urllib.urlencode(query)
+  status, body = fetch_url(url)
+  if status != 200:
+    raise CipdBootstrapError('Server replied with HTTP %d' % status)
+  try:
+    body = json.loads(body)
+  except ValueError:
+    raise CipdBootstrapError('Server returned invalid JSON')
+  status = body.get('status')
+  if status != 'SUCCESS':
+    m = body.get('error_message') or '<no error message>'
+    raise CipdBootstrapError('Server replied with error %s: %s' % (status, m))
+  return body
+
+
+def get_gce_metadata(key):
+  """Reads instance metadata attribute.
+
+  Returns:
+    Blob with attribute value or None if not found.
+  """
+  return fetch_url(
+      '%s/instance/attributes/%s' % (METADATA_SERVER, key),
+      headers={'Metadata-Flavor': 'Google'})[1]
+
+
+def fetch_url(url, headers=None):
+  """Sends GET request (with retries).
+
+  Args:
+    url: URL to fetch.
+    headers: dict with request headers.
+
+  Returns:
+    (200, reply body) on success.
+    (HTTP code, None) on HTTP 401, 403, or 404 reply.
+
+  Raises:
+    Whatever urllib2 raises.
+  """
+  req = urllib2.Request(url)
+  req.add_header('User-Agent', 'ccompute update-cipd-packages')
+  for k, v in (headers or {}).iteritems():
+    req.add_header(str(k), str(v))
+  i = 0
+  while True:
+    i += 1
+    try:
+      print('GET %s', url)
+      return 200, urllib2.urlopen(req, timeout=60).read()
+    except Exception as e:
+      if isinstance(e, urllib2.HTTPError):
+        print('Failed to fetch %s, server returned HTTP %d', url, e.code)
+        if e.code in (401, 403, 404):
+          return e.code, None
+      else:
+        print('Failed to fetch %s', url)
+      if i == 20:
+        raise
+    print('Retrying in %d sec.', i)
+    time.sleep(i)
+
+
+def reroot(root, path):
+  """Moves an absolute path to be under a new root."""
+  if not root.startswith('/'):
+    raise ValueError('Not an absolute path: %s' % root)
+  if not path.startswith('/'):
+    raise ValueError('Not an absolute path: %s' % path)
+  return os.path.join(root, path[1:])
+
+
+def ensure_directory(path):
+  """Creates a directory."""
+  # Handle a case where a file is being converted into a directory.
+  chunks = path.split(os.sep)
+  for i in xrange(len(chunks)):
+    p = os.sep.join(chunks[:i+1])
+    if os.path.exists(p) and not os.path.isdir(p):
+      os.remove(p)
+      break
+  try:
+    os.makedirs(path)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      raise
+
+
+def read_file(path):
+  """Returns contents of a file or None if missing."""
+  try:
+    with open(path, 'r') as f:
+      return f.read()
+  except IOError as e:
+    if e.errno == errno.ENOENT:
+      return None
+    raise
+
+
+def write_file(path, data):
+  """Puts a file on disk, atomically."""
+  assert sys.platform in ('linux2', 'darwin')
+  ensure_directory(os.path.dirname(path))
+  fd, temp_file = tempfile.mkstemp(dir=os.path.dirname(path))
+  with os.fdopen(fd, 'w') as f:
+    f.write(data)
+  os.rename(temp_file, path)
+
+
+def chown(path, owner, group):
+  """Recursively changes owner of a given path if running as root.
+
+  Does nothing if not running as root.
+
+  Args:
+    owner: user name of a new owner.
+    group: name of a new owning group.
+
+  Returns:
+    chown process exit code.
+  """
+  if os.geteuid() != 0:
+    LOG.warning('Skipping chown(%s, %s:%s), not root', path, owner, group)
+    return 0
+  return execute(['chown', '-R', '%s:%s' % (owner, group), path])
+
+
+def execute(cmd):
+  """Runs a subprocess redirecting its stdout and stderr to the log.
+
+  Args:
+    cmd: list with command line arguments.
+
+  Returns:
+    Process exit code.
+  """
+  print('Running ' + ' '.join(cmd))
+  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  line = ''
+  while True:
+    buf = proc.stdout.read(1)
+    if not buf:
+      if line:
+        print(line.strip())
+      break
+    if buf == '\n':
+      print(line.strip())
+      line = ''
+    else:
+      line += buf
+  code = proc.wait()
+  if code:
+    print('Failed with exit code: %d', code)
+  return code
+
+
+def dump_json(obj):
+  """Pretty-formats object to JSON."""
+  return json.dumps(obj, indent=2, sort_keys=True, separators=(',',':'))
+
+
+def setup_logging(log_file):
+  """Configures logging to log to a file."""
+  global LOG
+  logger = logging.getLogger()
+  formatter = logging.Formatter(
+      '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+  handlers = [logging.StreamHandler(stream=sys.stdout)]
+  if log_file:
+    handlers.append(
+        logging.handlers.RotatingFileHandler(log_file, maxBytes=1*1024*1024))
+  for h in handlers:
+    h.setFormatter(formatter)
+    logger.addHandler(h)
+  logger.setLevel(logging.DEBUG)
+  logger.info('-'*40)
+  LOG = logger
+
+
+def setup_urllib2_ssl(cacert):
+  """Configures urllib2 to validate SSL certs.
+
+  See http://stackoverflow.com/a/14320202/3817699.
+  """
+  cacert = os.path.abspath(cacert)
+
+  class ValidHTTPSConnection(httplib.HTTPConnection):
+    default_port = httplib.HTTPS_PORT
+
+    def __init__(self, *args, **kwargs):
+      httplib.HTTPConnection.__init__(self, *args, **kwargs)
+
+    def connect(self):
+      sock = socket.create_connection(
+          (self.host, self.port), self.timeout, self.source_address)
+      if self._tunnel_host:
+        self.sock = sock
+        self._tunnel()
+      self.sock = ssl.wrap_socket(
+          sock, ca_certs=cacert, cert_reqs=ssl.CERT_REQUIRED)
+
+  class ValidHTTPSHandler(urllib2.HTTPSHandler):
+    def https_open(self, req):
+      return self.do_open(ValidHTTPSConnection, req)
+
+  urllib2.install_opener(urllib2.build_opener(ValidHTTPSHandler))
+
+
+def update_cipd():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--source', help='Path to JSON file to read instead of GCE metadata')
+  parser.add_argument(
+      '--root', help='Base directory for all paths specified in the JSON spec',
+      default='/')
+  parser.add_argument(
+      '--cacert',
+      help='Path to cacert.pem file with CA root certificates bundle',
+      default=DEFAULT_CERT_FILE)
+  parser.add_argument('--service-account-json', help='Credentials to use')
+  parser.add_argument('--log', help='Location of the log file')
+  opts = parser.parse_args()
+
+  source = os.path.abspath(opts.source) if opts.source else None
+  root = os.path.abspath(opts.root)
+  if opts.service_account_json:
+    service_account_json = os.path.abspath(opts.service_account_json)
+  else:
+    service_account_json = None
+
+  setup_logging(opts.log)
+  print('Package spec source: %s', source or 'GCE metadata')
+  print('Root: %s', root)
+  print('CA certs bundle: %s', opts.cacert)
+  print('Service account JSON: %s', service_account_json)
+  setup_urllib2_ssl(opts.cacert)
+
+  attempt = 0
+  while True:
+    attempt += 1
+    try:
+      print('Attempt #%d', attempt)
+      exit_code = run_update(source, root, service_account_json)
+      if not exit_code:
+        break
+    except Exception:
+      print('Uncaught exception')
+    sleep = min(5 * 60, attempt * 10)
+    print('Retrying in %s sec.', sleep)
+    time.sleep(sleep)
+  print('-'*40)
+
+
+def main():
+  data = json.load(sys.stdin)
+  package = data['package']
+  version = data['version']
+
+  bin_path = ("cipd_%s" % version)
+  print ("should install cipd at %s" % bin_path)
+
+  exit_code = 0
+  # return if this client version is already installed.
+  try:
+    if not os.path.isfile("%s/cipd" % bin_path): 
+      # To get a current version ID:
+      # Look for "cipd - upload packages" step in a recent build, e.g.
+      # http://build.chromium.org/p/chromium.infra/builders/infra-continuous-trusty-64/
+      # and the last step contains the package name and hash you see here:
+      install_cipd_client(bin_path, package, version)
+  except Exception as e:
+    print ("Exception installing cipd: %s" % e)
+    exit_code = 1
+
+  # move the binary into the current directory.
+  os.rename('%s/cipd' % bin_path, "cipd")
+
+  return exit_code
+
+if __name__ == '__main__':
+  sys.exit(main())