Add support for LUCI_CONTEXT.

recipe_modules/context/api.py

 # Copyright 2017 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """The context module provides APIs for manipulating a few pieces of 'ambient'
 data that affect how steps are run.
 
 The pieces of information which can be modified are:
   * cwd - The current working directory.
   * env - The environment variables.
@@ skipping 10 matching lines @@
 program.
 
 Example:
 ```python
 with api.context(cwd=api.path['start_dir'].join('subdir')):
   # this step is run inside of the subdir directory.
   api.step("cat subdir/foo", ['cat', './foo'])
 ```
 """
 
-
 import collections
-import os
-import types
+import copy
 
 from contextlib import contextmanager
 
-from recipe_engine import recipe_api
 from recipe_engine.config_types import Path
 from recipe_engine.recipe_api import RecipeApi
 
+from libs import luci_context as luci_ctx
+
 
 def check_type(name, var, expect):
   if not isinstance(var, expect):  # pragma: no cover
     raise TypeError('%s is not %s: %r (%s)' % (
       name, expect.__name__, var, type(var).__name__))
 
 
 class ContextApi(RecipeApi):
 
   # TODO(iannucci): move implementation of these data directly into this class.
   def __init__(self, **kwargs):
     super(RecipeApi, self).__init__(**kwargs)
 
     self._cwd = [None]
     self._env_prefixes = [{}]
     self._env = [{}]
+    if self._test_data.enabled:
+      self._luci_context = [self._test_data.get('luci_context', {})]
+    else:  # pragma: no cover
+      self._luci_context = [luci_ctx.read_full()]
     self._infra_step = [False]
     self._name_prefix = ['']
     # this could be a number, but it makes the logic easier to use a stack.
     self._nest_level = [0]
 
   @contextmanager
-  def __call__(self, cwd=None, env_prefixes=None, env=None,
+  def __call__(self, cwd=None, env_prefixes=None, env=None, luci_context=None,
                increment_nest_level=None, infra_steps=None, name_prefix=None):
     """Allows adjustment of multiple context values in a single call.
 
     Args:
       * cwd (Path) - the current working directory to use for all steps.
         To 'reset' to the original cwd at the time recipes started, pass
         `api.path['start_dir']`.
       * env_prefixes (dict) - Environmental variable prefix augmentations. See
           below for more info.
       * env (dict) - Environmental variable overrides. See below for more info.
+      * luci_context (dict) - LUCI_CONTEXT overrides.

[iannucci, 2017/08/04 18:55:50]

"see below for more info"

       * increment_nest_level (True) - increment the nest level by 1 in this
         context. Typically you won't directly interact with this, but should
         use api.step.nest instead.
       * infra_steps (bool) - if steps in this context should be considered
         infrastructure steps. On failure, these will raise InfraFailure
         exceptions instead of StepFailure exceptions.
       * name_prefix (str) - A string to prepend to the names of all steps in
         this context. These compose with '.' characters if multiple name prefix
         contexts occur. See below for more info.
 
@@ skipping 25 matching lines @@
         Which, at the time the step executes, will inject the current value of
         $PATH.
 
     "env_prefix" is a list of Path or strings that get prefixed to their
     respective environment variables, delimited with the system's path
     separator. This can be used to add entries to environment variables such
     as "PATH" and "PYTHONPATH". If prefixes are specified and a value is also
     defined in "env", it will be installed as the last path component if it is
     not empty.
 
+    LUCI_CONTEXT overrides:
+
+    This is advanced stuff. LUCI_CONTEXT is used to pass ambient information
+    between layers of LUCI stack. 'luci_context' can be used to replace, add or
+    pop items in the current LUCI_CONTEXT. Unlike 'env', LUCI_CONTEXT contains
+    structured values, so values of 'luci_context' dict can be anything (not
+    only strings), and we are not attempting to merge them in any way, e.g
+    there's no equivalent of %(PATH)s trick.

[iannucci, 2017/08/04 18:55:50]

we're going to delete the %(PATH) trick anyway, so I would remove that bit.

I would also mention that the values must be either dict (to replace that value)
or None (to remove that value).

+
+    See https://github.com/luci/client-py/blob/master/LUCI_CONTEXT.md.
+
     **TODO(iannucci): combine nest_level and name_prefix**
 
     Look at the examples in "examples/" for examples of context module usage.
     """
     to_pop = []
     def _push(st, val):
       st.append(val)
       to_pop.append(st)
 
     if cwd is not None:
@@ skipping 45 matching lines @@
             # If the string contains any accidental sequential lookups, this
             # will raise an exception. If not, then this is a pluasible format
             # string.
             ('%(foo)s'+v) % collections.defaultdict(str)
           except Exception:
             raise ValueError(('Invalid %%-formatting parameter in envvar, '
                               'only %%(ENVVAR)s allowed: %r') % (v,))
         new[k] = v
       _push(self._env, new)
 
+    if luci_context is not None and len(luci_context) > 0:
+      check_type('luci_context', luci_context, dict)
+      new = dict(self._luci_context[-1])
+      for k, v in luci_context.iteritems():
+        k = str(k)
+        if v is None:
+          new.pop(k, None)
+        elif isinstance(v, dict):
+          new[k] = copy.deepcopy(v)

[iannucci, 2017/08/04 18:55:51]

can we assert that v is serializable to json? Otherwise we'll get a weird error
from somewhere deeper in the stack.

+        else:
+          raise TypeError(
+              'Bad type for LUCI_CONTEXT[%r]: %s', k, type(v).__name__)
+      _push(self._luci_context, new)
+
     try:
       yield
     finally:
       for p in to_pop:
         p.pop()
 
   @property
   def cwd(self):
     """Returns the current working directory that steps will run in.
 
@@ skipping 26 matching lines @@
     prefixes registered with the environment.
 
     **Returns (dict)** - The env-key -> value(Path) mapping of current
     environment prefix modifications.
     """
     # TODO(iannucci): store env in an immutable way to avoid excessive copies.
     # TODO(iannucci): handle case-insensitive keys on windows
     return dict(self._env_prefixes[-1])
 
   @property
+  def luci_context(self):
+    """Returns a dict with current state of LUCI_CONTEXT.
+
+    This is advanced stuff. LUCI_CONTEXT is used to pass ambient information
+    between layers of LUCI stack. Unlike 'env', it MAY be populated with
+    information about LUCI execution environment when recipe engine starts.
+
+    Do not dump the entirety of LUCI_CONTEXT into logs, it may contain secrets.

[iannucci, 2017/08/04 18:55:50]

s/may contain/contains

make them scared :)

+
+    See https://github.com/luci/client-py/blob/master/LUCI_CONTEXT.md.
+    """
+    return dict(self._luci_context[-1])
+
+  @property
   def infra_step(self):
     """Returns the current value of the infra_step setting.
 
     **Returns (bool)** - True iff steps are currently considered infra steps.
     """
     return self._infra_step[-1]
 
   @property
   def name_prefix(self):
     """Gets the current step name prefix.
 
     **Returns (str)** - The string prefix that every step will have prepended to
     it.
     """
     return self._name_prefix[-1]
 
   @property
   def nest_level(self):
     """Returns the current 'nesting' level.
 
     Note: This api is low-level, and you should always prefer to use
     `api.step.nest`. This api is included for completeness and documentation
     purposes.
 
     **Returns (int)** - The current nesting level.
     """
     return self._nest_level[-1]