[context] Split "env" values and prefixes.

recipe_modules/context/api.py

 # Copyright 2017 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """The context module provides APIs for manipulating a few pieces of 'ambient'
 data that affect how steps are run:
   cwd - The current working directory.
   env - The environment variables.
   infra_step - Whether or not failures should be treated as infrastructure
     failures vs. normal failures.
@@ skipping 24 matching lines @@
 from recipe_engine.config_types import Path
 from recipe_engine.recipe_api import RecipeApi
 
 
 def check_type(name, var, expect):
   if not isinstance(var, expect):  # pragma: no cover
     raise TypeError('%s is not %s: %r (%s)' % (
       name, expect.__name__, var, type(var).__name__))
 
 
-_EnvPathComponent = collections.namedtuple('_EnvPathComponent', (
-    'paths',))
-
-# prefixes is a list of strings
-# value is either a string or None
-_EnvValue = collections.namedtuple('_EnvValue', (
-    'prefixes', 'value'))
-
-
 class ContextApi(RecipeApi):
 
   # TODO(iannucci): move implementation of these data directly into this class.
   def __init__(self, **kwargs):
     super(RecipeApi, self).__init__(**kwargs)
 
     self._cwd = [None]
+    self._env_prefixes = [{}]
     self._env = [{}]
     self._infra_step = [False]
     self._name_prefix = ['']
     # this could be a number, but it makes the logic easier to use a stack.
     self._nest_level = [0]
 
   @contextmanager
-  def __call__(self, cwd=None, env=None, increment_nest_level=None,
-               infra_steps=None, name_prefix=None):
+  def __call__(self, cwd=None, env_prefixes=None, env=None,
+               increment_nest_level=None, infra_steps=None, name_prefix=None):
     """Allows adjustment of multiple context values in a single call.
 
     Contextual data:
       * cwd (Path) - the current working directory to use for all steps.
         To 'reset' to the original cwd at the time recipes started, pass
         `api.path['start_dir']`.
+      * env_prefixes (dict) - Environmental variable prefix augmentations. See
+          below for more info.
+      * env (dict) - Environmental variable overrides. See below for more info.
+      * increment_nest_level (True) - increment the nest level by 1 in this
+        context. Typically you won't directly interact with this, but should
+        use api.step.nest instead.
       * infra_steps (bool) - if steps in this context should be considered
         infrastructure steps. On failure, these will raise InfraFailure
         exceptions instead of StepFailure exceptions.
-      * increment_nest_level (True) - increment the nest level by 1 in this
-        context. Typically you won't directly interact with this, but should
-        use api.step.nest instead.
       * name_prefix (str) - A string to prepend to the names of all steps in
         this context. These compose with '.' characters if multiple name prefix
         contexts occur. See below for more info.
-      * env (dict) - Environmental variable overrides. See below for more info.
 
     Name prefixes:
 
     Multiple invocations concatenate values with '.'.
 
     Example:
       with api.context(name_prefix='hello'):
         # has name 'hello.something'
         api.step('something', ['echo', 'something'])
 
         with api.context(name_prefix='world'):
           # has name 'hello.world.other'
           api.step('other', ['echo', 'other'])
 
     Environmental Variable Overrides:
 
     Env is a mapping of environment variable name to the value you want that
     environment variable to have. The value is one of:
       * None, indicating that the environment variable should be removed from
         the environment when the step runs.
       * A string value. Note that string values will be %-formatted with the
         current value of the environment at the time the step runs. This means
         that you can have a value like:
             "/path/to/my/stuff:%(PATH)s"
         Which, at the time the step executes, will inject the current value of
         $PATH.
-      * A sentinel value such as Prefix to attach a specific component to a
-        pathsep-delimited list variable.
 
-    TODO(iannucci,dnj): Disallow "env" values to be mixes of string or
-    Prefix/Suffix.
+    "env_prefix" is a list of Path or strings that get prefixed to their
+    respective environment variables, delimited with the system's path
+    separator. This can be used to add entries to environment variables such
+    as "PATH" and "PYTHONPATH". If prefixes are specified and a value is also
+    defined in "env", it will be installed as the last path component if it is
+    not empty.
 
     TODO(iannucci): combine nest_level and name_prefix
 
     Look at the examples in "examples/" for examples of context module usage.
     """
     to_pop = []
+    def _push(st, val):
+      st.append(val)
+      to_pop.append(st)
 
     if cwd is not None:
       check_type('cwd', cwd, Path)
-      self._cwd.append(cwd)
-      to_pop.append(self._cwd)
+      _push(self._cwd, cwd)
 
     if infra_steps is not None:
       check_type('infra_steps', infra_steps, bool)
-      self._infra_step.append(infra_steps)
-      to_pop.append(self._infra_step)
+      _push(self._infra_step, infra_steps)
 
     if increment_nest_level is not None:
       check_type('increment_nest_level', increment_nest_level, bool)
       if not increment_nest_level:
         raise ValueError('increment_nest_level=False makes no sense')
-      self._nest_level.append(self.nest_level+1)
-      to_pop.append(self._nest_level)
+      _push(self._nest_level, self.nest_level+1)
 
     if name_prefix is not None:
       check_type('name_prefix', name_prefix, str)
       cur = self.name_prefix
       if cur:
-        self._name_prefix.append('%s.%s' % (cur, name_prefix))
-      else:
-        self._name_prefix.append(name_prefix)
-      to_pop.append(self._name_prefix)
+        name_prefix = '%s.%s' % (cur, name_prefix)
+      _push(self._name_prefix, name_prefix)
 
-    if env is not None and env != {}:
+    if env_prefixes is not None and len(env_prefixes) > 0:
+      check_type('env_prefixes', env_prefixes, dict)
+      new = dict(self._env_prefixes[-1])
+      for k, v in env_prefixes.iteritems():
+        if not v:
+          continue
+        k = str(k)
+        new[k] = tuple(v) + new.get(k, ())
+      _push(self._env_prefixes, new)
+
+    if env is not None and len(env) > 0:
       check_type('env', env, dict)
       # we hit _env directly to avoid an extra copy.
       new = dict(self._env[-1])
       for k, v in env.iteritems():
         k = str(k)
-        if v is None:
-          ev = _EnvValue(prefixes=(), value=None)
-        else:
-          ev = new.get(k, _EnvValue(prefixes=(), value=''))
-          if isinstance(v, _EnvPathComponent):
-            ev = ev._replace(prefixes=v.paths+ev.prefixes)
-          else:
-            v = str(v)
-            try:
-              # This odd little piece of code does the following:
-              #   * add a bogus dictionary format %(foo)s to v. This forces %
-              #     into 'dictionary lookup' mode
-              #   * format the result with a defaultdict. This allows all
-              #     `%(key)s` format lookups to succeed, but any sequential `%s`
-              #     lookups to fail.
-              # If the string contains any accidental sequential lookups, this
-              # will raise an exception. If not, then this is a pluasible format
-              # string.
-              ('%(foo)s'+v) % collections.defaultdict(str)
-            except Exception:
-              raise ValueError(('Invalid %%-formatting parameter in envvar, '
-                                'only %%(ENVVAR)s allowed: %r') % (v,))
-            ev = ev._replace(value=v)
-        new[k] = ev
-      self._env.append(new)
-      to_pop.append(self._env)
+        if v is not None:
+          v = str(v)
+          try:
+            # This odd little piece of code does the following:
+            #   * add a bogus dictionary format %(foo)s to v. This forces %
+            #     into 'dictionary lookup' mode
+            #   * format the result with a defaultdict. This allows all
+            #     `%(key)s` format lookups to succeed, but any sequential `%s`
+            #     lookups to fail.
+            # If the string contains any accidental sequential lookups, this
+            # will raise an exception. If not, then this is a pluasible format
+            # string.
+            ('%(foo)s'+v) % collections.defaultdict(str)
+          except Exception:
+            raise ValueError(('Invalid %%-formatting parameter in envvar, '
+                              'only %%(ENVVAR)s allowed: %r') % (v,))
+        new[k] = v
+      _push(self._env, new)
 
     try:
       yield
     finally:
       for p in to_pop:
         p.pop()
 
   @property
   def cwd(self):
     """Returns the current working directory that steps will run in.
@@ skipping 10 matching lines @@
 
     By default this is empty; There's no facility to observe the program's
     startup environment. If you want to pass data to the recipe, it should be
     done with properties.
 
     Returns (dict) - The env-key -> value mapping of current environment
       modifications.
     """
     # TODO(iannucci): store env in an immutable way to avoid excessive copies.
     # TODO(iannucci): handle case-insensitive keys on windows
-    def parts(ev):
-      for p in ev.prefixes:
-        yield str(p)
-      if ev.value:
-        yield ev.value
+    return dict(self._env[-1])
 
-    ret = {}
-    for k, ev in self._env[-1].iteritems():
-      if ev.value is None:
-        ret[k] = None
-      else:
-        ret[k] = self.m.path.pathsep.join(parts(ev))
+  @property
+  def env_prefixes(self):
+    """Returns Path prefix modifications to the environment.
 
-    return ret
+    This will return a mapping of environment key to Path tuple for Path
+    prefixes registered with the environment.
+
+    Returns (dict) - The env-key -> value(Path) mapping of current environment
+      prefix modifications.
+    """
+    # TODO(iannucci): store env in an immutable way to avoid excessive copies.
+    # TODO(iannucci): handle case-insensitive keys on windows
+    return dict(self._env_prefixes[-1])
 
   @property
   def infra_step(self):
     """Returns the current value of the infra_step setting.
 
     Returns (bool) - True iff steps are currently considered infra steps.
     """
     return self._infra_step[-1]
 
   @property
   def name_prefix(self):
     """Gets the current step name prefix.
 
     Returns (str) - The string prefix that every step will have prepended to it.
     """
     return self._name_prefix[-1]
 
   @property
   def nest_level(self):
     """Returns the current 'nesting' level.
 
     Note: This api is low-level, and you should always prefer to use
     `api.step.nest`. This api is included for completeness and documentation
     purposes.
 
     Returns (int) - The current nesting level.
     """
     return self._nest_level[-1]
-
-  def Prefix(self, *paths):
-    """Returns: an assignable "env" value that prefixes the specified paths to
-    the beginning of an environment variable.
-
-    Each path in paths is added, in order, as a prefix to the environment
-    variable, delimited by the OS path separator. This can be used for
-    easy manipulation of path environment variables such as PATH and PYTHONPATH.
-
-    Args:
-      paths (...Path): The list of paths to prefix.
-    """
-    for i, path in enumerate(paths):
-        check_type('path element %d' % (i,), path, Path)
-    return _EnvPathComponent(paths=paths)