[context] Add path prefix/suffix manipulation.

recipe_engine/step_runner.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import calendar
 import collections
 import contextlib
 import datetime
 import itertools
 import json
@@ skipping 407 matching lines @@
     * copy() returns self
 
   The 'formatted' result can be obtained by looking at .data.
   """
   def __init__(self):
     self.data = {}
 
   def __getitem__(self, key):
     return '<%s>' % key
 
+  def keys(self):
+    return self.data.keys()
+
   def __delitem__(self, key):
     self.data[key] = None
 
   def __contains__(self, key):
     return True
 
   def __setitem__(self, key, value):
     self.data[key] = value
 
   def copy(self):
@@ skipping 250 matching lines @@
 def _merge_envs(original, override):
   """Merges two environments.
 
   Returns a new environment dict with entries from |override| overwriting
   corresponding entries in |original|. Keys whose value is None will completely
   remove the environment variable. Values can contain %(KEY)s strings, which
   will be substituted with the values from the original (useful for amending, as
   opposed to overwriting, variables like PATH).
   """
   result = original.copy()
+  subst = (original if isinstance(original, fakeEnviron)
+           else collections.defaultdict(lambda: '', **original))
   if not override:
     return result
   for k, v in override.items():
     if v is None:
       if k in result:
         del result[k]
     else:
-      result[str(k)] = str(v) % original
+      result[str(k)] = str(v) % subst

[dnj, 2017/06/03 15:37:33]

This fixes a bug where an environment variable that is referenced (e.g.,
"IDONTEXIST") will actually cause a KeyError instead of just being substituted
as an empty string for real runs. I think we're just lucky so far that most
referenced environment variables are "PATH" / "PYTHONPATH", and that both are
always defined.

[iannucci, 2017/06/05 19:23:04]

Yeah, and with this CL I'd like to eventually eliminate the substitution
entirely (so that it's a straight assign/delete, and the pre/suffix path stuff).

[dnj, 2017/06/07 02:59:28]

ACK, but I think for now we still need it for backwards compatibility. Would be
nice to remove, make env either full append or full prescribe. The substitution
is def one of the most obscure and ugly part of this API.

OTOH referencing environment might be necessary, e.g.,

LOGDOG_STREAM_BASE=foo/bar

"Oh I want my step to log to a substream!"

env={'LOGDOG_STREAM_BASE': '%(LOGDOG_STREAM_BASE)s/baz'}

   return result
 
 
 if sys.platform == "win32":
   _hunt_path_exts = ('.exe', '.bat')
   def _hunt_path(rendered_step, step_env):
     """This takes the lazy cross-product of PATH and ('.exe', '.bat') to find
     what cmd.exe would have found for the command if we used shell=True.
 
     This must be called on the render_step AFTER _merge_envs has produced
@@ skipping 67 matching lines @@
   supplied command, and only uses the |env| kwarg for modifying the environment
   of the child process.
   """
   saved_path = os.environ['PATH']
   try:
     if path is not None:
       os.environ['PATH'] = path
     yield
   finally:
     os.environ['PATH'] = saved_path