Move ENV into custom deps, allow it to be random.

recipes.py

 #!/usr/bin/env python
 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Tool to interact with recipe repositories.
 
 This tool operates on the nearest ancestor directory containing an
 infra/config/recipes.cfg.
 """
 
 import argparse
 import collections
 import json
 import logging
 import os
+import shutil
 import subprocess
 import sys
+import tempfile
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, ROOT_DIR)
 
 from recipe_engine import env
 from recipe_engine import arguments_pb2
 from google.protobuf import json_format as jsonpb
 from google.protobuf import text_format as textpb
 
 def get_package_config(args):
@@ skipping 338 matching lines @@
   """Creates a properties dictionary from an arguments_pb2.PropertyMap entry.
 
   Args:
     pmap (arguments_pb2.PropertyMap): Map to convert to dictionary form.
   Returns (dict): A dictionary derived from the properties in "pmap".
   """
   return dict((k, _op_property_value(pmap[k])) for k in pmap)
 
 
 def main():
-  from recipe_engine import package
-
   # Super-annoyingly, we need to manually parse for simulation_test since
   # argparse is bonkers and doesn't allow us to forward --help to subcommands.
   # Save old_args for if we're using bootstrap
   original_sys_argv = sys.argv[:]
   if 'simulation_test' in sys.argv:
     index = sys.argv.index('simulation_test')
     sys.argv = sys.argv[:index+1] + [json.dumps(sys.argv[index+1:])]
 
   parser = argparse.ArgumentParser(
       description='Interact with the recipe system.')
 
   parser.add_argument(
       '--package',
       help='Path to recipes.cfg of the recipe package to operate on'
         ', usually in infra/config/recipes.cfg')
   parser.add_argument(
       '--deps-path',
-      help='Path where recipe engine dependencies will be extracted.')
+      help='Path where recipe engine dependencies will be extracted. Specify '
+           '"-" to use a temporary directory for deps, which will be cleaned '
+           'up on exit.')
   parser.add_argument(
       '--verbose', '-v', action='count',
       help='Increase logging verboisty')
   # TODO(phajdan.jr): Figure out if we need --no-fetch; remove if not.
   parser.add_argument(
       '--no-fetch', action='store_true',
       help='Disable automatic fetching')
   parser.add_argument('-O', '--project-override', metavar='ID=PATH',
       action=ProjectOverrideAction,
       help='Override a project repository path with a local one.')
@@ skipping 147 matching lines @@
 
   args = parser.parse_args()
 
   # Load/parse operational arguments.
   op_args = arguments_pb2.Arguments()
   if args.operational_args_path is not None:
     with open(args.operational_args_path) as fd:
       data = fd.read()
     jsonpb.Parse(data, op_args)
 
-  if args.use_bootstrap and not os.environ.pop('RECIPES_RUN_BOOTSTRAP', None):
-    subprocess.check_call(
-        [sys.executable, 'bootstrap/bootstrap.py', '--deps-file',
-         'bootstrap/deps.pyl', 'ENV'],
-        cwd=os.path.dirname(os.path.realpath(__file__)))
-
-    os.environ['RECIPES_RUN_BOOTSTRAP'] = '1'
-    args = sys.argv
-    return subprocess.call(
-        [os.path.join(
-            os.path.dirname(os.path.realpath(__file__)), 'ENV/bin/python'),
-         os.path.join(ROOT_DIR, 'recipes.py')] + original_sys_argv[1:])
-
   if args.verbose > 1:
     logging.getLogger().setLevel(logging.DEBUG)
   elif args.verbose > 0:
     logging.getLogger().setLevel(logging.INFO)
 
+  # If we're using a temporary deps directory, create it.
+  temp_deps_dir = None
+  try:
+    # When bootstrapping, re-use the calling wrapper's deps directory instead of
+    # creating a new one.
+    args.deps_path = os.environ.pop('RECIPES_RUN_BOOTSTRAP_DEPS_DIR',
+                                    args.deps_path)
+    if args.deps_path == '-':
+      # "-" means use a temporary deps path.
+      temp_deps_dir = tempfile.mkdtemp(dir=ROOT_DIR, suffix='_deps')
+      args.deps_path = temp_deps_dir
+
+    if args.deps_path:
+      logging.debug('Using custom deps path: %s', args.deps_path)

[iannucci, 2016/11/17 21:15:04]

let's make this warning?

[dnj, 2016/11/17 21:27:56]

Done.

[Michael Moss, 2016/12/02 17:08:57]

Am I doing something wrong, or is expected that I see this warning every time I
run recipe_simulation_test.py nowadays?

+
+    # If we're bootstrapping, construct our bootstrap environment. If we're
+    # using a custom deps path, install our enviornment there too.
+    if args.use_bootstrap and not os.environ.pop('RECIPES_RUN_BOOTSTRAP', None):
+      # Propagate our deps path, if specified, so we re-use our temporary
+      # directory.
+      if args.deps_path:
+        env_path = os.path.join(args.deps_path, 'ENV')

[iannucci, 2016/11/17 21:15:04]

let's just make it .ENV

[dnj, 2016/11/17 21:27:56]

Done.

+        os.environ['RECIPES_RUN_BOOTSTRAP_DEPS_DIR'] = args.deps_path
+      else:
+        env_path = os.path.join(ROOT_DIR, 'ENV')
+
+      logging.debug('Installing bootstrap environment into: %s', env_path)
+      subprocess.check_call(
+          [
+            sys.executable,
+            os.path.join(ROOT_DIR, 'bootstrap', 'bootstrap.py'),
+            '--deps-file', os.path.join(ROOT_DIR, 'bootstrap', 'deps.pyl'),
+            env_path,
+          ],
+          cwd=ROOT_DIR)
+
+      # Mark that we're bootstrapping, so the next invocation falls through to
+      # standard recipe operation.
+      os.environ['RECIPES_RUN_BOOTSTRAP'] = '1'
+      args = sys.argv
+      return subprocess.call(
+          [
+            os.path.join(env_path, 'bin', 'python'),
+            os.path.join(ROOT_DIR, 'recipes.py'),
+          ] + original_sys_argv[1:])
+
+    # Standard recipe engine operation.
+    return _real_main(args, op_args)
+
+  finally:
+    # If we're using a temporary deps directory, clean it up here.
+    if temp_deps_dir:
+      logging.info('Cleaning up temporary deps path: %s', temp_deps_dir)
+      shutil.rmtree(temp_deps_dir)

[iannucci, 2016/11/17 21:15:04]

can we log the warning instead of throwing if this fails

[dnj, 2016/11/17 21:27:56]

Done.

+
+
+def _real_main(args, op_args):
+  from recipe_engine import package
+
   # Commands which do not require config_file, package_deps, and other objects
   # initialized later.
   if args.command == 'remote':
     return remote(args)
 
   repo_root, config_file = get_package_config(args)
 
   try:
     # TODO(phajdan.jr): gracefully handle inconsistent deps when rolling.
     # This fails if the starting point does not have consistent dependency
@@ skipping 60 matching lines @@
 
   if not isinstance(ret, int):
     if ret is None:
       ret = 0
     else:
       print >> sys.stderr, ret
       ret = 1
   sys.stdout.flush()
   sys.stderr.flush()
   os._exit(ret)