[autoroll] make autoroller propagate changes to recipes.py.

recipe_engine/autoroll.py

Index: recipe_engine/autoroll.py
diff --git a/recipe_engine/autoroll.py b/recipe_engine/autoroll.py
index ca3b71ccc4c4940873ec83c92e8832851128f803..980e62f6c35d43353c04b58647984aa5af1f5f35 100644
--- a/recipe_engine/autoroll.py
+++ b/recipe_engine/autoroll.py
@@ -12,6 +12,8 @@ import sys
 from . import package
 
 
+NUL = open(os.devnull, 'w')
+
 ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
 
 
@@ -23,7 +25,98 @@ def default_json_encode(o):
   return repr(o)
 
 
-def run_simulation_test(repo_root, package_spec, additional_args=None):
+# This is the path within the recipes-py repo to the per-repo recipes.py script.
+# Ideally we'd read this somehow from each candidate engine repo version, but
+# for now assume it lives in a fixed location within the engine.
+RECIPES_PY_REL_PATH = ('doc', 'recipes.py')
+
+# These are the lines to look for in doc/recipes.py as well as the target repo's
+# copy of that file. Any lines found between these lines will be replaced
+# verbatim in the new recipes.py file.
+EDIT_HEADER = '#### PER-REPO CONFIGURATION (editable) ####\n'
+EDIT_FOOTER = '#### END PER-REPO CONFIGURATION ####\n'
+
+
+def write_new_recipes_py(context, spec, repo_cfg_block):
+  """Uses the doc/recipes.py script from the currently-checked-out version of
+  the recipe_engine (in `context`) as a template, and writes it to the
+  recipes_dir of the destination repo (also from `context`). Replaces the lines
+  between the EDIT_HEADER and EDIT_FOOTER with the lines from repo_cfg_block,
+  verbatim.
+
+  Args:
+    context (PackageContext) - The context of where to find the checked-out
+      recipe_engine as well as where to put the new recipes.py.
+    spec (PackageSpec) - The rolled spec (result of
+      RollCandidate.get_rolled_spec())
+    repo_cfg_block (list(str)) - The list of lines (including newlines)
+      extracted from the repo's original recipes.py file (using the
+      extract_repo_cfg_block function).
+  """
+  source_path = os.path.join(spec.deps['recipe_engine'].path,
+                             *RECIPES_PY_REL_PATH)
+  dest_path = os.path.join(context.recipes_dir, 'recipes.py')
+  with open(source_path, 'rb') as source:
+    with open(dest_path, 'wb') as dest:
+      for line in source:
+        dest.write(line)
+        if line == EDIT_HEADER:
+          break
+      dest.writelines(repo_cfg_block)
+      for line in source:
+        if line == EDIT_FOOTER:
+          dest.write(line)
+          break
+      dest.writelines(source)
+  if sys.platform != 'win32':
+    os.chmod(dest_path, os.stat(dest_path).st_mode|0111)
+
+
+def extract_repo_cfg_block(context):
+  """Extracts the lines between EDIT_HEADER and EDIT_FOOTER from the
+  to-be-autorolled-repo's recipes.py file.
+
+  Args:
+    context (PackageContext) - The context of where to find the repo's current
+      recipes.py file.
+
+  Returns list(str) - The list of lines (including newlines) which occur between
+    the EDIT_HEADER and EDIT_FOOTER in the repo's recipes.py file.
+  """
+  recipes_py_path = os.path.join(context.recipes_dir, 'recipes.py')
+  block = []
+  with open(recipes_py_path, 'rb') as f:
+    in_section = False
+    for line in f:
+      if not in_section and line == EDIT_HEADER:
+        in_section = True
+      elif in_section:
+        if line == EDIT_FOOTER:
+          break
+        block.append(line)
+  if not block:
+    raise ValueError('unable to find configuration section in %r' %
+                     (recipes_py_path,))
+  return block
+
+
+def fetch(repo_root, package_spec):
+  """
+  Just fetch the recipes to the newly configured version.
+  """
+  # Use _local_ recipes.py, so that it checks out the pinned recipe engine,
+  # rather than running recipe engine which may be at a different revision
+  # than the pinned one.
+  args = [
+    sys.executable,
+    os.path.join(repo_root, package_spec.recipes_path, 'recipes.py'),
+    'fetch',
+  ]
+  subprocess.check_call(args, stdout=NUL, stderr=NUL)
+
+
+def run_simulation_test(repo_root, package_spec, additional_args=None,
+                        allow_fetch=False):
   """
   Runs recipe simulation test for given package.
 
@@ -35,8 +128,10 @@ def run_simulation_test(repo_root, package_spec, additional_args=None):
   args = [
     sys.executable,
     os.path.join(repo_root, package_spec.recipes_path, 'recipes.py'),
-    'simulation_test',
   ]
+  if not allow_fetch:
+    args.append('--no-fetch')
+  args.append('simulation_test')
   if additional_args:
     args.extend(additional_args)
   p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -50,6 +145,8 @@ def process_candidates(candidates, context, config_file, package_spec):
   trivial = None
   picked_roll_details = None
 
+  repo_cfg_block = extract_repo_cfg_block(context)
+
   print('looking for a trivial roll...')
 
   # Fill basic information about all the candidates. In later loops
@@ -67,7 +164,11 @@ def process_candidates(candidates, context, config_file, package_spec):
   for i, candidate in enumerate(candidates):
     print('  processing candidate #%d... ' % (i + 1), end='')
 
-    config_file.write(candidate.get_rolled_spec().dump())
+    spec = candidate.get_rolled_spec()
+    config_file.write(spec.dump())
+    fetch(context.repo_root, package_spec)
+    write_new_recipes_py(context, spec, repo_cfg_block)
+
     rc, output = run_simulation_test(context.repo_root, package_spec)
     roll_details[i]['recipes_simulation_test'] = {
       'output': output,
@@ -91,7 +192,10 @@ def process_candidates(candidates, context, config_file, package_spec):
     for i, candidate in reversed(list(enumerate(candidates))):
       print('  processing candidate #%d... ' % (i + 1), end='')
 
-      config_file.write(candidate.get_rolled_spec().dump())
+      spec = candidate.get_rolled_spec()
+      config_file.write(spec.dump())
+      fetch(context.repo_root, package_spec)
+      write_new_recipes_py(context, spec, repo_cfg_block)
 
       rc, output = run_simulation_test(
           context.repo_root, package_spec, ['train'])
@@ -164,7 +268,8 @@ def main(args, repo_root, config_file):
       # Restore initial state. Since we could be running simulation tests
       # on other revisions, re-run them now as well.
       config_file.write(package_spec.dump())
-      run_simulation_test(context.repo_root, package_spec, ['train'])
+      run_simulation_test(context.repo_root, package_spec, ['train'],
+                          allow_fetch=True)
 
   if args.output_json:
     with open(args.output_json, 'w') as f: