[autoroll] make autoroller propagate changes to recipes.py.

doc/recipes.py

 #!/usr/bin/env python
 
 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Bootstrap script to clone and forward to the recipe engine tool.
 
 ***********************************************************************
 ** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. **
 ***********************************************************************
 
 This is a copy of https://github.com/luci/recipes-py/blob/master/doc/recipes.py.
 To fix bugs, fix in the github repo then copy it back to here and fix the
 PER-REPO CONFIGURATION section to look like this one.
 """
 
 import os
 
+# IMPORTANT: Do not alter the header or footer line for the
+# "PER-REPO CONFIGURATION" section below, or the autoroller will not be able
+# to automatically update this file! All lines between the header and footer
+# lines will be retained verbatim by the autoroller.
+
 #### PER-REPO CONFIGURATION (editable) ####
 # The root of the repository relative to the directory of this file.
 REPO_ROOT = None # os.path.join(os.pardir, os.pardir)
 # The path of the recipes.cfg file relative to the root of the repository.
 RECIPES_CFG = None # os.path.join('infra', 'config', 'recipes.cfg')
 #### END PER-REPO CONFIGURATION ####
 
 BOOTSTRAP_VERSION = 1
 
 import argparse
 import ast
 import json
 import logging
 import random
 import re
 import subprocess
 import sys
 import time
 import traceback
+import urlparse
 
 from cStringIO import StringIO
 
 
 def parse(repo_root, recipes_cfg_path):
   """Parse is transitional code which parses a recipes.cfg file as either jsonpb
   or as textpb.
 
   Args:
     repo_root (str) - native path to the root of the repo we're trying to run
@@ skipping 16 matching lines @@
     data = fh.read()
 
   if data.lstrip().startswith('{'):
     pb = json.loads(data)
     engine = next(
       (d for d in pb['deps'] if d['project_id'] == 'recipe_engine'), None)
     if engine is None:
       raise ValueError('could not find recipe_engine dep in %r'
                        % recipes_cfg_path)
     engine_url = engine['url']
-    engine_revision = engine['revision']
+    engine_revision = engine.get('revision', '')

[iannucci, 2017/03/16 05:19:55]

these changes are needed because the autoroller tests now actually put a proper
recipes.py file in place before calling 'train' or whatever. They use a file://
url scheme for the recipe_engine to point at the recipes repo under test (and so
there's no 'revision' string)

     engine_subpath = engine.get('path_override', '')
     recipes_path = pb.get('recipes_path', '')
   else:
     def get_unique(things):
       if len(things) == 1:
         return things[0]
       elif len(things) == 0:
         raise ValueError("Expected to get one thing, but dinna get none.")
       else:
         logging.warn('Expected to get one thing, but got a bunch: %s\n%s' %
                      (things, traceback.format_stack()))
         return things[0]
 
     protobuf = parse_textpb(StringIO(data))
 
     engine_buf = get_unique([
         b for b in protobuf.get('deps', [])
         if b.get('project_id') == ['recipe_engine'] ])
     engine_url = get_unique(engine_buf['url'])
-    engine_revision = get_unique(engine_buf['revision'])
+    engine_revision = get_unique(engine_buf.get('revision', ['']))
     engine_subpath = (get_unique(engine_buf.get('path_override', ['']))
                       .replace('/', os.path.sep))
     recipes_path = get_unique(protobuf.get('recipes_path', ['']))
 
   recipes_path = os.path.join(repo_root, recipes_path.replace('/', os.path.sep))
   return engine_url, engine_revision, engine_subpath, recipes_path
 
 
 def parse_textpb(fh):
   """Parse the protobuf text format just well enough to understand recipes.cfg.
@@ skipping 89 matching lines @@
     git = 'git'
 
   # Find the repository and config file to operate on.
   repo_root = os.path.abspath(
       os.path.join(os.path.dirname(__file__), REPO_ROOT))
   recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG)
 
   engine_url, engine_revision, engine_subpath, recipes_path = parse(
     repo_root, recipes_cfg_path)
 
-  deps_path = os.path.join(recipes_path, '.recipe_deps')
   engine_path = find_engine_override(sys.argv[1:])
+  if not engine_path and engine_url.startswith('file://'):

[iannucci, 2017/03/16 05:19:55]

this recipes.py file now works correctly with file urls. and this change will
automatically roll downstream! yay!

+    engine_path = urlparse.urlparse(engine_url).path
+
   if not engine_path:
+    deps_path = os.path.join(recipes_path, '.recipe_deps')
     # Ensure that we have the recipe engine cloned.
     engine_root_path = os.path.join(deps_path, 'recipe_engine')
     engine_path = os.path.join(engine_root_path, engine_subpath)
     def ensure_engine():
       if not os.path.exists(deps_path):
         os.makedirs(deps_path)
       if not os.path.exists(engine_root_path):
         _subprocess_check_call([git, 'clone', engine_url, engine_root_path])
 
       needs_fetch = _subprocess_call(
@@ skipping 13 matching lines @@
       time.sleep(random.uniform(2,5))
       ensure_engine()
 
   args = ['--package', recipes_cfg_path] + sys.argv[1:]
   return _subprocess_call([
       sys.executable, '-u',
       os.path.join(engine_path, 'recipes.py')] + args)
 
 if __name__ == '__main__':
   sys.exit(main())