Add an AOSP builder recipe.

scripts/slave/recipe_util.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This module holds utilities which make writing recipes easier."""
 
 import contextlib as _contextlib
 import os as _os
 
 # These imports are intended to be passed through to recipes
@@ skipping 45 matching lines @@
 
 The actual checkout root is filled in by annotated_run after the recipe
 completes, and is dependent on the implementation of 'root()' in
 annotated_checkout for the checkout type that you've selected.
 
 NOTE: In order for this function to work, your recipe MUST use the 'checkout'
 functionality provided by annotated_run.
 """  # pylint: disable=W0105
 checkout_path = _path_method('checkout_path', '%(CheckoutRootPlaceholder)s')
 
+def path_exists(path):
+  return _os.path.exists(path)
 
 @_contextlib.contextmanager
-def mock_paths():
+def mock_paths(mock_path_exists=None):

[agable, 2013/05/22 17:21:23]

Thanks for adding this functionality. I find mock_path_exists to be a confusing
name, though. It's a list of paths, so it should be plural, for one thing. Maybe
"paths_to_mock"? Not a big deal but I had to read all the way through the code
to figure out what it was doing because the variable name didn't give me a good
hint.

[mkosiba (inactive), 2013/05/23 13:13:38]

I struggled quite a bit with a good name for this one. Thanks for the
suggestion!

   """Used by unittest/recipes_test.py to temporarily override the paths
   generated by the various path functions in this module.
 
   This is necessary to give equivalent output when running the tests on any
   checkout configuration. Instead of real paths, recipes which use these
   functions will get paths like '[DEPOT_TOOLS_ROOT]'.
   """
   path_base_names = ['depot_tools', 'build_internal', 'build', 'slave_build',
                      'root']
   g = globals()
   tokens = {}
   path_funcs = {}
+  mock_path_exists = frozenset(mock_path_exists or [])
   try:
     for name in path_base_names:
       token_name = (name + '_root').upper()
       token_val = '[%s]' % token_name
       path_func_name = (name + '_path')
 
       if token_name in g:
         tokens[token_name] = g[token_name]
         g[token_name] = token_val
 
       if path_func_name in g:
         path_funcs[path_func_name] = g[path_func_name]
         g[path_func_name] = _path_method(path_func_name, token_val)
+
+    path_funcs['path_exists'] = g['path_exists']
+    g['path_exists'] = lambda path: path in mock_path_exists
+
     yield
   finally:
     g.update(tokens)
     g.update(path_funcs)
 
 
 def deep_set(obj, key_vals):
   """Take an object (a dict or list), and a list of key/value pairs to set,
   and transform it by replacing items in obj at the key locations with the
   respective values.
@@ skipping 255 matching lines @@
             'recursive': recursive,
           },
         }),
       self.git('fetch', 'origin', *recursive_args),
       self.git('update-ref', 'refs/heads/'+branch, 'origin/'+branch),
       self.git('clean', '-f', '-d', '-X'),
       self.git('checkout', '-f', branch),
       self.git('submodule', 'update', '--init', '--recursive'),
     ]
 
-  def gclient_checkout(self, common_repo_name, git_mode=False):
+  def gclient_checkout(self, common_repo_name_or_spec, git_mode=False,
+                       spec_name=None):
     """Returns a step generator function for gclient checkouts."""
-    spec = GCLIENT_COMMON_SPECS[common_repo_name](self)
+    if isinstance(common_repo_name_or_spec, basestring):
+      spec = GCLIENT_COMMON_SPECS[common_repo_name_or_spec](self)
+    else:
+      spec = common_repo_name_or_spec
     spec_string = ''
+    if not spec_name:
+      step_name = lambda n: 'gclient ' + n
+    else:
+      step_name = lambda n: '[spec: %s] gclient %s' % (spec_name, n)

[agable, 2013/05/22 17:21:23]

I'd rather have the spec_name emitted as a @@@STEP_TEXT@@@ (so it shows up like
the revision annotations on normal gclient update steps) than directly included
in the step name. If that's too hard, though, feel free to leave as-is.

[mkosiba (inactive), 2013/05/23 13:13:38]

The reason for this is not to differentiate in the output but to work around the
fact that you can't have two steps with the same name in the steps_history list
(which is a dict whose keys are step names).

[agable, 2013/05/23 17:20:12]

D'oh. SGTM, then.

     for key in spec:
       # We should be using json.dumps here, but gclient directly execs the dict
       # that it receives as the argument to --spec, so we have to have True,
       # False, and None instead of JSON's true, false, and null.
       spec_string += '%s = %s\n' % (key, str(spec[key]))
     gclient = depot_tools_path('gclient') + ('.bat' if IsWindows() else '')
 
     if not git_mode:
-      clean_step = self.step('gclient clean', [gclient, 'revert', '--nohooks'])
-      sync_step = self.step('gclient sync', [gclient, 'sync', '--nohooks'])
+      clean_step = self.step(step_name('clean'),
+                             [gclient, 'revert', '--nohooks'])
+      sync_step = self.step(step_name('sync'), [gclient, 'sync', '--nohooks'])
     else:
       # clean() isn't used because the gclient sync flags passed in checkout()
       # do much the same thing, and they're more correct than doing a separate
       # 'gclient revert' because it makes sure the other args are correct when
       # a repo was deleted and needs to be re-cloned (notably
       # --with_branch_heads), whereas 'revert' uses default args for clone
       # operations.
       #
       # TODO(mmoss): To be like current official builders, this step could just
       # delete the whole <slave_name>/build/ directory and start each build
       # from scratch. That might be the least bad solution, at least until we
       # have a reliable gclient method to produce a pristine working dir for
       # git-based builds (e.g. maybe some combination of 'git reset/clean -fx'
       # and removing the 'out' directory).
       clean_step = None
-      sync_step = self.step('gclient sync', [
+      sync_step = self.step(step_name('sync'), [
         gclient, 'sync', '--verbose', '--with_branch_heads', '--nohooks',
                      '--reset', '--delete_unversioned_trees', '--force'])
     steps = [
       self.step(
-        'gclient setup',
+        step_name('setup'),
         [gclient, 'config', '--spec', spec_string],
         static_json_data={
           'CheckoutRoot': slave_build_path(spec['solutions'][0]['name']),
           'CheckoutSCM': 'gclient',
           'CheckoutSpec': spec
         }
       ),
     ]
     if clean_step:
       steps.append(clean_step)
     if sync_step:
       steps.append(sync_step)
 
     return steps