Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(81)

Side by Side Diff: scripts/slave/recipe_api.py

Issue 23889036: Refactor the way that TestApi works so that it is actually useful. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/build
Patch Set: Yeeeeaaaahhhhhh! Created 7 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright 2013 The Chromium Authors. All rights reserved. 1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import functools 5 import functools
6 import imp
7 import inspect
8 import os
9 import sys
10 import tempfile
11 6
7 from .recipe_test_api import DisabledTestData, ModuleTestData, StepTestData
12 8
13 class Placeholder(object): 9 from .recipe_util import ModuleInjectionSite
14 """Base class for json placeholders. Do not use directly."""
15 def render(self, test_data): # pragma: no cover
16 """Return [cmd items]*"""
17 raise NotImplementedError
18
19 def step_finished(self, presentation, step_result, test_data):
20 """Called after step completion. Intended to modify step_result."""
21 pass
22
23
24 class InputDataPlaceholder(Placeholder):
25 def __init__(self, data, suffix):
26 assert isinstance(data, basestring)
27 self.data = data
28 self.suffix = suffix
29 self.input_file = None
30 super(InputDataPlaceholder, self).__init__()
31
32 def render(self, test_data):
33 if test_data is not None:
34 # cheat and pretend like we're going to pass the data on the
35 # cmdline for test expectation purposes.
36 return [self.data]
37 else: # pragma: no cover
38 input_fd, self.input_file = tempfile.mkstemp(suffix=self.suffix)
39 os.write(input_fd, self.data)
40 os.close(input_fd)
41 return [self.input_file]
42
43 def step_finished(self, presentation, step_result, test_data):
44 if test_data is None: # pragma: no cover
45 os.unlink(self.input_file)
46
47
48 class ModuleInjectionSite(object):
49 pass
50 10
51 11
52 class RecipeApi(object): 12 class RecipeApi(object):
53 """ 13 """
54 Framework class for handling recipe_modules. 14 Framework class for handling recipe_modules.
55 15
56 Inherit from this in your recipe_modules/<name>/api.py . This class provides 16 Inherit from this in your recipe_modules/<name>/api.py . This class provides
57 wiring for your config context (in self.c and methods, and for dependency 17 wiring for your config context (in self.c and methods, and for dependency
58 injection (in self.m). 18 injection (in self.m).
59 19
60 Dependency injection takes place in load_recipe_modules() below. 20 Dependency injection takes place in load_recipe_modules() below.
61 """ 21 """
62 def __init__(self, module=None, mock=None, **_kwargs): 22 def __init__(self, module=None, test_data=DisabledTestData(), **_kwargs):
63 """Note: Injected dependencies are NOT available in __init__().""" 23 """Note: Injected dependencies are NOT available in __init__()."""
64 self.c = None 24 self.c = None
65 self._module = module 25 self._module = module
66 self._mock = mock 26
27 assert isinstance(test_data, (ModuleTestData, DisabledTestData))
28 self._test_data = test_data
67 29
68 # If we're the 'root' api, inject directly into 'self'. 30 # If we're the 'root' api, inject directly into 'self'.
69 # Otherwise inject into 'self.m' 31 # Otherwise inject into 'self.m'
70 self.m = self if module is None else ModuleInjectionSite() 32 self.m = self if module is None else ModuleInjectionSite()
71 33
34 # If our module has a test api, it gets injected here.
35 self.test_api = None
36
72 def _sanitize_config_vars(self, config_name, CONFIG_VARS): 37 def _sanitize_config_vars(self, config_name, CONFIG_VARS):
73 params = self.get_config_defaults(config_name) 38 params = self.get_config_defaults(config_name)
74 params.update(CONFIG_VARS) 39 params.update(CONFIG_VARS)
75 return params 40 return params
76 41
77 def get_config_defaults(self, _config_name): # pylint: disable=R0201 42 def get_config_defaults(self, _config_name): # pylint: disable=R0201
78 """ 43 """
79 Allows your api to dynamically determine static default values for configs. 44 Allows your api to dynamically determine static default values for configs.
80 """ 45 """
81 return {} 46 return {}
(...skipping 27 matching lines...) Expand all
109 self.c = config 74 self.c = config
110 # TODO(iannucci): This is 'inefficient', since if a dep comes up multiple 75 # TODO(iannucci): This is 'inefficient', since if a dep comes up multiple
111 # times in this recursion, it will get set_config()'d multiple times 76 # times in this recursion, it will get set_config()'d multiple times
112 for dep in self._module.DEPS: 77 for dep in self._module.DEPS:
113 getattr(self.m, dep).set_config(config_name, optional=True, **params) 78 getattr(self.m, dep).set_config(config_name, optional=True, **params)
114 79
115 def apply_config(self, config_name, config_object=None): 80 def apply_config(self, config_name, config_object=None):
116 """Apply a named configuration to the provided config object or self.""" 81 """Apply a named configuration to the provided config object or self."""
117 self._module.CONFIG_CTX.CONFIG_ITEMS[config_name](config_object or self.c) 82 self._module.CONFIG_CTX.CONFIG_ITEMS[config_name](config_object or self.c)
118 83
119 84 @property
120 def load_recipe_modules(mod_dirs): 85 def name(self):
121 def patchup_module(submod): 86 return self._module.NAME
122 submod.CONFIG_CTX = getattr(submod, 'CONFIG_CTX', None)
123 submod.API = getattr(submod, 'API', None)
124 submod.DEPS = frozenset(getattr(submod, 'DEPS', ()))
125
126 if hasattr(submod, 'config'):
127 for v in submod.config.__dict__.itervalues():
128 if hasattr(v, 'I_AM_A_CONFIG_CTX'):
129 assert not submod.CONFIG_CTX, (
130 'More than one configuration context: %s' % (submod.config))
131 submod.CONFIG_CTX = v
132 assert submod.CONFIG_CTX, 'Config file, but no config context?'
133
134 for v in submod.api.__dict__.itervalues():
135 if inspect.isclass(v) and issubclass(v, RecipeApi):
136 assert not submod.API, (
137 'More than one Api subclass: %s' % submod.api)
138 submod.API = v
139
140 assert submod.API, 'Submodule has no api? %s' % (submod)
141
142 RM = 'RECIPE_MODULES'
143 def find_and_load(fullname, modname, path):
144 if fullname not in sys.modules or fullname == RM:
145 try:
146 fil, pathname, descr = imp.find_module(modname,
147 [os.path.dirname(path)])
148 imp.load_module(fullname, fil, pathname, descr)
149 finally:
150 if fil:
151 fil.close()
152 return sys.modules[fullname]
153
154 def recursive_import(path, prefix=None, skip_fn=lambda name: False):
155 modname = os.path.splitext(os.path.basename(path))[0]
156 if prefix:
157 fullname = '%s.%s' % (prefix, modname)
158 else:
159 fullname = RM
160 m = find_and_load(fullname, modname, path)
161 if not os.path.isdir(path):
162 return m
163
164 for subitem in os.listdir(path):
165 subpath = os.path.join(path, subitem)
166 subname = os.path.splitext(subitem)[0]
167 if skip_fn(subname):
168 continue
169 if os.path.isdir(subpath):
170 if not os.path.exists(os.path.join(subpath, '__init__.py')):
171 continue
172 elif not subpath.endswith('.py') or subitem.startswith('__init__.py'):
173 continue
174
175 submod = recursive_import(subpath, fullname, skip_fn=skip_fn)
176
177 if not hasattr(m, subname):
178 setattr(m, subname, submod)
179 else:
180 prev = getattr(m, subname)
181 assert submod is prev, (
182 'Conflicting modules: %s and %s' % (prev, m))
183
184 return m
185
186 imp.acquire_lock()
187 try:
188 if RM not in sys.modules:
189 sys.modules[RM] = imp.new_module(RM)
190 # First import all the APIs and configs
191 for root in mod_dirs:
192 if os.path.isdir(root):
193 recursive_import(root, skip_fn=lambda name: name.endswith('_config'))
194
195 # Then fixup all the modules
196 for name, submod in sys.modules[RM].__dict__.iteritems():
197 if name[0] == '_':
198 continue
199 patchup_module(submod)
200
201 # Then import all the config extenders.
202 for root in mod_dirs:
203 if os.path.isdir(root):
204 recursive_import(root)
205 return sys.modules[RM]
206 finally:
207 imp.release_lock()
208 87
209 88
210 def CreateRecipeApi(names, mod_dirs, mocks=None, **kwargs): 89 def inject_test_data(func):
211 """ 90 """
212 Given a list of module names, return an instance of RecipeApi which contains 91 Decorator which injects mock data from this module's test_api method into
213 those modules as direct members. 92 the return value of the decorated function.
214 93
215 So, if you pass ['foobar'], you'll get an instance back which contains a 94 The return value of func MUST be a single step dictionary (specifically,
216 'foobar' attribute which itself is a RecipeApi instance from the 'foobar' 95 |func| must not be a generator, nor must it return a list of steps, etc.)
217 module.
218 96
219 Args: 97 When the decorated function is called, |func| is called normally. If we are
220 names (list): A list of module names to include in the returned RecipeApi. 98 in test mode, we will then also call self.test_api.<func.__name__>, whose
221 mod_dirs (list): A list of paths to directories which contain modules. 99 return value will be assigned into the step dictionary retuned by |func|.
222 mocks (dict): An optional dict of {<modname>: <mock data>}. Each module 100
223 expects its own mock data. 101 It is an error for the function to not exist in the test_api.
224 **kwargs: Data passed to each module api. Usually this will contain: 102 It is an error for the return value of |func| to already contain test data.
225 properties (dict): the properties dictionary (used by the properties
226 module)
227 step_history (OrderedDict): the step history object (used by the
228 step_history module!)
229 """ 103 """
230 104 @functools.wraps(func)
231 recipe_modules = load_recipe_modules(mod_dirs) 105 def inner(self, *args, **kwargs):
232 106 assert isinstance(self, RecipeApi)
233 inst_map = {None: RecipeApi()} 107 ret = func(self, *args, **kwargs)
234 dep_map = {None: set(names)} 108 if self._test_data.enabled: # pylint: disable=W0212
235 def create_maps(name): 109 test_fn = getattr(self.test_api, func.__name__, None)
236 if name not in dep_map: 110 assert test_fn, (
237 module = getattr(recipe_modules, name) 111 "Method %(meth)s in module %(mod)s is @inject_test_data, but test_api"
238 112 " does not contain %(meth)s."
239 dep_map[name] = set(module.DEPS) 113 % {
240 map(create_maps, dep_map[name]) 114 'meth': func.__name__,
241 115 'mod': self._module, # pylint: disable=W0212
242 mock = None if mocks is None else mocks.get(name, {}) 116 })
243 inst_map[name] = module.API(module=module, mock=mock, **kwargs) 117 assert 'default_step_data' not in ret
244 map(create_maps, names) 118 data = test_fn(*args, **kwargs)
245 119 assert isinstance(data, StepTestData)
246 # NOTE: this is 'inefficient', but correct and compact. 120 ret['default_step_data'] = data
247 did_something = True 121 return ret
248 while dep_map: 122 return inner
249 did_something = False
250 to_pop = []
251 for api_name, deps in dep_map.iteritems():
252 to_remove = []
253 for dep in [d for d in deps if d not in dep_map]:
254 # Grab the injection site
255 obj = inst_map[api_name].m
256 assert not hasattr(obj, dep)
257 setattr(obj, dep, inst_map[dep])
258 to_remove.append(dep)
259 did_something = True
260 map(deps.remove, to_remove)
261 if not deps:
262 to_pop.append(api_name)
263 did_something = True
264 map(dep_map.pop, to_pop)
265 assert did_something, 'Did nothing on this loop. %s' % dep_map
266
267 return inst_map[None]
268
269
270 def wrap_followup(kwargs, pre=False):
271 """
272 Decorator for a new followup_fn.
273
274 Will pop the existing fn out of kwargs (if any), and return a decorator for
275 the new folloup_fn.
276
277 Args:
278 kwargs - dictionary possibly containing folloup_fn
279 pre - If true, the old folloup_fn is called before the wrapped function.
280 Otherwise, the old followup_fn is called after the wrapped function.
281 """
282 null_fn = lambda _: None
283 old_followup = kwargs.pop('followup_fn', null_fn)
284 def decorator(f):
285 @functools.wraps(f)
286 def _inner(step_result):
287 if pre:
288 old_followup(step_result)
289 f(step_result)
290 else:
291 f(step_result)
292 old_followup(step_result)
293 if old_followup is not null_fn:
294 _inner.__name__ += '[%s]' % old_followup.__name__
295 return _inner
296 return decorator
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698