OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 """Recipe module to ensure a checkout is consistant on a bot.""" | 6 """Recipe module to ensure a checkout is consistant on a bot.""" |
7 | 7 |
8 from recipe_engine import recipe_api | 8 from recipe_engine import recipe_api |
9 | 9 |
10 | 10 |
(...skipping 30 matching lines...) Expand all Loading... |
41 ret += ']' | 41 ret += ']' |
42 elif isinstance(spec, basestring): | 42 elif isinstance(spec, basestring): |
43 ret = repr(str(spec)) | 43 ret = repr(str(spec)) |
44 else: | 44 else: |
45 ret = repr(spec) | 45 ret = repr(spec) |
46 return ret | 46 return ret |
47 | 47 |
48 | 48 |
49 class BotUpdateApi(recipe_api.RecipeApi): | 49 class BotUpdateApi(recipe_api.RecipeApi): |
50 | 50 |
51 def __init__(self, *args, **kwargs): | 51 def __init__(self, mastername, buildername, slavename, issue, patchset, |
52 self._properties = {} | 52 patch_url, repository, gerrit_ref, rietveld, revision, |
53 super(BotUpdateApi, self).__init__(*args, **kwargs) | 53 parent_got_revision, deps_revision_overrides, fail_patch, |
| 54 *args, **kwargs): |
| 55 self._mastername = mastername |
| 56 self._buildername = buildername |
| 57 self._slavename = slavename |
| 58 self._issue = issue |
| 59 self._patchset = patchset |
| 60 self._patch_url = patch_url |
| 61 self._repository = repository |
| 62 self._gerrit_ref = gerrit_ref |
| 63 self._rietveld = rietveld |
| 64 self._revision = revision |
| 65 self._parent_got_revision = parent_got_revision |
| 66 self._deps_revision_overrides = deps_revision_overrides |
| 67 self._fail_patch = fail_patch |
| 68 |
| 69 self._last_returned_properties = {} |
| 70 super(BotUpdateApi, self).__init__(*args, **kwargs) |
54 | 71 |
55 def __call__(self, name, cmd, **kwargs): | 72 def __call__(self, name, cmd, **kwargs): |
56 """Wrapper for easy calling of bot_update.""" | 73 """Wrapper for easy calling of bot_update.""" |
57 assert isinstance(cmd, (list, tuple)) | 74 assert isinstance(cmd, (list, tuple)) |
58 bot_update_path = self.resource('bot_update.py') | 75 bot_update_path = self.resource('bot_update.py') |
59 kwargs.setdefault('infra_step', True) | 76 kwargs.setdefault('infra_step', True) |
60 kwargs.setdefault('env', {}) | 77 kwargs.setdefault('env', {}) |
61 kwargs['env'].setdefault('PATH', '%(PATH)s') | 78 kwargs['env'].setdefault('PATH', '%(PATH)s') |
62 kwargs['env']['PATH'] = self.m.path.pathsep.join([ | 79 kwargs['env']['PATH'] = self.m.path.pathsep.join([ |
63 kwargs['env']['PATH'], str(self._module.PACKAGE_REPO_ROOT)]) | 80 kwargs['env']['PATH'], str(self._module.PACKAGE_REPO_ROOT)]) |
64 return self.m.python(name, bot_update_path, cmd, **kwargs) | 81 return self.m.python(name, bot_update_path, cmd, **kwargs) |
65 | 82 |
66 @property | 83 @property |
67 def properties(self): | 84 def last_returned_properties(self): |
68 return self._properties | 85 return self._last_returned_properties |
69 | 86 |
70 def ensure_checkout(self, gclient_config=None, suffix=None, | 87 def ensure_checkout(self, gclient_config=None, suffix=None, |
71 patch=True, update_presentation=True, | 88 patch=True, update_presentation=True, |
72 force=False, patch_root=None, no_shallow=False, | 89 force=False, patch_root=None, no_shallow=False, |
73 with_branch_heads=False, refs=None, | 90 with_branch_heads=False, refs=None, |
74 patch_project_roots=None, patch_oauth2=False, | 91 patch_project_roots=None, patch_oauth2=False, |
75 output_manifest=True, clobber=False, | 92 output_manifest=True, clobber=False, |
76 root_solution_revision=None, **kwargs): | 93 root_solution_revision=None, **kwargs): |
77 refs = refs or [] | 94 refs = refs or [] |
78 # We can re-use the gclient spec from the gclient module, since all the | 95 # We can re-use the gclient spec from the gclient module, since all the |
79 # data bot_update needs is already configured into the gclient spec. | 96 # data bot_update needs is already configured into the gclient spec. |
80 cfg = gclient_config or self.m.gclient.c | 97 cfg = gclient_config or self.m.gclient.c |
81 spec_string = jsonish_to_python(cfg.as_jsonish(), True) | 98 spec_string = jsonish_to_python(cfg.as_jsonish(), True) |
82 | 99 |
83 # Used by bot_update to determine if we want to run or not. | 100 # Used by bot_update to determine if we want to run or not. |
84 master = self.m.properties['mastername'] | 101 master = self._mastername |
85 builder = self.m.properties['buildername'] | 102 builder = self._buildername |
86 slave = self.m.properties['slavename'] | 103 slave = self._slavename |
87 | 104 |
88 # Construct our bot_update command. This basically be inclusive of | 105 # Construct our bot_update command. This basically be inclusive of |
89 # everything required for bot_update to know: | 106 # everything required for bot_update to know: |
90 root = patch_root | 107 root = patch_root |
91 if root is None: | 108 if root is None: |
92 root = cfg.solutions[0].name | 109 root = cfg.solutions[0].name |
93 additional = self.m.rietveld.calculate_issue_root(patch_project_roots) | 110 additional = self.m.rietveld.calculate_issue_root(patch_project_roots) |
94 if additional: | 111 if additional: |
95 root = self.m.path.join(root, additional) | 112 root = self.m.path.join(root, additional) |
96 | 113 |
97 if patch: | 114 if patch: |
98 issue = self.m.properties.get('issue') | 115 issue = self._issue |
99 patchset = self.m.properties.get('patchset') | 116 patchset = self._patchset |
100 patch_url = self.m.properties.get('patch_url') | 117 patch_url = self._patch_url |
101 gerrit_repo = self.m.properties.get('repository') | 118 gerrit_repo = self._repository |
102 gerrit_ref = self.m.properties.get('event.patchSet.ref') | 119 gerrit_ref = self._gerrit_ref |
103 else: | 120 else: |
104 # The trybot recipe sometimes wants to de-apply the patch. In which case | 121 # The trybot recipe sometimes wants to de-apply the patch. In which case |
105 # we pretend the issue/patchset/patch_url never existed. | 122 # we pretend the issue/patchset/patch_url never existed. |
106 issue = patchset = patch_url = email_file = key_file = None | 123 issue = patchset = patch_url = email_file = key_file = None |
107 gerrit_repo = gerrit_ref = None | 124 gerrit_repo = gerrit_ref = None |
108 | 125 |
109 # Issue and patchset must come together. | 126 # Issue and patchset must come together. |
110 if issue: | 127 if issue: |
111 assert patchset | 128 assert patchset |
112 if patchset: | 129 if patchset: |
(...skipping 29 matching lines...) Expand all Loading... |
142 # 2. What do we want to check out (spec/root/rev/rev_map). | 159 # 2. What do we want to check out (spec/root/rev/rev_map). |
143 ['--spec', spec_string], | 160 ['--spec', spec_string], |
144 ['--root', root], | 161 ['--root', root], |
145 ['--revision_mapping_file', self.m.json.input(rev_map)], | 162 ['--revision_mapping_file', self.m.json.input(rev_map)], |
146 ['--git-cache-dir', cfg.cache_dir], | 163 ['--git-cache-dir', cfg.cache_dir], |
147 | 164 |
148 # 3. How to find the patch, if any (issue/patchset/patch_url). | 165 # 3. How to find the patch, if any (issue/patchset/patch_url). |
149 ['--issue', issue], | 166 ['--issue', issue], |
150 ['--patchset', patchset], | 167 ['--patchset', patchset], |
151 ['--patch_url', patch_url], | 168 ['--patch_url', patch_url], |
152 ['--rietveld_server', self.m.properties.get('rietveld')], | 169 ['--rietveld_server', self._rietveld], |
153 ['--gerrit_repo', gerrit_repo], | 170 ['--gerrit_repo', gerrit_repo], |
154 ['--gerrit_ref', gerrit_ref], | 171 ['--gerrit_ref', gerrit_ref], |
155 ['--apply_issue_email_file', email_file], | 172 ['--apply_issue_email_file', email_file], |
156 ['--apply_issue_key_file', key_file], | 173 ['--apply_issue_key_file', key_file], |
157 | 174 |
158 # 4. Hookups to JSON output back into recipes. | 175 # 4. Hookups to JSON output back into recipes. |
159 ['--output_json', self.m.json.output()],] | 176 ['--output_json', self.m.json.output()],] |
160 | 177 |
161 | 178 |
162 # Collect all fixed revisions to simulate them in the json output. | 179 # Collect all fixed revisions to simulate them in the json output. |
163 # Fixed revision are the explicit input revisions of bot_update.py, i.e. | 180 # Fixed revision are the explicit input revisions of bot_update.py, i.e. |
164 # every command line parameter "--revision name@value". | 181 # every command line parameter "--revision name@value". |
165 fixed_revisions = {} | 182 fixed_revisions = {} |
166 | 183 |
167 revisions = {} | 184 revisions = {} |
168 for solution in cfg.solutions: | 185 for solution in cfg.solutions: |
169 if solution.revision: | 186 if solution.revision: |
170 revisions[solution.name] = solution.revision | 187 revisions[solution.name] = solution.revision |
171 elif solution == cfg.solutions[0]: | 188 elif solution == cfg.solutions[0]: |
172 revisions[solution.name] = ( | 189 revisions[solution.name] = ( |
173 self.m.properties.get('parent_got_revision') or | 190 self._parent_got_revision or |
174 self.m.properties.get('revision') or | 191 self._revision or |
175 'HEAD') | 192 'HEAD') |
176 if self.m.gclient.c and self.m.gclient.c.revisions: | 193 if self.m.gclient.c and self.m.gclient.c.revisions: |
177 revisions.update(self.m.gclient.c.revisions) | 194 revisions.update(self.m.gclient.c.revisions) |
178 if cfg.solutions and root_solution_revision: | 195 if cfg.solutions and root_solution_revision: |
179 revisions[cfg.solutions[0].name] = root_solution_revision | 196 revisions[cfg.solutions[0].name] = root_solution_revision |
180 # Allow for overrides required to bisect into rolls. | 197 # Allow for overrides required to bisect into rolls. |
181 revisions.update(self.m.properties.get('deps_revision_overrides', {})) | 198 revisions.update(self._deps_revision_overrides) |
182 for name, revision in sorted(revisions.items()): | 199 for name, revision in sorted(revisions.items()): |
183 fixed_revision = self.m.gclient.resolve_revision(revision) | 200 fixed_revision = self.m.gclient.resolve_revision(revision) |
184 if fixed_revision: | 201 if fixed_revision: |
185 fixed_revisions[name] = fixed_revision | 202 fixed_revisions[name] = fixed_revision |
186 flags.append(['--revision', '%s@%s' % (name, fixed_revision)]) | 203 flags.append(['--revision', '%s@%s' % (name, fixed_revision)]) |
187 | 204 |
188 # Add extra fetch refspecs. | 205 # Add extra fetch refspecs. |
189 for ref in refs: | 206 for ref in refs: |
190 flags.append(['--refs', ref]) | 207 flags.append(['--refs', ref]) |
191 | 208 |
192 # Filter out flags that are None. | 209 # Filter out flags that are None. |
193 cmd = [item for flag_set in flags | 210 cmd = [item for flag_set in flags |
194 for item in flag_set if flag_set[1] is not None] | 211 for item in flag_set if flag_set[1] is not None] |
195 | 212 |
196 if clobber: | 213 if clobber: |
197 cmd.append('--clobber') | 214 cmd.append('--clobber') |
198 if force: | 215 if force: |
199 cmd.append('--force') | 216 cmd.append('--force') |
200 if no_shallow: | 217 if no_shallow: |
201 cmd.append('--no_shallow') | 218 cmd.append('--no_shallow') |
202 if output_manifest: | 219 if output_manifest: |
203 cmd.append('--output_manifest') | 220 cmd.append('--output_manifest') |
204 if with_branch_heads or cfg.with_branch_heads: | 221 if with_branch_heads or cfg.with_branch_heads: |
205 cmd.append('--with_branch_heads') | 222 cmd.append('--with_branch_heads') |
206 | 223 |
207 # Inject Json output for testing. | 224 # Inject Json output for testing. |
208 git_mode = self.m.properties.get('mastername') not in SVN_MASTERS | 225 git_mode = self._mastername not in SVN_MASTERS |
209 first_sln = cfg.solutions[0].name | 226 first_sln = cfg.solutions[0].name |
210 step_test_data = lambda: self.test_api.output_json( | 227 step_test_data = lambda: self.test_api.output_json( |
211 master, builder, slave, root, first_sln, rev_map, git_mode, force, | 228 master, builder, slave, root, first_sln, rev_map, git_mode, force, |
212 self.m.properties.get('fail_patch', False), | 229 self._fail_patch, |
213 output_manifest=output_manifest, fixed_revisions=fixed_revisions) | 230 output_manifest=output_manifest, fixed_revisions=fixed_revisions) |
214 | 231 |
215 # Add suffixes to the step name, if specified. | 232 # Add suffixes to the step name, if specified. |
216 name = 'bot_update' | 233 name = 'bot_update' |
217 if not patch: | 234 if not patch: |
218 name += ' (without patch)' | 235 name += ' (without patch)' |
219 if suffix: | 236 if suffix: |
220 name += ' - %s' % suffix | 237 name += ' - %s' % suffix |
221 | 238 |
222 # Ah hah! Now that everything is in place, lets run bot_update! | 239 # Ah hah! Now that everything is in place, lets run bot_update! |
223 try: | 240 try: |
224 # 87 and 88 are the 'patch failure' codes for patch download and patch | 241 # 87 and 88 are the 'patch failure' codes for patch download and patch |
225 # apply, respectively. We don't actually use the error codes, and instead | 242 # apply, respectively. We don't actually use the error codes, and instead |
226 # rely on emitted json to determine cause of failure. | 243 # rely on emitted json to determine cause of failure. |
227 self(name, cmd, step_test_data=step_test_data, | 244 self(name, cmd, step_test_data=step_test_data, |
228 ok_ret=(0, 87, 88), **kwargs) | 245 ok_ret=(0, 87, 88), **kwargs) |
229 finally: | 246 finally: |
230 step_result = self.m.step.active_result | 247 step_result = self.m.step.active_result |
231 self._properties = step_result.json.output.get('properties', {}) | 248 self._last_returned_properties = step_result.json.output.get( |
| 249 'properties', {}) |
232 | 250 |
233 if update_presentation: | 251 if update_presentation: |
234 # Set properties such as got_revision. | 252 # Set properties such as got_revision. |
235 for prop_name, prop_value in self.properties.iteritems(): | 253 for prop_name, prop_value in self.last_returned_properties.iteritems(): |
236 step_result.presentation.properties[prop_name] = prop_value | 254 step_result.presentation.properties[prop_name] = prop_value |
237 # Add helpful step description in the step UI. | 255 # Add helpful step description in the step UI. |
238 if 'step_text' in step_result.json.output: | 256 if 'step_text' in step_result.json.output: |
239 step_text = step_result.json.output['step_text'] | 257 step_text = step_result.json.output['step_text'] |
240 step_result.presentation.step_text = step_text | 258 step_result.presentation.step_text = step_text |
241 # Add log line output. | 259 # Add log line output. |
242 if 'log_lines' in step_result.json.output: | 260 if 'log_lines' in step_result.json.output: |
243 for log_name, log_lines in step_result.json.output['log_lines']: | 261 for log_name, log_lines in step_result.json.output['log_lines']: |
244 step_result.presentation.logs[log_name] = log_lines.splitlines() | 262 step_result.presentation.logs[log_name] = log_lines.splitlines() |
245 | 263 |
(...skipping 24 matching lines...) Expand all Loading... |
270 | 288 |
271 # bot_update actually just sets root to be the folder name of the | 289 # bot_update actually just sets root to be the folder name of the |
272 # first solution. | 290 # first solution. |
273 if step_result.json.output['did_run']: | 291 if step_result.json.output['did_run']: |
274 co_root = step_result.json.output['root'] | 292 co_root = step_result.json.output['root'] |
275 cwd = kwargs.get('cwd', self.m.path['slave_build']) | 293 cwd = kwargs.get('cwd', self.m.path['slave_build']) |
276 if 'checkout' not in self.m.path: | 294 if 'checkout' not in self.m.path: |
277 self.m.path['checkout'] = cwd.join(*co_root.split(self.m.path.sep)) | 295 self.m.path['checkout'] = cwd.join(*co_root.split(self.m.path.sep)) |
278 | 296 |
279 return step_result | 297 return step_result |
OLD | NEW |