| OLD | NEW |
| 1 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Rolls recipes.cfg dependencies.""" | 5 """Rolls recipes.cfg dependencies.""" |
| 6 | 6 |
| 7 DEPS = [ | 7 DEPS = [ |
| 8 'depot_tools/bot_update', | 8 'depot_tools/bot_update', |
| 9 'depot_tools/gclient', | 9 'depot_tools/gclient', |
| 10 'file', | 10 'file', |
| (...skipping 130 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 141 for proj, targets in deps.items(): | 141 for proj, targets in deps.items(): |
| 142 for target in targets: | 142 for target in targets: |
| 143 downstream_projects[target].add(proj) | 143 downstream_projects[target].add(proj) |
| 144 | 144 |
| 145 return deps, downstream_projects | 145 return deps, downstream_projects |
| 146 | 146 |
| 147 def get_url_mapping(api, headers=None): | 147 def get_url_mapping(api, headers=None): |
| 148 """Fetch the mapping of project id to url from luci-config. | 148 """Fetch the mapping of project id to url from luci-config. |
| 149 | 149 |
| 150 Args: | 150 Args: |
| 151 auth_token: an OAuth2 access token for querying luci-config. | 151 headers: Optional authentication headers to pass to luci-config. |
| 152 | 152 |
| 153 Returns: | 153 Returns: |
| 154 A dictionary mapping project id to its luci-config project spec (among | 154 A dictionary mapping project id to its luci-config project spec (among |
| 155 which there is a repo_url key). | 155 which there is a repo_url key). |
| 156 """ | 156 """ |
| 157 url = 'https://luci-config.appspot.com/_ah/api/config/v1/projects' | 157 url = 'https://luci-config.appspot.com/_ah/api/config/v1/projects' |
| 158 | 158 |
| 159 fetch_result = api.url.fetch(url, step_name='Get project urls', | 159 fetch_result = api.url.fetch(url, step_name='Get project urls', |
| 160 headers=headers, | 160 headers=headers, |
| 161 step_test_data=lambda: api.raw_io.test_api.output(json.dumps({ | 161 step_test_data=lambda: api.raw_io.test_api.output(json.dumps({ |
| (...skipping 15 matching lines...) Expand all Loading... |
| 177 for project in json.loads(fetch_result)['projects']: | 177 for project in json.loads(fetch_result)['projects']: |
| 178 project = {str(k): str(v) for k, v in project.items()} | 178 project = {str(k): str(v) for k, v in project.items()} |
| 179 mapping[project['id']] = project | 179 mapping[project['id']] = project |
| 180 return mapping | 180 return mapping |
| 181 | 181 |
| 182 | 182 |
| 183 RietveldPatch = collections.namedtuple( | 183 RietveldPatch = collections.namedtuple( |
| 184 'RietveldPatch', 'project server issue patchset') | 184 'RietveldPatch', 'project server issue patchset') |
| 185 | 185 |
| 186 | 186 |
| 187 def parse_patches(api, patches_raw): | 187 def parse_patches(api, patches_raw, rietveld, issue, patchset, patch_project): |
| 188 """ | 188 """ |
| 189 gives mapping of project to patch | 189 gives mapping of project to patch |
| 190 expect input of | 190 expect input of |
| 191 project1:https://a.b.c/1342342#ps1,project2:https://d.ce.f/1231231#ps1 | 191 project1:https://a.b.c/1342342#ps1,project2:https://d.ce.f/1231231#ps1 |
| 192 """ | 192 """ |
| 193 result = {} |
| 194 |
| 195 if rietveld and issue and patchset and patch_project: |
| 196 result[patch_project] = RietveldPatch( |
| 197 patch_project, rietveld, issue, patchset) |
| 198 |
| 193 if not patches_raw: | 199 if not patches_raw: |
| 194 return {} | 200 return result |
| 195 | |
| 196 result = {} | |
| 197 | 201 |
| 198 for patch_raw in patches_raw.split(','): | 202 for patch_raw in patches_raw.split(','): |
| 199 project, url = patch_raw.split(':', 1) | 203 project, url = patch_raw.split(':', 1) |
| 200 server, issue_and_patchset = url.rsplit('/', 1) | 204 server, issue_and_patchset = url.rsplit('/', 1) |
| 201 issue, patchset = issue_and_patchset.split('#') | 205 issue, patchset = issue_and_patchset.split('#') |
| 202 patchset = patchset[2:] | 206 patchset = patchset[2:] |
| 203 | 207 |
| 204 if project in result: | 208 if project in result: |
| 205 api.python.failing_step( | 209 api.python.failing_step( |
| 206 "Invalid patchset list", | 210 "Invalid patchset list", |
| (...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 295 | 299 |
| 296 kwargs = { | 300 kwargs = { |
| 297 'gclient_config': config, | 301 'gclient_config': config, |
| 298 'force': True, | 302 'force': True, |
| 299 'cwd': checkout_path, | 303 'cwd': checkout_path, |
| 300 } | 304 } |
| 301 if patch: | 305 if patch: |
| 302 kwargs['rietveld'] = patch.server | 306 kwargs['rietveld'] = patch.server |
| 303 kwargs['issue'] = patch.issue | 307 kwargs['issue'] = patch.issue |
| 304 kwargs['patchset'] = patch.patchset | 308 kwargs['patchset'] = patch.patchset |
| 309 else: |
| 310 kwargs['patch'] = False |
| 305 | 311 |
| 306 api.bot_update.ensure_checkout(**kwargs) | 312 api.bot_update.ensure_checkout(**kwargs) |
| 307 return repo_path | 313 return repo_path |
| 308 | 314 |
| 309 | 315 |
| 310 PROJECTS_TO_TRY = [ | 316 PROJECTS_TO_TRY = [ |
| 311 'build', | 317 'build', |
| 312 'build_limited_scripts_slave', | 318 'build_limited_scripts_slave', |
| 313 'recipe_engine', | 319 'recipe_engine', |
| 314 'depot_tools', | 320 'depot_tools', |
| 315 ] | 321 ] |
| 316 | 322 |
| 317 PROPERTIES = { | 323 PROPERTIES = { |
| 318 'patches': Property(kind=str, param_name='patches_raw', default="", | 324 'patches': Property(kind=str, param_name='patches_raw', default="", |
| 319 help="Patches to apply. Format is" | 325 help="Patches to apply. Format is" |
| 320 "project1:https://url.to.codereview/123456#ps01 where" | 326 "project1:https://url.to.codereview/123456#ps01 where" |
| 321 "url.to.codereview is the address of the code review site" | 327 "url.to.codereview is the address of the code review site" |
| 322 ", 123456 is the issue number, and ps01 is the patchset" | 328 ", 123456 is the issue number, and ps01 is the patchset" |
| 323 "number"), | 329 "number"), |
| 330 # This recipe can be used as a tryjob by setting the rietveld, issue, and |
| 331 # patchset properties, like a normal tryjob. If those are set, it will use |
| 332 # those, as well as any data sent in the regular properties, as patches to |
| 333 # apply. |
| 334 "rietveld": Property(kind=str, default="", |
| 335 help="The Rietveld instance the issue is from"), |
| 336 "issue": Property(kind=int, default=None, |
| 337 help="The Rietveld issue number to pull data from"), |
| 338 "patchset": Property(kind=int, default=None, |
| 339 help="The patchset number for the supplied issue"), |
| 340 "patch_project": Property( |
| 341 kind=str, default=None, |
| 342 help="The luci-config name of the project this patch belongs to"), |
| 324 } | 343 } |
| 325 | 344 |
| 326 def RunSteps(api, patches_raw): | 345 def RunSteps(api, patches_raw, rietveld, issue, patchset, patch_project): |
| 327 headers = {'Authorization': 'Bearer %s' % get_auth_token(api)} | 346 headers = {'Authorization': 'Bearer %s' % get_auth_token(api)} |
| 328 | 347 |
| 329 patches = parse_patches(api, patches_raw) | 348 patches = parse_patches( |
| 349 api, patches_raw, rietveld, issue, patchset, patch_project) |
| 330 | 350 |
| 331 root_dir = api.path['slave_build'] | 351 root_dir = api.path['slave_build'] |
| 332 | 352 |
| 333 url_mapping = get_url_mapping(api, headers) | 353 url_mapping = get_url_mapping(api, headers) |
| 334 | 354 |
| 335 # luci config project name to recipe config namedtuple | 355 # luci config project name to recipe config namedtuple |
| 336 recipe_configs = {} | 356 recipe_configs = {} |
| 337 | 357 |
| 338 # List of all the projects we care about testing. luci-config names | 358 # List of all the projects we care about testing. luci-config names |
| 339 all_projects = set(p for p in url_mapping if p in PROJECTS_TO_TRY) | 359 all_projects = set(p for p in url_mapping if p in PROJECTS_TO_TRY) |
| 340 | 360 |
| 341 recipe_configs = { | 361 recipe_configs = { |
| 342 p: get_project_config(api, p, headers) for p in all_projects} | 362 p: get_project_config(api, p, headers) for p in all_projects} |
| 343 | 363 |
| 344 downstream_projects, deps = get_deps_info(all_projects, recipe_configs) | 364 deps, downstream_projects = get_deps_info(all_projects, recipe_configs) |
| 345 | 365 |
| 346 projs_to_test, locations = checkout_projects( | 366 projs_to_test, locations = checkout_projects( |
| 347 api, all_projects, url_mapping, downstream_projects, root_dir, patches) | 367 api, all_projects, url_mapping, downstream_projects, root_dir, patches) |
| 348 | 368 |
| 349 with api.step.defer_results(): | 369 with api.step.defer_results(): |
| 350 for proj in projs_to_test: | 370 for proj in projs_to_test: |
| 351 deps_locs = {dep: locations[dep] for dep in deps[proj]} | 371 deps_locs = {dep: locations[dep] for dep in deps[proj]} |
| 352 | 372 |
| 353 simulation_test( | 373 simulation_test( |
| 354 api, proj, recipe_configs[proj], locations[proj], deps_locs) | 374 api, proj, recipe_configs[proj], locations[proj], deps_locs) |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 403 api.properties( | 423 api.properties( |
| 404 patches="build:https://f.e.w/1#ps1,build:https://f.e.w/1#ps1") | 424 patches="build:https://f.e.w/1#ps1,build:https://f.e.w/1#ps1") |
| 405 ) | 425 ) |
| 406 | 426 |
| 407 yield ( | 427 yield ( |
| 408 api.test('deps') + | 428 api.test('deps') + |
| 409 api.step_data("Get build deps", project('build', ['recipe_engine'])) + | 429 api.step_data("Get build deps", project('build', ['recipe_engine'])) + |
| 410 api.step_data("Get recipe_engine deps", project('recipe_engine')) | 430 api.step_data("Get recipe_engine deps", project('recipe_engine')) |
| 411 ) | 431 ) |
| 412 | 432 |
| 433 yield ( |
| 434 api.test('tryjob') + |
| 435 api.properties( |
| 436 rietveld="https://fake.code.review", |
| 437 issue=12345678, |
| 438 patchset=1, |
| 439 patch_project="build", |
| 440 ) + |
| 441 api.step_data("Get build deps", project('build', ['recipe_engine'])) + |
| 442 api.step_data("Get recipe_engine deps", project('recipe_engine')) |
| 443 ) |
| 444 |
| OLD | NEW |