OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 | 2 |
3 # Copyright 2016 The LUCI Authors. All rights reserved. | 3 # Copyright 2016 The LUCI Authors. All rights reserved. |
4 # Use of this source code is governed under the Apache License, Version 2.0 | 4 # Use of this source code is governed under the Apache License, Version 2.0 |
5 # that can be found in the LICENSE file. | 5 # that can be found in the LICENSE file. |
6 | 6 |
7 """Bootstrap script to clone and forward to the recipe engine tool. | 7 """Bootstrap script to clone and forward to the recipe engine tool. |
8 | 8 |
9 *********************************************************************** | 9 *********************************************************************** |
10 ** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. ** | 10 ** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. ** |
(...skipping 14 matching lines...) Expand all Loading... | |
25 #### PER-REPO CONFIGURATION (editable) #### | 25 #### PER-REPO CONFIGURATION (editable) #### |
26 # The root of the repository relative to the directory of this file. | 26 # The root of the repository relative to the directory of this file. |
27 REPO_ROOT = None # os.path.join(os.pardir, os.pardir) | 27 REPO_ROOT = None # os.path.join(os.pardir, os.pardir) |
28 # The path of the recipes.cfg file relative to the root of the repository. | 28 # The path of the recipes.cfg file relative to the root of the repository. |
29 RECIPES_CFG = None # os.path.join('infra', 'config', 'recipes.cfg') | 29 RECIPES_CFG = None # os.path.join('infra', 'config', 'recipes.cfg') |
30 #### END PER-REPO CONFIGURATION #### | 30 #### END PER-REPO CONFIGURATION #### |
31 | 31 |
32 BOOTSTRAP_VERSION = 1 | 32 BOOTSTRAP_VERSION = 1 |
33 | 33 |
34 import argparse | 34 import argparse |
35 import ast | |
36 import json | 35 import json |
37 import logging | 36 import logging |
38 import random | 37 import random |
39 import re | |
40 import subprocess | 38 import subprocess |
41 import sys | 39 import sys |
42 import time | 40 import time |
43 import traceback | |
44 import urlparse | 41 import urlparse |
45 | 42 |
46 from cStringIO import StringIO | 43 from cStringIO import StringIO |
47 | 44 |
48 | 45 |
49 def parse(repo_root, recipes_cfg_path): | 46 def parse(repo_root, recipes_cfg_path): |
50 """Parse is transitional code which parses a recipes.cfg file as either jsonpb | 47 """Parse is transitional code which parses a recipes.cfg file as either jsonpb |
51 or as textpb. | 48 or as textpb. |
52 | 49 |
53 Args: | 50 Args: |
54 repo_root (str) - native path to the root of the repo we're trying to run | 51 repo_root (str) - native path to the root of the repo we're trying to run |
55 recipes for. | 52 recipes for. |
56 recipes_cfg_path (str) - native path to the recipes.cfg file to process. | 53 recipes_cfg_path (str) - native path to the recipes.cfg file to process. |
57 | 54 |
58 Returns (as tuple): | 55 Returns (as tuple): |
59 engine_url (str) - the url to the engine repo we want to use. | 56 engine_url (str) - the url to the engine repo we want to use. |
60 engine_revision (str) - the git revision for the engine to get. | 57 engine_revision (str) - the git revision for the engine to get. |
61 engine_subpath (str) - the subdirectory in the engine repo we should use to | 58 engine_subpath (str) - the subdirectory in the engine repo we should use to |
62 find it's recipes.py entrypoint. This is here for completeness, but will | 59 find it's recipes.py entrypoint. This is here for completeness, but will |
63 essentially always be empty. It would be used if the recipes-py repo was | 60 essentially always be empty. It would be used if the recipes-py repo was |
64 merged as a subdirectory of some other repo and you depended on that | 61 merged as a subdirectory of some other repo and you depended on that |
65 subdirectory. | 62 subdirectory. |
66 recipes_path (str) - native path to where the recipes live inside of the | 63 recipes_path (str) - native path to where the recipes live inside of the |
67 current repo (i.e. the folder containing `recipes/` and/or | 64 current repo (i.e. the folder containing `recipes/` and/or |
68 `recipe_modules`) | 65 `recipe_modules`) |
69 """ | 66 """ |
70 with open(recipes_cfg_path, 'rU') as fh: | 67 with open(recipes_cfg_path, 'rU') as fh: |
71 data = fh.read() | 68 pb = json.load(fh) |
72 | 69 |
73 if data.lstrip().startswith('{'): | 70 engine = next( |
74 pb = json.loads(data) | 71 (d for d in pb['deps'] if d['project_id'] == 'recipe_engine'), None) |
75 engine = next( | 72 if engine is None: |
76 (d for d in pb['deps'] if d['project_id'] == 'recipe_engine'), None) | 73 raise ValueError('could not find recipe_engine dep in %r' |
77 if engine is None: | 74 % recipes_cfg_path) |
78 raise ValueError('could not find recipe_engine dep in %r' | 75 engine_url = engine['url'] |
79 % recipes_cfg_path) | 76 engine_revision = engine.get('revision', '') |
80 engine_url = engine['url'] | 77 engine_subpath = engine.get('path_override', '') |
81 engine_revision = engine.get('revision', '') | 78 recipes_path = pb.get('recipes_path', '') |
82 engine_subpath = engine.get('path_override', '') | |
83 recipes_path = pb.get('recipes_path', '') | |
84 else: | |
85 def get_unique(things): | |
86 if len(things) == 1: | |
87 return things[0] | |
88 elif len(things) == 0: | |
89 raise ValueError("Expected to get one thing, but dinna get none.") | |
90 else: | |
91 logging.warn('Expected to get one thing, but got a bunch: %s\n%s' % | |
92 (things, traceback.format_stack())) | |
93 return things[0] | |
94 | |
95 protobuf = parse_textpb(StringIO(data)) | |
96 | |
97 engine_buf = get_unique([ | |
98 b for b in protobuf.get('deps', []) | |
99 if b.get('project_id') == ['recipe_engine'] ]) | |
100 engine_url = get_unique(engine_buf['url']) | |
101 engine_revision = get_unique(engine_buf.get('revision', [''])) | |
102 engine_subpath = (get_unique(engine_buf.get('path_override', [''])) | |
103 .replace('/', os.path.sep)) | |
104 recipes_path = get_unique(protobuf.get('recipes_path', [''])) | |
105 | 79 |
106 recipes_path = os.path.join(repo_root, recipes_path.replace('/', os.path.sep)) | 80 recipes_path = os.path.join(repo_root, recipes_path.replace('/', os.path.sep)) |
107 return engine_url, engine_revision, engine_subpath, recipes_path | 81 return engine_url, engine_revision, engine_subpath, recipes_path |
108 | 82 |
109 | 83 |
110 def parse_textpb(fh): | |
dnj
2017/03/25 01:54:42
Uh .. wow, I forgot about this. Nice!
| |
111 """Parse the protobuf text format just well enough to understand recipes.cfg. | |
112 | |
113 We don't use the protobuf library because we want to be as self-contained | |
114 as possible in this bootstrap, so it can be simply vendored into a client | |
115 repo. | |
116 | |
117 We assume all fields are repeated since we don't have a proto spec to work | |
118 with. | |
119 | |
120 Args: | |
121 fh: a filehandle containing the text format protobuf. | |
122 Returns: | |
123 A recursive dictionary of lists. | |
124 """ | |
125 def parse_atom(field, text): | |
126 if text == 'true': | |
127 return True | |
128 if text == 'false': | |
129 return False | |
130 | |
131 # repo_type is an enum. Since it does not have quotes, | |
132 # invoking literal_eval would fail. | |
133 if field == 'repo_type': | |
134 return text | |
135 | |
136 return ast.literal_eval(text) | |
137 | |
138 ret = {} | |
139 for line in fh: | |
140 line = line.strip() | |
141 m = re.match(r'(\w+)\s*:\s*(.*)', line) | |
142 if m: | |
143 ret.setdefault(m.group(1), []).append(parse_atom(m.group(1), m.group(2))) | |
144 continue | |
145 | |
146 m = re.match(r'(\w+)\s*{', line) | |
147 if m: | |
148 subparse = parse_textpb(fh) | |
149 ret.setdefault(m.group(1), []).append(subparse) | |
150 continue | |
151 | |
152 if line == '}': | |
153 return ret | |
154 if line == '': | |
155 continue | |
156 | |
157 raise ValueError('Could not understand line: <%s>' % line) | |
158 | |
159 return ret | |
160 | |
161 | |
162 def _subprocess_call(argv, **kwargs): | 84 def _subprocess_call(argv, **kwargs): |
163 logging.info('Running %r', argv) | 85 logging.info('Running %r', argv) |
164 return subprocess.call(argv, **kwargs) | 86 return subprocess.call(argv, **kwargs) |
165 | 87 |
166 | 88 |
167 def _subprocess_check_call(argv, **kwargs): | 89 def _subprocess_check_call(argv, **kwargs): |
168 logging.info('Running %r', argv) | 90 logging.info('Running %r', argv) |
169 subprocess.check_call(argv, **kwargs) | 91 subprocess.check_call(argv, **kwargs) |
170 | 92 |
171 | 93 |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
240 time.sleep(random.uniform(2,5)) | 162 time.sleep(random.uniform(2,5)) |
241 ensure_engine() | 163 ensure_engine() |
242 | 164 |
243 args = ['--package', recipes_cfg_path] + sys.argv[1:] | 165 args = ['--package', recipes_cfg_path] + sys.argv[1:] |
244 return _subprocess_call([ | 166 return _subprocess_call([ |
245 sys.executable, '-u', | 167 sys.executable, '-u', |
246 os.path.join(engine_path, 'recipes.py')] + args) | 168 os.path.join(engine_path, 'recipes.py')] + args) |
247 | 169 |
248 if __name__ == '__main__': | 170 if __name__ == '__main__': |
249 sys.exit(main()) | 171 sys.exit(main()) |
OLD | NEW |