OLD | NEW |
1 # Copyright 2014 The Swarming Authors. All rights reserved. | 1 # Copyright 2014 The Swarming Authors. All rights reserved. |
2 # Use of this source code is governed by the Apache v2.0 license that can be | 2 # Use of this source code is governed by the Apache v2.0 license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Imports groups from some external tar.gz bundle or plain text list. | 5 """Imports groups from some external tar.gz bundle or plain text list. |
6 | 6 |
7 External URL should serve *.tar.gz file with the following file structure: | 7 External URL should serve *.tar.gz file with the following file structure: |
8 <external group system name>/<group name>: | 8 <external group system name>/<group name>: |
9 userid | 9 userid |
10 userid | 10 userid |
(...skipping 17 matching lines...) Expand all Loading... |
28 the tarball. | 28 the tarball. |
29 | 29 |
30 Plain list format should have one userid per line and can only describe a single | 30 Plain list format should have one userid per line and can only describe a single |
31 group in a single system. Such groups will be added to 'external/*' groups | 31 group in a single system. Such groups will be added to 'external/*' groups |
32 namespace. Removing such group from importer config will remove it from | 32 namespace. Removing such group from importer config will remove it from |
33 service too. | 33 service too. |
34 """ | 34 """ |
35 | 35 |
36 import collections | 36 import collections |
37 import contextlib | 37 import contextlib |
| 38 import json |
38 import logging | 39 import logging |
39 import StringIO | 40 import StringIO |
40 import tarfile | 41 import tarfile |
41 | 42 |
42 from google.appengine.ext import ndb | 43 from google.appengine.ext import ndb |
43 | 44 |
| 45 from google import protobuf |
| 46 |
44 from components import auth | 47 from components import auth |
45 from components import net | 48 from components import net |
46 from components import utils | 49 from components import utils |
47 from components.auth import model | 50 from components.auth import model |
48 | 51 |
| 52 from proto import config_pb2 |
| 53 |
49 | 54 |
50 class BundleImportError(Exception): | 55 class BundleImportError(Exception): |
51 """Base class for errors while fetching external bundle.""" | 56 """Base class for errors while fetching external bundle.""" |
52 | 57 |
53 | 58 |
54 class BundleFetchError(BundleImportError): | 59 class BundleFetchError(BundleImportError): |
55 """Failed to fetch the archive from remote URL.""" | 60 """Failed to fetch the archive from remote URL.""" |
56 | 61 |
57 def __init__(self, url, status_code, content): | 62 def __init__(self, url, status_code, content): |
58 super(BundleFetchError, self).__init__() | 63 super(BundleFetchError, self).__init__() |
(...skipping 28 matching lines...) Expand all Loading... |
87 return 'Bundle contains invalid group file: %s' % self.inner_exc | 92 return 'Bundle contains invalid group file: %s' % self.inner_exc |
88 | 93 |
89 | 94 |
90 def config_key(): | 95 def config_key(): |
91 """Key of GroupImporterConfig singleton entity.""" | 96 """Key of GroupImporterConfig singleton entity.""" |
92 return ndb.Key('GroupImporterConfig', 'config') | 97 return ndb.Key('GroupImporterConfig', 'config') |
93 | 98 |
94 | 99 |
95 class GroupImporterConfig(ndb.Model): | 100 class GroupImporterConfig(ndb.Model): |
96 """Singleton entity with group importer configuration JSON.""" | 101 """Singleton entity with group importer configuration JSON.""" |
97 config = ndb.JsonProperty() | 102 config = ndb.TextProperty() # legacy field with JSON config |
| 103 config_proto = ndb.TextProperty() |
98 modified_by = auth.IdentityProperty(indexed=False) | 104 modified_by = auth.IdentityProperty(indexed=False) |
99 modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False) | 105 modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False) |
100 | 106 |
101 | 107 |
102 def is_valid_config(config): | 108 def legacy_json_config_to_proto(config_json): |
103 """Checks config for correctness.""" | 109 """Converts legacy JSON config to config_pb2.GroupImporterConfig message. |
104 if not isinstance(config, list): | |
105 return False | |
106 | 110 |
107 seen_systems = set(['external']) | 111 TODO(vadimsh): Remove once all instances of auth service use protobuf configs. |
108 seen_groups = set() | 112 """ |
| 113 try: |
| 114 config = json.loads(config_json) |
| 115 except ValueError as ex: |
| 116 logging.error('Invalid JSON: %s', ex) |
| 117 return None |
| 118 msg = config_pb2.GroupImporterConfig() |
109 for item in config: | 119 for item in config: |
110 if not isinstance(item, dict): | |
111 return False | |
112 | |
113 # 'format' is an optional string describing the format of the imported | |
114 # source. The default format is 'tarball'. | |
115 fmt = item.get('format', 'tarball') | 120 fmt = item.get('format', 'tarball') |
116 if fmt not in ['tarball', 'plainlist']: | |
117 return False | |
118 | |
119 # 'url' is a required string: where to fetch groups from. | |
120 url = item.get('url') | |
121 if not url or not isinstance(url, basestring): | |
122 return False | |
123 | |
124 # 'oauth_scopes' is an optional list of strings: used when generating OAuth | |
125 # access_token to put in Authorization header. | |
126 oauth_scopes = item.get('oauth_scopes') | |
127 if oauth_scopes is not None: | |
128 if not all(isinstance(x, basestring) for x in oauth_scopes): | |
129 return False | |
130 | |
131 # 'domain' is an optional string: will be used when constructing emails from | |
132 # naked usernames found in imported groups. | |
133 domain = item.get('domain') | |
134 if domain and not isinstance(domain, basestring): | |
135 return False | |
136 | |
137 # 'tarball' format uses 'systems' and 'groups' fields. | |
138 if fmt == 'tarball': | 121 if fmt == 'tarball': |
139 # 'systems' is a required list of strings: group systems expected to be | 122 entry = msg.tarball.add() |
140 # found in the archive (they act as prefixes to group names, e.g 'ldap'). | |
141 systems = item.get('systems') | |
142 if not systems or not isinstance(systems, list): | |
143 return False | |
144 if not all(isinstance(x, basestring) for x in systems): | |
145 return False | |
146 | |
147 # There should be no overlap in systems between different bundles. | |
148 if set(systems) & seen_systems: | |
149 return False | |
150 seen_systems.update(systems) | |
151 | |
152 # 'groups' is an optional list of strings: if given, filters imported | |
153 # groups only to this list. | |
154 groups = item.get('groups') | |
155 if groups and not all(isinstance(x, basestring) for x in groups): | |
156 return False | |
157 elif fmt == 'plainlist': | 123 elif fmt == 'plainlist': |
158 # 'group' is a required name of imported group. The full group name will | 124 entry = msg.plainlist.add() |
159 # be 'external/<group>'. | |
160 group = item.get('group') | |
161 if not group or not isinstance(group, basestring) or group in seen_groups: | |
162 return False | |
163 seen_groups.add(group) | |
164 else: | 125 else: |
165 assert False, 'Unreachable' | 126 logging.error('Unrecognized format: %s', fmt) |
166 | 127 continue |
167 return True | 128 entry.url = item.get('url') or '' |
| 129 entry.oauth_scopes.extend(item.get('oauth_scopes') or []) |
| 130 if 'domain' in item: |
| 131 entry.domain = item['domain'] |
| 132 if fmt == 'tarball': |
| 133 entry.systems.extend(item.get('systems') or []) |
| 134 entry.groups.extend(item.get('groups') or []) |
| 135 elif fmt == 'plainlist': |
| 136 entry.group = item.get('group') or '' |
| 137 else: |
| 138 assert False, 'Not reachable' |
| 139 return msg |
168 | 140 |
169 | 141 |
170 def read_config(): | 142 def validate_config(config): |
171 """Returns currently stored config or [] if not set.""" | 143 """Checks config_pb2.GroupImporterConfig for correctness. |
172 e = config_key().get() | 144 |
173 return (e.config if e else []) or [] | 145 Raises: |
| 146 ValueError if config has invalid structure. |
| 147 """ |
| 148 if not isinstance(config, config_pb2.GroupImporterConfig): |
| 149 raise ValueError('Not GroupImporterConfig proto message') |
| 150 |
| 151 # TODO(vadimsh): Can be made stricter. |
| 152 |
| 153 # Validate fields common to Tarball and Plainlist. |
| 154 for entry in list(config.tarball) + list(config.plainlist): |
| 155 if not entry.url: |
| 156 raise ValueError( |
| 157 '"url" field is required in %s' % entry.__class__.__name__) |
| 158 |
| 159 # Validate tarball fields. |
| 160 seen_systems = set(['external']) |
| 161 for tarball in config.tarball: |
| 162 if not tarball.systems: |
| 163 raise ValueError( |
| 164 '"tarball" entry "%s" needs "systems" field' % tarball.url) |
| 165 # There should be no overlap in systems between different bundles. |
| 166 twice = set(tarball.systems) & seen_systems |
| 167 if twice: |
| 168 raise ValueError( |
| 169 'A system is imported twice by "%s": %s' % |
| 170 (tarball.url, sorted(twice))) |
| 171 seen_systems.update(tarball.systems) |
| 172 |
| 173 # Validate plainlist fields. |
| 174 seen_groups = set() |
| 175 for plainlist in config.plainlist: |
| 176 if not plainlist.group: |
| 177 raise ValueError( |
| 178 '"plainlist" entry "%s" needs "group" field' % plainlist.url) |
| 179 if plainlist.group in seen_groups: |
| 180 raise ValueError( |
| 181 'In "%s" the group is imported twice: %s' % |
| 182 (plainlist.url, plainlist.group)) |
| 183 seen_groups.add(plainlist.group) |
174 | 184 |
175 | 185 |
176 def write_config(config): | 186 def read_config_text(): |
177 """Updates stored configuration.""" | 187 """Returns importer config as a text blob (or '' if not set).""" |
178 if not is_valid_config(config): | 188 e = config_key().get() |
179 raise ValueError('Invalid config') | 189 if not e: |
| 190 return '' |
| 191 if e.config_proto: |
| 192 return e.config_proto |
| 193 if e.config: |
| 194 msg = legacy_json_config_to_proto(e.config) |
| 195 if not msg: |
| 196 return '' |
| 197 return protobuf.text_format.MessageToString(msg) |
| 198 return '' |
| 199 |
| 200 |
| 201 def read_legacy_config(): |
| 202 """Returns legacy JSON config stored in GroupImporterConfig entity. |
| 203 |
| 204 TODO(vadimsh): Remove once all instance of auth service use protobuf configs. |
| 205 """ |
| 206 # Note: we do not care to do it in transaction. |
| 207 e = config_key().get() |
| 208 return e.config if e else None |
| 209 |
| 210 |
| 211 def write_config_text(text): |
| 212 """Validates config text blobs and puts it into the datastore. |
| 213 |
| 214 Raises: |
| 215 ValueError on invalid format. |
| 216 """ |
| 217 msg = config_pb2.GroupImporterConfig() |
| 218 try: |
| 219 protobuf.text_format.Merge(text, msg) |
| 220 except protobuf.text_format.ParseError as ex: |
| 221 raise ValueError('Config is badly formated: %s' % ex) |
| 222 validate_config(msg) |
180 e = GroupImporterConfig( | 223 e = GroupImporterConfig( |
181 key=config_key(), | 224 key=config_key(), |
182 config=config, | 225 config=read_legacy_config(), |
| 226 config_proto=text, |
183 modified_by=auth.get_current_identity()) | 227 modified_by=auth.get_current_identity()) |
184 e.put() | 228 e.put() |
185 | 229 |
186 | 230 |
187 def import_external_groups(): | 231 def import_external_groups(): |
188 """Refetches all external groups. | 232 """Refetches all external groups. |
189 | 233 |
190 Runs as a cron task. Raises BundleImportError in case of import errors. | 234 Runs as a cron task. Raises BundleImportError in case of import errors. |
191 """ | 235 """ |
192 # Missing config is not a error. | 236 # Missing config is not a error. |
193 config = read_config() | 237 config_text = read_config_text() |
194 if not config: | 238 if not config_text: |
195 logging.info('Not configured') | 239 logging.info('Not configured') |
196 return | 240 return |
197 if not is_valid_config(config): | 241 config = config_pb2.GroupImporterConfig() |
198 raise BundleImportError('Bad config') | 242 try: |
| 243 protobuf.text_format.Merge(config_text, config) |
| 244 except protobuf.text_format.ParseError as ex: |
| 245 raise BundleImportError('Bad config format: %s' % ex) |
| 246 try: |
| 247 validate_config(config) |
| 248 except ValueError as ex: |
| 249 raise BundleImportError('Bad config structure: %s' % ex) |
199 | 250 |
200 # Fetch all files specified in config in parallel. | 251 # Fetch all files specified in config in parallel. |
201 futures = [fetch_file_async(p['url'], p.get('oauth_scopes')) for p in config] | 252 entries = list(config.tarball) + list(config.plainlist) |
| 253 futures = [fetch_file_async(e.url, e.oauth_scopes) for e in entries] |
202 | 254 |
203 # {system name -> group name -> list of identities} | 255 # {system name -> group name -> list of identities} |
204 bundles = {} | 256 bundles = {} |
205 for p, future in zip(config, futures): | 257 for e, future in zip(entries, futures): |
206 fmt = p.get('format', 'tarball') | |
207 | |
208 # Unpack tarball into {system name -> group name -> list of identities}. | 258 # Unpack tarball into {system name -> group name -> list of identities}. |
209 if fmt == 'tarball': | 259 if isinstance(e, config_pb2.GroupImporterConfig.TarballEntry): |
210 fetched = load_tarball( | 260 fetched = load_tarball( |
211 future.get_result(), p['systems'], p.get('groups'), p.get('domain')) | 261 future.get_result(), e.systems, e.groups, e.domain) |
212 assert not ( | 262 assert not ( |
213 set(fetched) & set(bundles)), (fetched.keys(), bundles.keys()) | 263 set(fetched) & set(bundles)), (fetched.keys(), bundles.keys()) |
214 bundles.update(fetched) | 264 bundles.update(fetched) |
215 continue | 265 continue |
216 | 266 |
217 # Add plainlist group to 'external/*' bundle. | 267 # Add plainlist group to 'external/*' bundle. |
218 if fmt == 'plainlist': | 268 if isinstance(e, config_pb2.GroupImporterConfig.PlainlistEntry): |
219 group = load_group_file(future.get_result(), p.get('domain')) | 269 group = load_group_file(future.get_result(), e.domain) |
220 name = 'external/%s' % p['group'] | 270 name = 'external/%s' % e.group |
221 if 'external' not in bundles: | 271 if 'external' not in bundles: |
222 bundles['external'] = {} | 272 bundles['external'] = {} |
223 assert name not in bundles['external'], name | 273 assert name not in bundles['external'], name |
224 bundles['external'][name] = group | 274 bundles['external'][name] = group |
225 continue | 275 continue |
226 | 276 |
227 assert False, 'Unreachable' | 277 assert False, 'Unreachable' |
228 | 278 |
229 # Nothing to process? | 279 # Nothing to process? |
230 if not bundles: | 280 if not bundles: |
(...skipping 197 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
428 | 478 |
429 # Create new groups. | 479 # Create new groups. |
430 for group_name in (set(imported_groups) - set(system_groups)): | 480 for group_name in (set(imported_groups) - set(system_groups)): |
431 create_group(group_name) | 481 create_group(group_name) |
432 | 482 |
433 # Update existing groups. | 483 # Update existing groups. |
434 for group_name in (set(imported_groups) & set(system_groups)): | 484 for group_name in (set(imported_groups) & set(system_groups)): |
435 update_group(group_name) | 485 update_group(group_name) |
436 | 486 |
437 return to_put, to_delete | 487 return to_put, to_delete |
OLD | NEW |