OLD | NEW |
---|---|
1 # Copyright 2014 The Swarming Authors. All rights reserved. | 1 # Copyright 2014 The Swarming Authors. All rights reserved. |
2 # Use of this source code is governed by the Apache v2.0 license that can be | 2 # Use of this source code is governed by the Apache v2.0 license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Imports groups from some external tar.gz bundle or plain text list. | 5 """Imports groups from some external tar.gz bundle or plain text list. |
6 | 6 |
7 External URL should serve *.tar.gz file with the following file structure: | 7 External URL should serve *.tar.gz file with the following file structure: |
8 <external group system name>/<group name>: | 8 <external group system name>/<group name>: |
9 userid | 9 userid |
10 userid | 10 userid |
(...skipping 17 matching lines...) Expand all Loading... | |
28 the tarball. | 28 the tarball. |
29 | 29 |
30 Plain list format should have one userid per line and can only describe a single | 30 Plain list format should have one userid per line and can only describe a single |
31 group in a single system. Such groups will be added to 'external/*' groups | 31 group in a single system. Such groups will be added to 'external/*' groups |
32 namespace. Removing such group from importer config will remove it from | 32 namespace. Removing such group from importer config will remove it from |
33 service too. | 33 service too. |
34 """ | 34 """ |
35 | 35 |
36 import collections | 36 import collections |
37 import contextlib | 37 import contextlib |
38 import json | |
38 import logging | 39 import logging |
39 import StringIO | 40 import StringIO |
40 import tarfile | 41 import tarfile |
41 | 42 |
42 from google.appengine.ext import ndb | 43 from google.appengine.ext import ndb |
43 | 44 |
45 from google import protobuf | |
46 | |
44 from components import auth | 47 from components import auth |
45 from components import net | 48 from components import net |
46 from components import utils | 49 from components import utils |
47 from components.auth import model | 50 from components.auth import model |
48 | 51 |
52 from proto import config_pb2 | |
53 | |
49 | 54 |
50 class BundleImportError(Exception): | 55 class BundleImportError(Exception): |
51 """Base class for errors while fetching external bundle.""" | 56 """Base class for errors while fetching external bundle.""" |
52 | 57 |
53 | 58 |
54 class BundleFetchError(BundleImportError): | 59 class BundleFetchError(BundleImportError): |
55 """Failed to fetch the archive from remote URL.""" | 60 """Failed to fetch the archive from remote URL.""" |
56 | 61 |
57 def __init__(self, url, status_code, content): | 62 def __init__(self, url, status_code, content): |
58 super(BundleFetchError, self).__init__() | 63 super(BundleFetchError, self).__init__() |
(...skipping 28 matching lines...) Expand all Loading... | |
87 return 'Bundle contains invalid group file: %s' % self.inner_exc | 92 return 'Bundle contains invalid group file: %s' % self.inner_exc |
88 | 93 |
89 | 94 |
90 def config_key(): | 95 def config_key(): |
91 """Key of GroupImporterConfig singleton entity.""" | 96 """Key of GroupImporterConfig singleton entity.""" |
92 return ndb.Key('GroupImporterConfig', 'config') | 97 return ndb.Key('GroupImporterConfig', 'config') |
93 | 98 |
94 | 99 |
95 class GroupImporterConfig(ndb.Model): | 100 class GroupImporterConfig(ndb.Model): |
96 """Singleton entity with group importer configuration JSON.""" | 101 """Singleton entity with group importer configuration JSON.""" |
97 config = ndb.JsonProperty() | 102 config = ndb.TextProperty() # legacy field with JSON config |
103 config_proto = ndb.TextProperty() | |
98 modified_by = auth.IdentityProperty(indexed=False) | 104 modified_by = auth.IdentityProperty(indexed=False) |
99 modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False) | 105 modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False) |
100 | 106 |
101 | 107 |
102 def is_valid_config(config): | 108 def legacy_json_config_to_proto(config_json): |
103 """Checks config for correctness.""" | 109 """Converts legacy JSON config to config_pb2.GroupImporterConfig message. |
104 if not isinstance(config, list): | |
105 return False | |
106 | 110 |
107 seen_systems = set(['external']) | 111 TODO(vadimsh): Remove once all instances of auth service use protobuf configs. |
108 seen_groups = set() | 112 """ |
113 try: | |
114 config = json.loads(config_json) | |
115 except ValueError as ex: | |
116 logging.error('Invalid JSON: %s', ex) | |
117 return None | |
118 msg = config_pb2.GroupImporterConfig() | |
109 for item in config: | 119 for item in config: |
110 if not isinstance(item, dict): | |
111 return False | |
112 | |
113 # 'format' is an optional string describing the format of the imported | |
114 # source. The default format is 'tarball'. | |
115 fmt = item.get('format', 'tarball') | 120 fmt = item.get('format', 'tarball') |
116 if fmt not in ['tarball', 'plainlist']: | |
117 return False | |
118 | |
119 # 'url' is a required string: where to fetch groups from. | |
120 url = item.get('url') | |
121 if not url or not isinstance(url, basestring): | |
122 return False | |
123 | |
124 # 'oauth_scopes' is an optional list of strings: used when generating OAuth | |
125 # access_token to put in Authorization header. | |
126 oauth_scopes = item.get('oauth_scopes') | |
127 if oauth_scopes is not None: | |
128 if not all(isinstance(x, basestring) for x in oauth_scopes): | |
129 return False | |
130 | |
131 # 'domain' is an optional string: will be used when constructing emails from | |
132 # naked usernames found in imported groups. | |
133 domain = item.get('domain') | |
134 if domain and not isinstance(domain, basestring): | |
135 return False | |
136 | |
137 # 'tarball' format uses 'systems' and 'groups' fields. | |
138 if fmt == 'tarball': | 121 if fmt == 'tarball': |
139 # 'systems' is a required list of strings: group systems expected to be | 122 entry = msg.tarball.add() |
140 # found in the archive (they act as prefixes to group names, e.g 'ldap'). | |
141 systems = item.get('systems') | |
142 if not systems or not isinstance(systems, list): | |
143 return False | |
144 if not all(isinstance(x, basestring) for x in systems): | |
145 return False | |
146 | |
147 # There should be no overlap in systems between different bundles. | |
148 if set(systems) & seen_systems: | |
149 return False | |
150 seen_systems.update(systems) | |
151 | |
152 # 'groups' is an optional list of strings: if given, filters imported | |
153 # groups only to this list. | |
154 groups = item.get('groups') | |
155 if groups and not all(isinstance(x, basestring) for x in groups): | |
156 return False | |
157 elif fmt == 'plainlist': | 123 elif fmt == 'plainlist': |
158 # 'group' is a required name of imported group. The full group name will | 124 entry = msg.plainlist.add() |
159 # be 'external/<group>'. | |
160 group = item.get('group') | |
161 if not group or not isinstance(group, basestring) or group in seen_groups: | |
162 return False | |
163 seen_groups.add(group) | |
164 else: | 125 else: |
165 assert False, 'Unreachable' | 126 logging.error('Unrecognized format: %s', fmt) |
166 | 127 continue |
167 return True | 128 entry.url = item.get('url') or '' |
129 entry.oauth_scopes.extend(item.get('oauth_scopes') or []) | |
130 if 'domain' in item: | |
131 entry.domain = item['domain'] | |
132 if fmt == 'tarball': | |
133 entry.systems.extend(item.get('systems') or []) | |
134 entry.groups.extend(item.get('groups') or []) | |
135 elif fmt == 'plainlist': | |
136 entry.group = item.get('group') or '' | |
137 else: | |
138 assert False, 'Not reachable' | |
139 return msg | |
168 | 140 |
169 | 141 |
170 def read_config(): | 142 def validate_config(config): |
171 """Returns currently stored config or [] if not set.""" | 143 """Checks config_pb2.GroupImporterConfig for correctness. |
172 e = config_key().get() | 144 |
173 return (e.config if e else []) or [] | 145 Raises: |
146 ValueError if config has invalid structure. | |
147 """ | |
148 if not isinstance(config, config_pb2.GroupImporterConfig): | |
149 raise ValueError('Not GroupImporterConfig proto message') | |
150 | |
151 # TODO(vadimsh): Can be made stricter. | |
152 | |
153 # Validate fields common to Tarball and Plainlist. | |
154 for entry in list(config.tarball) + list(config.plainlist): | |
155 if not entry.url: | |
156 raise ValueError( | |
157 '\'url\' field is required in %s' % entry.__class__.__name__) | |
158 | |
159 # Validate tarball fields. | |
160 seen_systems = set(['external']) | |
161 for tarball in config.tarball: | |
162 if not tarball.systems: | |
163 raise ValueError('\'tarball\' entry needs \'systems\' field') | |
164 # There should be no overlap in systems between different bundles. | |
165 if set(tarball.systems) & seen_systems: | |
166 raise ValueError('A system is imported twice') | |
nodir
2015/06/02 00:19:20
include the system name in the error message
Vadim Sh.
2015/06/02 00:52:17
Done.
| |
167 seen_systems.update(tarball.systems) | |
168 | |
169 # Validate plainlist fields. | |
170 seen_groups = set() | |
171 for plainlist in config.plainlist: | |
172 if not plainlist.group: | |
173 raise ValueError('\'plainlist\' entry needs \'group\' field') | |
174 if plainlist.group in seen_groups: | |
175 raise ValueError('Group %r is imported twice' % str(plainlist.group)) | |
nodir
2015/06/02 00:19:20
Why %r and str(arg) at the same time? You are goin
Vadim Sh.
2015/06/02 00:52:17
because I hate u in "Group u"name" is imported twi
nodir
2015/06/02 16:52:29
I see. None is not gonna happen anyway
| |
176 seen_groups.add(plainlist.group) | |
174 | 177 |
175 | 178 |
176 def write_config(config): | 179 def read_config_text(): |
177 """Updates stored configuration.""" | 180 """Returns importer config as a text blob (or '' if not set).""" |
178 if not is_valid_config(config): | 181 e = config_key().get() |
179 raise ValueError('Invalid config') | 182 if not e: |
183 return '' | |
184 if e.config_proto: | |
185 return e.config_proto | |
186 if e.config: | |
187 msg = legacy_json_config_to_proto(e.config) | |
188 if not msg: | |
189 return '' | |
190 return protobuf.text_format.MessageToString(msg) | |
191 return '' | |
192 | |
193 | |
194 def read_legacy_config(): | |
195 """Returns legacy JSON config stored in GroupImporterConfig entity. | |
196 | |
197 TODO(vadimsh): Remove once all instance of auth service use protobuf configs. | |
198 """ | |
199 # Note: we do not care to do it in transaction. | |
200 e = config_key().get() | |
201 return e.config if e else None | |
202 | |
203 | |
204 def write_config_text(text): | |
205 """Validates config text blobs and puts it into the datastore. | |
206 | |
207 Raises: | |
208 ValueError on invalid format. | |
209 """ | |
210 msg = config_pb2.GroupImporterConfig() | |
211 try: | |
212 protobuf.text_format.Merge(text, msg) | |
213 except protobuf.text_format.ParseError as ex: | |
214 raise ValueError('Config is badly formated: %s' % ex) | |
215 validate_config(msg) | |
180 e = GroupImporterConfig( | 216 e = GroupImporterConfig( |
181 key=config_key(), | 217 key=config_key(), |
182 config=config, | 218 config=read_legacy_config(), |
219 config_proto=text, | |
183 modified_by=auth.get_current_identity()) | 220 modified_by=auth.get_current_identity()) |
184 e.put() | 221 e.put() |
185 | 222 |
186 | 223 |
187 def import_external_groups(): | 224 def import_external_groups(): |
188 """Refetches all external groups. | 225 """Refetches all external groups. |
189 | 226 |
190 Runs as a cron task. Raises BundleImportError in case of import errors. | 227 Runs as a cron task. Raises BundleImportError in case of import errors. |
191 """ | 228 """ |
192 # Missing config is not a error. | 229 # Missing config is not a error. |
193 config = read_config() | 230 config_text = read_config_text() |
194 if not config: | 231 if not config_text: |
195 logging.info('Not configured') | 232 logging.info('Not configured') |
196 return | 233 return |
197 if not is_valid_config(config): | 234 config = config_pb2.GroupImporterConfig() |
198 raise BundleImportError('Bad config') | 235 try: |
236 protobuf.text_format.Merge(config_text, config) | |
237 except protobuf.text_format.ParseError as ex: | |
238 raise BundleImportError('Bad config format: %s' % ex) | |
239 try: | |
240 validate_config(config) | |
241 except ValueError as ex: | |
242 raise BundleImportError('Bad config structure: %s' % ex) | |
199 | 243 |
200 # Fetch all files specified in config in parallel. | 244 # Fetch all files specified in config in parallel. |
201 futures = [fetch_file_async(p['url'], p.get('oauth_scopes')) for p in config] | 245 entries = list(config.tarball) + list(config.plainlist) |
246 futures = [fetch_file_async(e.url, e.oauth_scopes) for e in entries] | |
202 | 247 |
203 # {system name -> group name -> list of identities} | 248 # {system name -> group name -> list of identities} |
204 bundles = {} | 249 bundles = {} |
205 for p, future in zip(config, futures): | 250 for e, future in zip(entries, futures): |
206 fmt = p.get('format', 'tarball') | |
207 | |
208 # Unpack tarball into {system name -> group name -> list of identities}. | 251 # Unpack tarball into {system name -> group name -> list of identities}. |
209 if fmt == 'tarball': | 252 if isinstance(e, config_pb2.GroupImporterConfig.TarballEntry): |
210 fetched = load_tarball( | 253 fetched = load_tarball( |
211 future.get_result(), p['systems'], p.get('groups'), p.get('domain')) | 254 future.get_result(), e.systems, e.groups, e.domain) |
212 assert not ( | 255 assert not ( |
213 set(fetched) & set(bundles)), (fetched.keys(), bundles.keys()) | 256 set(fetched) & set(bundles)), (fetched.keys(), bundles.keys()) |
214 bundles.update(fetched) | 257 bundles.update(fetched) |
215 continue | 258 continue |
216 | 259 |
217 # Add plainlist group to 'external/*' bundle. | 260 # Add plainlist group to 'external/*' bundle. |
218 if fmt == 'plainlist': | 261 if isinstance(e, config_pb2.GroupImporterConfig.PlainlistEntry): |
219 group = load_group_file(future.get_result(), p.get('domain')) | 262 group = load_group_file(future.get_result(), e.domain) |
220 name = 'external/%s' % p['group'] | 263 name = 'external/%s' % e.group |
221 if 'external' not in bundles: | 264 if 'external' not in bundles: |
222 bundles['external'] = {} | 265 bundles['external'] = {} |
223 assert name not in bundles['external'], name | 266 assert name not in bundles['external'], name |
224 bundles['external'][name] = group | 267 bundles['external'][name] = group |
225 continue | 268 continue |
226 | 269 |
227 assert False, 'Unreachable' | 270 assert False, 'Unreachable' |
228 | 271 |
229 # Nothing to process? | 272 # Nothing to process? |
230 if not bundles: | 273 if not bundles: |
(...skipping 197 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
428 | 471 |
429 # Create new groups. | 472 # Create new groups. |
430 for group_name in (set(imported_groups) - set(system_groups)): | 473 for group_name in (set(imported_groups) - set(system_groups)): |
431 create_group(group_name) | 474 create_group(group_name) |
432 | 475 |
433 # Update existing groups. | 476 # Update existing groups. |
434 for group_name in (set(imported_groups) & set(system_groups)): | 477 for group_name in (set(imported_groups) & set(system_groups)): |
435 update_group(group_name) | 478 update_group(group_name) |
436 | 479 |
437 return to_put, to_delete | 480 return to_put, to_delete |
OLD | NEW |