OLD | NEW |
1 # Copyright 2014 Google Inc. All rights reserved. | 1 # Copyright 2011 Google Inc. |
2 # | 2 # |
3 # Licensed under the Apache License, Version 2.0 (the "License"); | 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
4 # you may not use this file except in compliance with the License. | 4 # you may not use this file except in compliance with the License. |
5 # You may obtain a copy of the License at | 5 # You may obtain a copy of the License at |
6 # | 6 # |
7 # http://www.apache.org/licenses/LICENSE-2.0 | 7 # http://www.apache.org/licenses/LICENSE-2.0 |
8 # | 8 # |
9 # Unless required by applicable law or agreed to in writing, software | 9 # Unless required by applicable law or agreed to in writing, software |
10 # distributed under the License is distributed on an "AS IS" BASIS, | 10 # distributed under the License is distributed on an "AS IS" BASIS, |
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 # See the License for the specific language governing permissions and | 12 # See the License for the specific language governing permissions and |
13 # limitations under the License. | 13 # limitations under the License. |
14 | 14 |
15 """Multi-credential file store with lock support. | 15 """Multi-credential file store with lock support. |
16 | 16 |
17 This module implements a JSON credential store where multiple | 17 This module implements a JSON credential store where multiple |
18 credentials can be stored in one file. That file supports locking | 18 credentials can be stored in one file. That file supports locking |
19 both in a single process and across processes. | 19 both in a single process and across processes. |
20 | 20 |
21 The credential themselves are keyed off of: | 21 The credential themselves are keyed off of: |
22 | |
23 * client_id | 22 * client_id |
24 * user_agent | 23 * user_agent |
25 * scope | 24 * scope |
26 | 25 |
27 The format of the stored data is like so:: | 26 The format of the stored data is like so: |
28 | 27 { |
29 { | 28 'file_version': 1, |
30 'file_version': 1, | 29 'data': [ |
31 'data': [ | 30 { |
32 { | 31 'key': { |
33 'key': { | 32 'clientId': '<client id>', |
34 'clientId': '<client id>', | 33 'userAgent': '<user agent>', |
35 'userAgent': '<user agent>', | 34 'scope': '<scope>' |
36 'scope': '<scope>' | 35 }, |
37 }, | 36 'credential': { |
38 'credential': { | 37 # JSON serialized Credentials. |
39 # JSON serialized Credentials. | |
40 } | |
41 } | 38 } |
42 ] | 39 } |
43 } | 40 ] |
44 | 41 } |
45 """ | 42 """ |
46 | 43 |
47 __author__ = 'jbeda@google.com (Joe Beda)' | 44 __author__ = 'jbeda@google.com (Joe Beda)' |
48 | 45 |
| 46 import base64 |
49 import errno | 47 import errno |
50 import json | |
51 import logging | 48 import logging |
52 import os | 49 import os |
53 import threading | 50 import threading |
54 | 51 |
| 52 from anyjson import simplejson |
| 53 from .client import Storage as BaseStorage |
55 from .client import Credentials | 54 from .client import Credentials |
56 from .client import Storage as BaseStorage | |
57 from . import util | 55 from . import util |
58 from locked_file import LockedFile | 56 from locked_file import LockedFile |
59 | 57 |
60 logger = logging.getLogger(__name__) | 58 logger = logging.getLogger(__name__) |
61 | 59 |
62 # A dict from 'filename'->_MultiStore instances | 60 # A dict from 'filename'->_MultiStore instances |
63 _multistores = {} | 61 _multistores = {} |
64 _multistores_lock = threading.Lock() | 62 _multistores_lock = threading.Lock() |
65 | 63 |
66 | 64 |
67 class Error(Exception): | 65 class Error(Exception): |
68 """Base error for this module.""" | 66 """Base error for this module.""" |
| 67 pass |
69 | 68 |
70 | 69 |
71 class NewerCredentialStoreError(Error): | 70 class NewerCredentialStoreError(Error): |
72 """The credential store is a newer version than supported.""" | 71 """The credential store is a newer version that supported.""" |
| 72 pass |
73 | 73 |
74 | 74 |
75 @util.positional(4) | 75 @util.positional(4) |
76 def get_credential_storage(filename, client_id, user_agent, scope, | 76 def get_credential_storage(filename, client_id, user_agent, scope, |
77 warn_on_readonly=True): | 77 warn_on_readonly=True): |
78 """Get a Storage instance for a credential. | 78 """Get a Storage instance for a credential. |
79 | 79 |
80 Args: | 80 Args: |
81 filename: The JSON file storing a set of credentials | 81 filename: The JSON file storing a set of credentials |
82 client_id: The client_id for the credential | 82 client_id: The client_id for the credential |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
186 | 186 |
187 class _MultiStore(object): | 187 class _MultiStore(object): |
188 """A file backed store for multiple credentials.""" | 188 """A file backed store for multiple credentials.""" |
189 | 189 |
190 @util.positional(2) | 190 @util.positional(2) |
191 def __init__(self, filename, warn_on_readonly=True): | 191 def __init__(self, filename, warn_on_readonly=True): |
192 """Initialize the class. | 192 """Initialize the class. |
193 | 193 |
194 This will create the file if necessary. | 194 This will create the file if necessary. |
195 """ | 195 """ |
196 self._file = LockedFile(filename, 'r+', 'r') | 196 self._file = LockedFile(filename, 'r+b', 'rb') |
197 self._thread_lock = threading.Lock() | 197 self._thread_lock = threading.Lock() |
198 self._read_only = False | 198 self._read_only = False |
199 self._warn_on_readonly = warn_on_readonly | 199 self._warn_on_readonly = warn_on_readonly |
200 | 200 |
201 self._create_file_if_needed() | 201 self._create_file_if_needed() |
202 | 202 |
203 # Cache of deserialized store. This is only valid after the | 203 # Cache of deserialized store. This is only valid after the |
204 # _MultiStore is locked or _refresh_data_cache is called. This is | 204 # _MultiStore is locked or _refresh_data_cache is called. This is |
205 # of the form of: | 205 # of the form of: |
206 # | 206 # |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
264 """ | 264 """ |
265 self._multistore._delete_credential(self._key) | 265 self._multistore._delete_credential(self._key) |
266 | 266 |
267 def _create_file_if_needed(self): | 267 def _create_file_if_needed(self): |
268 """Create an empty file if necessary. | 268 """Create an empty file if necessary. |
269 | 269 |
270 This method will not initialize the file. Instead it implements a | 270 This method will not initialize the file. Instead it implements a |
271 simple version of "touch" to ensure the file has been created. | 271 simple version of "touch" to ensure the file has been created. |
272 """ | 272 """ |
273 if not os.path.exists(self._file.filename()): | 273 if not os.path.exists(self._file.filename()): |
274 old_umask = os.umask(0o177) | 274 old_umask = os.umask(0177) |
275 try: | 275 try: |
276 open(self._file.filename(), 'a+b').close() | 276 open(self._file.filename(), 'a+b').close() |
277 finally: | 277 finally: |
278 os.umask(old_umask) | 278 os.umask(old_umask) |
279 | 279 |
280 def _lock(self): | 280 def _lock(self): |
281 """Lock the entire multistore.""" | 281 """Lock the entire multistore.""" |
282 self._thread_lock.acquire() | 282 self._thread_lock.acquire() |
283 try: | 283 self._file.open_and_lock() |
284 self._file.open_and_lock() | |
285 except IOError as e: | |
286 if e.errno == errno.ENOSYS: | |
287 logger.warn('File system does not support locking the credentials ' | |
288 'file.') | |
289 elif e.errno == errno.ENOLCK: | |
290 logger.warn('File system is out of resources for writing the ' | |
291 'credentials file (is your disk full?).') | |
292 else: | |
293 raise | |
294 if not self._file.is_locked(): | 284 if not self._file.is_locked(): |
295 self._read_only = True | 285 self._read_only = True |
296 if self._warn_on_readonly: | 286 if self._warn_on_readonly: |
297 logger.warn('The credentials file (%s) is not writable. Opening in ' | 287 logger.warn('The credentials file (%s) is not writable. Opening in ' |
298 'read-only mode. Any refreshed credentials will only be ' | 288 'read-only mode. Any refreshed credentials will only be ' |
299 'valid for this run.', self._file.filename()) | 289 'valid for this run.' % self._file.filename()) |
300 if os.path.getsize(self._file.filename()) == 0: | 290 if os.path.getsize(self._file.filename()) == 0: |
301 logger.debug('Initializing empty multistore file') | 291 logger.debug('Initializing empty multistore file') |
302 # The multistore is empty so write out an empty file. | 292 # The multistore is empty so write out an empty file. |
303 self._data = {} | 293 self._data = {} |
304 self._write() | 294 self._write() |
305 elif not self._read_only or self._data is None: | 295 elif not self._read_only or self._data is None: |
306 # Only refresh the data if we are read/write or we haven't | 296 # Only refresh the data if we are read/write or we haven't |
307 # cached the data yet. If we are readonly, we assume is isn't | 297 # cached the data yet. If we are readonly, we assume is isn't |
308 # changing out from under us and that we only have to read it | 298 # changing out from under us and that we only have to read it |
309 # once. This prevents us from whacking any new access keys that | 299 # once. This prevents us from whacking any new access keys that |
310 # we have cached in memory but were unable to write out. | 300 # we have cached in memory but were unable to write out. |
311 self._refresh_data_cache() | 301 self._refresh_data_cache() |
312 | 302 |
313 def _unlock(self): | 303 def _unlock(self): |
314 """Release the lock on the multistore.""" | 304 """Release the lock on the multistore.""" |
315 self._file.unlock_and_close() | 305 self._file.unlock_and_close() |
316 self._thread_lock.release() | 306 self._thread_lock.release() |
317 | 307 |
318 def _locked_json_read(self): | 308 def _locked_json_read(self): |
319 """Get the raw content of the multistore file. | 309 """Get the raw content of the multistore file. |
320 | 310 |
321 The multistore must be locked when this is called. | 311 The multistore must be locked when this is called. |
322 | 312 |
323 Returns: | 313 Returns: |
324 The contents of the multistore decoded as JSON. | 314 The contents of the multistore decoded as JSON. |
325 """ | 315 """ |
326 assert self._thread_lock.locked() | 316 assert self._thread_lock.locked() |
327 self._file.file_handle().seek(0) | 317 self._file.file_handle().seek(0) |
328 return json.load(self._file.file_handle()) | 318 return simplejson.load(self._file.file_handle()) |
329 | 319 |
330 def _locked_json_write(self, data): | 320 def _locked_json_write(self, data): |
331 """Write a JSON serializable data structure to the multistore. | 321 """Write a JSON serializable data structure to the multistore. |
332 | 322 |
333 The multistore must be locked when this is called. | 323 The multistore must be locked when this is called. |
334 | 324 |
335 Args: | 325 Args: |
336 data: The data to be serialized and written. | 326 data: The data to be serialized and written. |
337 """ | 327 """ |
338 assert self._thread_lock.locked() | 328 assert self._thread_lock.locked() |
339 if self._read_only: | 329 if self._read_only: |
340 return | 330 return |
341 self._file.file_handle().seek(0) | 331 self._file.file_handle().seek(0) |
342 json.dump(data, self._file.file_handle(), sort_keys=True, indent=2, separato
rs=(',', ': ')) | 332 simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2) |
343 self._file.file_handle().truncate() | 333 self._file.file_handle().truncate() |
344 | 334 |
345 def _refresh_data_cache(self): | 335 def _refresh_data_cache(self): |
346 """Refresh the contents of the multistore. | 336 """Refresh the contents of the multistore. |
347 | 337 |
348 The multistore must be locked when this is called. | 338 The multistore must be locked when this is called. |
349 | 339 |
350 Raises: | 340 Raises: |
351 NewerCredentialStoreError: Raised when a newer client has written the | 341 NewerCredentialStoreError: Raised when a newer client has written the |
352 store. | 342 store. |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
390 Args: | 380 Args: |
391 cred_entry: A dict entry from the data member of our format | 381 cred_entry: A dict entry from the data member of our format |
392 | 382 |
393 Returns: | 383 Returns: |
394 (key, cred) where the key is the key tuple and the cred is the | 384 (key, cred) where the key is the key tuple and the cred is the |
395 OAuth2Credential object. | 385 OAuth2Credential object. |
396 """ | 386 """ |
397 raw_key = cred_entry['key'] | 387 raw_key = cred_entry['key'] |
398 key = util.dict_to_tuple_key(raw_key) | 388 key = util.dict_to_tuple_key(raw_key) |
399 credential = None | 389 credential = None |
400 credential = Credentials.new_from_json(json.dumps(cred_entry['credential'])) | 390 credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credenti
al'])) |
401 return (key, credential) | 391 return (key, credential) |
402 | 392 |
403 def _write(self): | 393 def _write(self): |
404 """Write the cached data back out. | 394 """Write the cached data back out. |
405 | 395 |
406 The multistore must be locked. | 396 The multistore must be locked. |
407 """ | 397 """ |
408 raw_data = {'file_version': 1} | 398 raw_data = {'file_version': 1} |
409 raw_creds = [] | 399 raw_creds = [] |
410 raw_data['data'] = raw_creds | 400 raw_data['data'] = raw_creds |
411 for (cred_key, cred) in self._data.items(): | 401 for (cred_key, cred) in self._data.items(): |
412 raw_key = dict(cred_key) | 402 raw_key = dict(cred_key) |
413 raw_cred = json.loads(cred.to_json()) | 403 raw_cred = simplejson.loads(cred.to_json()) |
414 raw_creds.append({'key': raw_key, 'credential': raw_cred}) | 404 raw_creds.append({'key': raw_key, 'credential': raw_cred}) |
415 self._locked_json_write(raw_data) | 405 self._locked_json_write(raw_data) |
416 | 406 |
417 def _get_all_credential_keys(self): | 407 def _get_all_credential_keys(self): |
418 """Gets all the registered credential keys in the multistore. | 408 """Gets all the registered credential keys in the multistore. |
419 | 409 |
420 Returns: | 410 Returns: |
421 A list of dictionaries corresponding to all the keys currently registered | 411 A list of dictionaries corresponding to all the keys currently registered |
422 """ | 412 """ |
423 return [dict(key) for key in self._data.keys()] | 413 return [dict(key) for key in self._data.keys()] |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
465 """Get a Storage object to get/set a credential. | 455 """Get a Storage object to get/set a credential. |
466 | 456 |
467 This Storage is a 'view' into the multistore. | 457 This Storage is a 'view' into the multistore. |
468 | 458 |
469 Args: | 459 Args: |
470 key: The key used to retrieve the credential | 460 key: The key used to retrieve the credential |
471 | 461 |
472 Returns: | 462 Returns: |
473 A Storage object that can be used to get/set this cred | 463 A Storage object that can be used to get/set this cred |
474 """ | 464 """ |
475 return self._Storage(self, key) | 465 return self._Storage(self, key) |
OLD | NEW |