| OLD | NEW |
| 1 # Copyright 2014 Google Inc. All rights reserved. | 1 # Copyright 2011 Google Inc. |
| 2 # | 2 # |
| 3 # Licensed under the Apache License, Version 2.0 (the "License"); | 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 # you may not use this file except in compliance with the License. | 4 # you may not use this file except in compliance with the License. |
| 5 # You may obtain a copy of the License at | 5 # You may obtain a copy of the License at |
| 6 # | 6 # |
| 7 # http://www.apache.org/licenses/LICENSE-2.0 | 7 # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 # | 8 # |
| 9 # Unless required by applicable law or agreed to in writing, software | 9 # Unless required by applicable law or agreed to in writing, software |
| 10 # distributed under the License is distributed on an "AS IS" BASIS, | 10 # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 # See the License for the specific language governing permissions and | 12 # See the License for the specific language governing permissions and |
| 13 # limitations under the License. | 13 # limitations under the License. |
| 14 | 14 |
| 15 """Multi-credential file store with lock support. | 15 """Multi-credential file store with lock support. |
| 16 | 16 |
| 17 This module implements a JSON credential store where multiple | 17 This module implements a JSON credential store where multiple |
| 18 credentials can be stored in one file. That file supports locking | 18 credentials can be stored in one file. That file supports locking |
| 19 both in a single process and across processes. | 19 both in a single process and across processes. |
| 20 | 20 |
| 21 The credential themselves are keyed off of: | 21 The credential themselves are keyed off of: |
| 22 | |
| 23 * client_id | 22 * client_id |
| 24 * user_agent | 23 * user_agent |
| 25 * scope | 24 * scope |
| 26 | 25 |
| 27 The format of the stored data is like so:: | 26 The format of the stored data is like so: |
| 28 | 27 { |
| 29 { | 28 'file_version': 1, |
| 30 'file_version': 1, | 29 'data': [ |
| 31 'data': [ | 30 { |
| 32 { | 31 'key': { |
| 33 'key': { | 32 'clientId': '<client id>', |
| 34 'clientId': '<client id>', | 33 'userAgent': '<user agent>', |
| 35 'userAgent': '<user agent>', | 34 'scope': '<scope>' |
| 36 'scope': '<scope>' | 35 }, |
| 37 }, | 36 'credential': { |
| 38 'credential': { | 37 # JSON serialized Credentials. |
| 39 # JSON serialized Credentials. | |
| 40 } | |
| 41 } | 38 } |
| 42 ] | 39 } |
| 43 } | 40 ] |
| 44 | 41 } |
| 45 """ | 42 """ |
| 46 | 43 |
| 47 __author__ = 'jbeda@google.com (Joe Beda)' | 44 __author__ = 'jbeda@google.com (Joe Beda)' |
| 48 | 45 |
| 46 import base64 |
| 49 import errno | 47 import errno |
| 50 import json | |
| 51 import logging | 48 import logging |
| 52 import os | 49 import os |
| 53 import threading | 50 import threading |
| 54 | 51 |
| 52 from anyjson import simplejson |
| 53 from .client import Storage as BaseStorage |
| 55 from .client import Credentials | 54 from .client import Credentials |
| 56 from .client import Storage as BaseStorage | |
| 57 from . import util | 55 from . import util |
| 58 from locked_file import LockedFile | 56 from locked_file import LockedFile |
| 59 | 57 |
| 60 logger = logging.getLogger(__name__) | 58 logger = logging.getLogger(__name__) |
| 61 | 59 |
| 62 # A dict from 'filename'->_MultiStore instances | 60 # A dict from 'filename'->_MultiStore instances |
| 63 _multistores = {} | 61 _multistores = {} |
| 64 _multistores_lock = threading.Lock() | 62 _multistores_lock = threading.Lock() |
| 65 | 63 |
| 66 | 64 |
| 67 class Error(Exception): | 65 class Error(Exception): |
| 68 """Base error for this module.""" | 66 """Base error for this module.""" |
| 67 pass |
| 69 | 68 |
| 70 | 69 |
| 71 class NewerCredentialStoreError(Error): | 70 class NewerCredentialStoreError(Error): |
| 72 """The credential store is a newer version than supported.""" | 71 """The credential store is a newer version that supported.""" |
| 72 pass |
| 73 | 73 |
| 74 | 74 |
| 75 @util.positional(4) | 75 @util.positional(4) |
| 76 def get_credential_storage(filename, client_id, user_agent, scope, | 76 def get_credential_storage(filename, client_id, user_agent, scope, |
| 77 warn_on_readonly=True): | 77 warn_on_readonly=True): |
| 78 """Get a Storage instance for a credential. | 78 """Get a Storage instance for a credential. |
| 79 | 79 |
| 80 Args: | 80 Args: |
| 81 filename: The JSON file storing a set of credentials | 81 filename: The JSON file storing a set of credentials |
| 82 client_id: The client_id for the credential | 82 client_id: The client_id for the credential |
| (...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 186 | 186 |
| 187 class _MultiStore(object): | 187 class _MultiStore(object): |
| 188 """A file backed store for multiple credentials.""" | 188 """A file backed store for multiple credentials.""" |
| 189 | 189 |
| 190 @util.positional(2) | 190 @util.positional(2) |
| 191 def __init__(self, filename, warn_on_readonly=True): | 191 def __init__(self, filename, warn_on_readonly=True): |
| 192 """Initialize the class. | 192 """Initialize the class. |
| 193 | 193 |
| 194 This will create the file if necessary. | 194 This will create the file if necessary. |
| 195 """ | 195 """ |
| 196 self._file = LockedFile(filename, 'r+', 'r') | 196 self._file = LockedFile(filename, 'r+b', 'rb') |
| 197 self._thread_lock = threading.Lock() | 197 self._thread_lock = threading.Lock() |
| 198 self._read_only = False | 198 self._read_only = False |
| 199 self._warn_on_readonly = warn_on_readonly | 199 self._warn_on_readonly = warn_on_readonly |
| 200 | 200 |
| 201 self._create_file_if_needed() | 201 self._create_file_if_needed() |
| 202 | 202 |
| 203 # Cache of deserialized store. This is only valid after the | 203 # Cache of deserialized store. This is only valid after the |
| 204 # _MultiStore is locked or _refresh_data_cache is called. This is | 204 # _MultiStore is locked or _refresh_data_cache is called. This is |
| 205 # of the form of: | 205 # of the form of: |
| 206 # | 206 # |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 264 """ | 264 """ |
| 265 self._multistore._delete_credential(self._key) | 265 self._multistore._delete_credential(self._key) |
| 266 | 266 |
| 267 def _create_file_if_needed(self): | 267 def _create_file_if_needed(self): |
| 268 """Create an empty file if necessary. | 268 """Create an empty file if necessary. |
| 269 | 269 |
| 270 This method will not initialize the file. Instead it implements a | 270 This method will not initialize the file. Instead it implements a |
| 271 simple version of "touch" to ensure the file has been created. | 271 simple version of "touch" to ensure the file has been created. |
| 272 """ | 272 """ |
| 273 if not os.path.exists(self._file.filename()): | 273 if not os.path.exists(self._file.filename()): |
| 274 old_umask = os.umask(0o177) | 274 old_umask = os.umask(0177) |
| 275 try: | 275 try: |
| 276 open(self._file.filename(), 'a+b').close() | 276 open(self._file.filename(), 'a+b').close() |
| 277 finally: | 277 finally: |
| 278 os.umask(old_umask) | 278 os.umask(old_umask) |
| 279 | 279 |
| 280 def _lock(self): | 280 def _lock(self): |
| 281 """Lock the entire multistore.""" | 281 """Lock the entire multistore.""" |
| 282 self._thread_lock.acquire() | 282 self._thread_lock.acquire() |
| 283 try: | 283 self._file.open_and_lock() |
| 284 self._file.open_and_lock() | |
| 285 except IOError as e: | |
| 286 if e.errno == errno.ENOSYS: | |
| 287 logger.warn('File system does not support locking the credentials ' | |
| 288 'file.') | |
| 289 elif e.errno == errno.ENOLCK: | |
| 290 logger.warn('File system is out of resources for writing the ' | |
| 291 'credentials file (is your disk full?).') | |
| 292 else: | |
| 293 raise | |
| 294 if not self._file.is_locked(): | 284 if not self._file.is_locked(): |
| 295 self._read_only = True | 285 self._read_only = True |
| 296 if self._warn_on_readonly: | 286 if self._warn_on_readonly: |
| 297 logger.warn('The credentials file (%s) is not writable. Opening in ' | 287 logger.warn('The credentials file (%s) is not writable. Opening in ' |
| 298 'read-only mode. Any refreshed credentials will only be ' | 288 'read-only mode. Any refreshed credentials will only be ' |
| 299 'valid for this run.', self._file.filename()) | 289 'valid for this run.' % self._file.filename()) |
| 300 if os.path.getsize(self._file.filename()) == 0: | 290 if os.path.getsize(self._file.filename()) == 0: |
| 301 logger.debug('Initializing empty multistore file') | 291 logger.debug('Initializing empty multistore file') |
| 302 # The multistore is empty so write out an empty file. | 292 # The multistore is empty so write out an empty file. |
| 303 self._data = {} | 293 self._data = {} |
| 304 self._write() | 294 self._write() |
| 305 elif not self._read_only or self._data is None: | 295 elif not self._read_only or self._data is None: |
| 306 # Only refresh the data if we are read/write or we haven't | 296 # Only refresh the data if we are read/write or we haven't |
| 307 # cached the data yet. If we are readonly, we assume is isn't | 297 # cached the data yet. If we are readonly, we assume is isn't |
| 308 # changing out from under us and that we only have to read it | 298 # changing out from under us and that we only have to read it |
| 309 # once. This prevents us from whacking any new access keys that | 299 # once. This prevents us from whacking any new access keys that |
| 310 # we have cached in memory but were unable to write out. | 300 # we have cached in memory but were unable to write out. |
| 311 self._refresh_data_cache() | 301 self._refresh_data_cache() |
| 312 | 302 |
| 313 def _unlock(self): | 303 def _unlock(self): |
| 314 """Release the lock on the multistore.""" | 304 """Release the lock on the multistore.""" |
| 315 self._file.unlock_and_close() | 305 self._file.unlock_and_close() |
| 316 self._thread_lock.release() | 306 self._thread_lock.release() |
| 317 | 307 |
| 318 def _locked_json_read(self): | 308 def _locked_json_read(self): |
| 319 """Get the raw content of the multistore file. | 309 """Get the raw content of the multistore file. |
| 320 | 310 |
| 321 The multistore must be locked when this is called. | 311 The multistore must be locked when this is called. |
| 322 | 312 |
| 323 Returns: | 313 Returns: |
| 324 The contents of the multistore decoded as JSON. | 314 The contents of the multistore decoded as JSON. |
| 325 """ | 315 """ |
| 326 assert self._thread_lock.locked() | 316 assert self._thread_lock.locked() |
| 327 self._file.file_handle().seek(0) | 317 self._file.file_handle().seek(0) |
| 328 return json.load(self._file.file_handle()) | 318 return simplejson.load(self._file.file_handle()) |
| 329 | 319 |
| 330 def _locked_json_write(self, data): | 320 def _locked_json_write(self, data): |
| 331 """Write a JSON serializable data structure to the multistore. | 321 """Write a JSON serializable data structure to the multistore. |
| 332 | 322 |
| 333 The multistore must be locked when this is called. | 323 The multistore must be locked when this is called. |
| 334 | 324 |
| 335 Args: | 325 Args: |
| 336 data: The data to be serialized and written. | 326 data: The data to be serialized and written. |
| 337 """ | 327 """ |
| 338 assert self._thread_lock.locked() | 328 assert self._thread_lock.locked() |
| 339 if self._read_only: | 329 if self._read_only: |
| 340 return | 330 return |
| 341 self._file.file_handle().seek(0) | 331 self._file.file_handle().seek(0) |
| 342 json.dump(data, self._file.file_handle(), sort_keys=True, indent=2, separato
rs=(',', ': ')) | 332 simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2) |
| 343 self._file.file_handle().truncate() | 333 self._file.file_handle().truncate() |
| 344 | 334 |
| 345 def _refresh_data_cache(self): | 335 def _refresh_data_cache(self): |
| 346 """Refresh the contents of the multistore. | 336 """Refresh the contents of the multistore. |
| 347 | 337 |
| 348 The multistore must be locked when this is called. | 338 The multistore must be locked when this is called. |
| 349 | 339 |
| 350 Raises: | 340 Raises: |
| 351 NewerCredentialStoreError: Raised when a newer client has written the | 341 NewerCredentialStoreError: Raised when a newer client has written the |
| 352 store. | 342 store. |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 390 Args: | 380 Args: |
| 391 cred_entry: A dict entry from the data member of our format | 381 cred_entry: A dict entry from the data member of our format |
| 392 | 382 |
| 393 Returns: | 383 Returns: |
| 394 (key, cred) where the key is the key tuple and the cred is the | 384 (key, cred) where the key is the key tuple and the cred is the |
| 395 OAuth2Credential object. | 385 OAuth2Credential object. |
| 396 """ | 386 """ |
| 397 raw_key = cred_entry['key'] | 387 raw_key = cred_entry['key'] |
| 398 key = util.dict_to_tuple_key(raw_key) | 388 key = util.dict_to_tuple_key(raw_key) |
| 399 credential = None | 389 credential = None |
| 400 credential = Credentials.new_from_json(json.dumps(cred_entry['credential'])) | 390 credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credenti
al'])) |
| 401 return (key, credential) | 391 return (key, credential) |
| 402 | 392 |
| 403 def _write(self): | 393 def _write(self): |
| 404 """Write the cached data back out. | 394 """Write the cached data back out. |
| 405 | 395 |
| 406 The multistore must be locked. | 396 The multistore must be locked. |
| 407 """ | 397 """ |
| 408 raw_data = {'file_version': 1} | 398 raw_data = {'file_version': 1} |
| 409 raw_creds = [] | 399 raw_creds = [] |
| 410 raw_data['data'] = raw_creds | 400 raw_data['data'] = raw_creds |
| 411 for (cred_key, cred) in self._data.items(): | 401 for (cred_key, cred) in self._data.items(): |
| 412 raw_key = dict(cred_key) | 402 raw_key = dict(cred_key) |
| 413 raw_cred = json.loads(cred.to_json()) | 403 raw_cred = simplejson.loads(cred.to_json()) |
| 414 raw_creds.append({'key': raw_key, 'credential': raw_cred}) | 404 raw_creds.append({'key': raw_key, 'credential': raw_cred}) |
| 415 self._locked_json_write(raw_data) | 405 self._locked_json_write(raw_data) |
| 416 | 406 |
| 417 def _get_all_credential_keys(self): | 407 def _get_all_credential_keys(self): |
| 418 """Gets all the registered credential keys in the multistore. | 408 """Gets all the registered credential keys in the multistore. |
| 419 | 409 |
| 420 Returns: | 410 Returns: |
| 421 A list of dictionaries corresponding to all the keys currently registered | 411 A list of dictionaries corresponding to all the keys currently registered |
| 422 """ | 412 """ |
| 423 return [dict(key) for key in self._data.keys()] | 413 return [dict(key) for key in self._data.keys()] |
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 465 """Get a Storage object to get/set a credential. | 455 """Get a Storage object to get/set a credential. |
| 466 | 456 |
| 467 This Storage is a 'view' into the multistore. | 457 This Storage is a 'view' into the multistore. |
| 468 | 458 |
| 469 Args: | 459 Args: |
| 470 key: The key used to retrieve the credential | 460 key: The key used to retrieve the credential |
| 471 | 461 |
| 472 Returns: | 462 Returns: |
| 473 A Storage object that can be used to get/set this cred | 463 A Storage object that can be used to get/set this cred |
| 474 """ | 464 """ |
| 475 return self._Storage(self, key) | 465 return self._Storage(self, key) |
| OLD | NEW |