OLD | NEW |
1 # Copyright 2011 Google Inc. | 1 # Copyright 2014 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Licensed under the Apache License, Version 2.0 (the "License"); | 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
4 # you may not use this file except in compliance with the License. | 4 # you may not use this file except in compliance with the License. |
5 # You may obtain a copy of the License at | 5 # You may obtain a copy of the License at |
6 # | 6 # |
7 # http://www.apache.org/licenses/LICENSE-2.0 | 7 # http://www.apache.org/licenses/LICENSE-2.0 |
8 # | 8 # |
9 # Unless required by applicable law or agreed to in writing, software | 9 # Unless required by applicable law or agreed to in writing, software |
10 # distributed under the License is distributed on an "AS IS" BASIS, | 10 # distributed under the License is distributed on an "AS IS" BASIS, |
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 # See the License for the specific language governing permissions and | 12 # See the License for the specific language governing permissions and |
13 # limitations under the License. | 13 # limitations under the License. |
14 | 14 |
15 """Multi-credential file store with lock support. | 15 """Multi-credential file store with lock support. |
16 | 16 |
17 This module implements a JSON credential store where multiple | 17 This module implements a JSON credential store where multiple |
18 credentials can be stored in one file. That file supports locking | 18 credentials can be stored in one file. That file supports locking |
19 both in a single process and across processes. | 19 both in a single process and across processes. |
20 | 20 |
21 The credential themselves are keyed off of: | 21 The credential themselves are keyed off of: |
| 22 |
22 * client_id | 23 * client_id |
23 * user_agent | 24 * user_agent |
24 * scope | 25 * scope |
25 | 26 |
26 The format of the stored data is like so: | 27 The format of the stored data is like so:: |
27 { | 28 |
28 'file_version': 1, | 29 { |
29 'data': [ | 30 'file_version': 1, |
30 { | 31 'data': [ |
31 'key': { | 32 { |
32 'clientId': '<client id>', | 33 'key': { |
33 'userAgent': '<user agent>', | 34 'clientId': '<client id>', |
34 'scope': '<scope>' | 35 'userAgent': '<user agent>', |
35 }, | 36 'scope': '<scope>' |
36 'credential': { | 37 }, |
37 # JSON serialized Credentials. | 38 'credential': { |
| 39 # JSON serialized Credentials. |
| 40 } |
38 } | 41 } |
39 } | 42 ] |
40 ] | 43 } |
41 } | 44 |
42 """ | 45 """ |
43 | 46 |
44 __author__ = 'jbeda@google.com (Joe Beda)' | 47 __author__ = 'jbeda@google.com (Joe Beda)' |
45 | 48 |
46 import base64 | |
47 import errno | 49 import errno |
| 50 import json |
48 import logging | 51 import logging |
49 import os | 52 import os |
50 import threading | 53 import threading |
51 | 54 |
52 from anyjson import simplejson | 55 from .client import Credentials |
53 from .client import Storage as BaseStorage | 56 from .client import Storage as BaseStorage |
54 from .client import Credentials | |
55 from . import util | 57 from . import util |
56 from locked_file import LockedFile | 58 from locked_file import LockedFile |
57 | 59 |
58 logger = logging.getLogger(__name__) | 60 logger = logging.getLogger(__name__) |
59 | 61 |
60 # A dict from 'filename'->_MultiStore instances | 62 # A dict from 'filename'->_MultiStore instances |
61 _multistores = {} | 63 _multistores = {} |
62 _multistores_lock = threading.Lock() | 64 _multistores_lock = threading.Lock() |
63 | 65 |
64 | 66 |
65 class Error(Exception): | 67 class Error(Exception): |
66 """Base error for this module.""" | 68 """Base error for this module.""" |
67 pass | |
68 | 69 |
69 | 70 |
70 class NewerCredentialStoreError(Error): | 71 class NewerCredentialStoreError(Error): |
71 """The credential store is a newer version that supported.""" | 72 """The credential store is a newer version than supported.""" |
72 pass | |
73 | 73 |
74 | 74 |
75 @util.positional(4) | 75 @util.positional(4) |
76 def get_credential_storage(filename, client_id, user_agent, scope, | 76 def get_credential_storage(filename, client_id, user_agent, scope, |
77 warn_on_readonly=True): | 77 warn_on_readonly=True): |
78 """Get a Storage instance for a credential. | 78 """Get a Storage instance for a credential. |
79 | 79 |
80 Args: | 80 Args: |
81 filename: The JSON file storing a set of credentials | 81 filename: The JSON file storing a set of credentials |
82 client_id: The client_id for the credential | 82 client_id: The client_id for the credential |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
186 | 186 |
187 class _MultiStore(object): | 187 class _MultiStore(object): |
188 """A file backed store for multiple credentials.""" | 188 """A file backed store for multiple credentials.""" |
189 | 189 |
190 @util.positional(2) | 190 @util.positional(2) |
191 def __init__(self, filename, warn_on_readonly=True): | 191 def __init__(self, filename, warn_on_readonly=True): |
192 """Initialize the class. | 192 """Initialize the class. |
193 | 193 |
194 This will create the file if necessary. | 194 This will create the file if necessary. |
195 """ | 195 """ |
196 self._file = LockedFile(filename, 'r+b', 'rb') | 196 self._file = LockedFile(filename, 'r+', 'r') |
197 self._thread_lock = threading.Lock() | 197 self._thread_lock = threading.Lock() |
198 self._read_only = False | 198 self._read_only = False |
199 self._warn_on_readonly = warn_on_readonly | 199 self._warn_on_readonly = warn_on_readonly |
200 | 200 |
201 self._create_file_if_needed() | 201 self._create_file_if_needed() |
202 | 202 |
203 # Cache of deserialized store. This is only valid after the | 203 # Cache of deserialized store. This is only valid after the |
204 # _MultiStore is locked or _refresh_data_cache is called. This is | 204 # _MultiStore is locked or _refresh_data_cache is called. This is |
205 # of the form of: | 205 # of the form of: |
206 # | 206 # |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
264 """ | 264 """ |
265 self._multistore._delete_credential(self._key) | 265 self._multistore._delete_credential(self._key) |
266 | 266 |
267 def _create_file_if_needed(self): | 267 def _create_file_if_needed(self): |
268 """Create an empty file if necessary. | 268 """Create an empty file if necessary. |
269 | 269 |
270 This method will not initialize the file. Instead it implements a | 270 This method will not initialize the file. Instead it implements a |
271 simple version of "touch" to ensure the file has been created. | 271 simple version of "touch" to ensure the file has been created. |
272 """ | 272 """ |
273 if not os.path.exists(self._file.filename()): | 273 if not os.path.exists(self._file.filename()): |
274 old_umask = os.umask(0177) | 274 old_umask = os.umask(0o177) |
275 try: | 275 try: |
276 open(self._file.filename(), 'a+b').close() | 276 open(self._file.filename(), 'a+b').close() |
277 finally: | 277 finally: |
278 os.umask(old_umask) | 278 os.umask(old_umask) |
279 | 279 |
280 def _lock(self): | 280 def _lock(self): |
281 """Lock the entire multistore.""" | 281 """Lock the entire multistore.""" |
282 self._thread_lock.acquire() | 282 self._thread_lock.acquire() |
283 self._file.open_and_lock() | 283 try: |
| 284 self._file.open_and_lock() |
| 285 except IOError as e: |
| 286 if e.errno == errno.ENOSYS: |
| 287 logger.warn('File system does not support locking the credentials ' |
| 288 'file.') |
| 289 elif e.errno == errno.ENOLCK: |
| 290 logger.warn('File system is out of resources for writing the ' |
| 291 'credentials file (is your disk full?).') |
| 292 else: |
| 293 raise |
284 if not self._file.is_locked(): | 294 if not self._file.is_locked(): |
285 self._read_only = True | 295 self._read_only = True |
286 if self._warn_on_readonly: | 296 if self._warn_on_readonly: |
287 logger.warn('The credentials file (%s) is not writable. Opening in ' | 297 logger.warn('The credentials file (%s) is not writable. Opening in ' |
288 'read-only mode. Any refreshed credentials will only be ' | 298 'read-only mode. Any refreshed credentials will only be ' |
289 'valid for this run.' % self._file.filename()) | 299 'valid for this run.', self._file.filename()) |
290 if os.path.getsize(self._file.filename()) == 0: | 300 if os.path.getsize(self._file.filename()) == 0: |
291 logger.debug('Initializing empty multistore file') | 301 logger.debug('Initializing empty multistore file') |
292 # The multistore is empty so write out an empty file. | 302 # The multistore is empty so write out an empty file. |
293 self._data = {} | 303 self._data = {} |
294 self._write() | 304 self._write() |
295 elif not self._read_only or self._data is None: | 305 elif not self._read_only or self._data is None: |
296 # Only refresh the data if we are read/write or we haven't | 306 # Only refresh the data if we are read/write or we haven't |
297 # cached the data yet. If we are readonly, we assume is isn't | 307 # cached the data yet. If we are readonly, we assume is isn't |
298 # changing out from under us and that we only have to read it | 308 # changing out from under us and that we only have to read it |
299 # once. This prevents us from whacking any new access keys that | 309 # once. This prevents us from whacking any new access keys that |
300 # we have cached in memory but were unable to write out. | 310 # we have cached in memory but were unable to write out. |
301 self._refresh_data_cache() | 311 self._refresh_data_cache() |
302 | 312 |
303 def _unlock(self): | 313 def _unlock(self): |
304 """Release the lock on the multistore.""" | 314 """Release the lock on the multistore.""" |
305 self._file.unlock_and_close() | 315 self._file.unlock_and_close() |
306 self._thread_lock.release() | 316 self._thread_lock.release() |
307 | 317 |
308 def _locked_json_read(self): | 318 def _locked_json_read(self): |
309 """Get the raw content of the multistore file. | 319 """Get the raw content of the multistore file. |
310 | 320 |
311 The multistore must be locked when this is called. | 321 The multistore must be locked when this is called. |
312 | 322 |
313 Returns: | 323 Returns: |
314 The contents of the multistore decoded as JSON. | 324 The contents of the multistore decoded as JSON. |
315 """ | 325 """ |
316 assert self._thread_lock.locked() | 326 assert self._thread_lock.locked() |
317 self._file.file_handle().seek(0) | 327 self._file.file_handle().seek(0) |
318 return simplejson.load(self._file.file_handle()) | 328 return json.load(self._file.file_handle()) |
319 | 329 |
320 def _locked_json_write(self, data): | 330 def _locked_json_write(self, data): |
321 """Write a JSON serializable data structure to the multistore. | 331 """Write a JSON serializable data structure to the multistore. |
322 | 332 |
323 The multistore must be locked when this is called. | 333 The multistore must be locked when this is called. |
324 | 334 |
325 Args: | 335 Args: |
326 data: The data to be serialized and written. | 336 data: The data to be serialized and written. |
327 """ | 337 """ |
328 assert self._thread_lock.locked() | 338 assert self._thread_lock.locked() |
329 if self._read_only: | 339 if self._read_only: |
330 return | 340 return |
331 self._file.file_handle().seek(0) | 341 self._file.file_handle().seek(0) |
332 simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2) | 342 json.dump(data, self._file.file_handle(), sort_keys=True, indent=2, separato
rs=(',', ': ')) |
333 self._file.file_handle().truncate() | 343 self._file.file_handle().truncate() |
334 | 344 |
335 def _refresh_data_cache(self): | 345 def _refresh_data_cache(self): |
336 """Refresh the contents of the multistore. | 346 """Refresh the contents of the multistore. |
337 | 347 |
338 The multistore must be locked when this is called. | 348 The multistore must be locked when this is called. |
339 | 349 |
340 Raises: | 350 Raises: |
341 NewerCredentialStoreError: Raised when a newer client has written the | 351 NewerCredentialStoreError: Raised when a newer client has written the |
342 store. | 352 store. |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
380 Args: | 390 Args: |
381 cred_entry: A dict entry from the data member of our format | 391 cred_entry: A dict entry from the data member of our format |
382 | 392 |
383 Returns: | 393 Returns: |
384 (key, cred) where the key is the key tuple and the cred is the | 394 (key, cred) where the key is the key tuple and the cred is the |
385 OAuth2Credential object. | 395 OAuth2Credential object. |
386 """ | 396 """ |
387 raw_key = cred_entry['key'] | 397 raw_key = cred_entry['key'] |
388 key = util.dict_to_tuple_key(raw_key) | 398 key = util.dict_to_tuple_key(raw_key) |
389 credential = None | 399 credential = None |
390 credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credenti
al'])) | 400 credential = Credentials.new_from_json(json.dumps(cred_entry['credential'])) |
391 return (key, credential) | 401 return (key, credential) |
392 | 402 |
393 def _write(self): | 403 def _write(self): |
394 """Write the cached data back out. | 404 """Write the cached data back out. |
395 | 405 |
396 The multistore must be locked. | 406 The multistore must be locked. |
397 """ | 407 """ |
398 raw_data = {'file_version': 1} | 408 raw_data = {'file_version': 1} |
399 raw_creds = [] | 409 raw_creds = [] |
400 raw_data['data'] = raw_creds | 410 raw_data['data'] = raw_creds |
401 for (cred_key, cred) in self._data.items(): | 411 for (cred_key, cred) in self._data.items(): |
402 raw_key = dict(cred_key) | 412 raw_key = dict(cred_key) |
403 raw_cred = simplejson.loads(cred.to_json()) | 413 raw_cred = json.loads(cred.to_json()) |
404 raw_creds.append({'key': raw_key, 'credential': raw_cred}) | 414 raw_creds.append({'key': raw_key, 'credential': raw_cred}) |
405 self._locked_json_write(raw_data) | 415 self._locked_json_write(raw_data) |
406 | 416 |
407 def _get_all_credential_keys(self): | 417 def _get_all_credential_keys(self): |
408 """Gets all the registered credential keys in the multistore. | 418 """Gets all the registered credential keys in the multistore. |
409 | 419 |
410 Returns: | 420 Returns: |
411 A list of dictionaries corresponding to all the keys currently registered | 421 A list of dictionaries corresponding to all the keys currently registered |
412 """ | 422 """ |
413 return [dict(key) for key in self._data.keys()] | 423 return [dict(key) for key in self._data.keys()] |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
455 """Get a Storage object to get/set a credential. | 465 """Get a Storage object to get/set a credential. |
456 | 466 |
457 This Storage is a 'view' into the multistore. | 467 This Storage is a 'view' into the multistore. |
458 | 468 |
459 Args: | 469 Args: |
460 key: The key used to retrieve the credential | 470 key: The key used to retrieve the credential |
461 | 471 |
462 Returns: | 472 Returns: |
463 A Storage object that can be used to get/set this cred | 473 A Storage object that can be used to get/set this cred |
464 """ | 474 """ |
465 return self._Storage(self, key) | 475 return self._Storage(self, key) |
OLD | NEW |