| OLD | NEW |
| 1 # -*- coding: utf-8 -*- |
| 1 # Copyright 2013 Google Inc. All Rights Reserved. | 2 # Copyright 2013 Google Inc. All Rights Reserved. |
| 2 # | 3 # |
| 3 # Licensed under the Apache License, Version 2.0 (the "License"); | 4 # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 # you may not use this file except in compliance with the License. | 5 # you may not use this file except in compliance with the License. |
| 5 # You may obtain a copy of the License at | 6 # You may obtain a copy of the License at |
| 6 # | 7 # |
| 7 # http://www.apache.org/licenses/LICENSE-2.0 | 8 # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 # | 9 # |
| 9 # Unless required by applicable law or agreed to in writing, software | 10 # Unless required by applicable law or agreed to in writing, software |
| 10 # distributed under the License is distributed on an "AS IS" BASIS, | 11 # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 # See the License for the specific language governing permissions and | 13 # See the License for the specific language governing permissions and |
| 13 # limitations under the License. | 14 # limitations under the License. |
| 15 """Additional help about CRC32C and installing crcmod.""" |
| 14 | 16 |
| 15 from gslib.help_provider import HELP_NAME | 17 from __future__ import absolute_import |
| 16 from gslib.help_provider import HELP_NAME_ALIASES | 18 |
| 17 from gslib.help_provider import HELP_ONE_LINE_SUMMARY | |
| 18 from gslib.help_provider import HelpProvider | 19 from gslib.help_provider import HelpProvider |
| 19 from gslib.help_provider import HELP_TEXT | |
| 20 from gslib.help_provider import HelpType | |
| 21 from gslib.help_provider import HELP_TYPE | |
| 22 | 20 |
| 23 _detailed_help_text = (""" | 21 _DETAILED_HELP_TEXT = (""" |
| 24 <B>OVERVIEW</B> | 22 <B>OVERVIEW</B> |
| 25 Google Cloud Storage provides a cyclic redundancy check (CRC) header that | 23 Google Cloud Storage provides a cyclic redundancy check (CRC) header that |
| 26 allows clients to verify the integrity of composite object contents. All other | 24 allows clients to verify the integrity of object contents. For non-composite |
| 27 download operations currently use MD5 for integrity checks, but support for | 25 objects GCS also provides an MD5 header to allow clients to verify object |
| 28 CRC may be added in the future. The CRC variant used by Google Cloud Storage | 26 integrity, but for composite objects only the CRC is available. gsutil |
| 29 is called CRC32C (Castagnoli), which is not available in the standard Python | 27 automatically performs integrity checks on all uploads and downloads. |
| 30 distribution. The implementation of CRC32C used by gsutil is provided by a | 28 Additionally, you can use the "gsutil hash" command to calculate a CRC for |
| 31 third-party Python module called | 29 any local file. |
| 30 |
| 31 The CRC variant used by Google Cloud Storage is called CRC32C (Castagnoli), |
| 32 which is not available in the standard Python distribution. The implementation |
| 33 of CRC32C used by gsutil is provided by a third-party Python module called |
| 32 `crcmod <https://pypi.python.org/pypi/crcmod>`_. | 34 `crcmod <https://pypi.python.org/pypi/crcmod>`_. |
| 33 | 35 |
| 34 The crcmod module contains a pure-Python implementation of CRC32C, but using | 36 The crcmod module contains a pure-Python implementation of CRC32C, but using |
| 35 it results in degraded performance, as the CPU becomes the bottleneck for | 37 it results in very poor performance. A Python C extension is also provided by |
| 36 transfers. A Python C extension is also provided by crcmod, which requires | 38 crcmod, which requires compiling into a binary module for use. gsutil ships |
| 37 compiling into a binary module for use. gsutil ships with a precompiled | 39 with a precompiled crcmod C extension for Mac OS X; for other platforms, see |
| 38 crcmod C extension for Mac OS X; for other platforms, see installation | 40 the installation instructions below. |
| 39 instructions below. | |
| 40 | 41 |
| 41 Since gsutil is platform agnostic, the | 42 At the end of each copy operation, the gsutil cp and rsync commands validate |
| 42 compiled version of crcmod is not distributed with the gsutil release. | 43 that the checksum of the source file/object matches the checksum of the |
| 44 destination file/object. If the checksums do not match, gsutil will delete |
| 45 the invalid copy and print a warning message. This very rarely happens, but |
| 46 if it does, please contact gs-team@google.com. |
| 43 | 47 |
| 44 | 48 |
| 45 <B>CONFIGURATION</B> | 49 <B>CONFIGURATION</B> |
| 46 To determine if the compiled version of crcmod is available in your Python | 50 To determine if the compiled version of crcmod is available in your Python |
| 47 environment, you can inspect the output of the gsutil version command for the | 51 environment, you can inspect the output of the gsutil version command for the |
| 48 "compiled crcmod" entry:: | 52 "compiled crcmod" entry:: |
| 49 | 53 |
| 50 $ gsutil version -l | 54 $ gsutil version -l |
| 51 ... | 55 ... |
| 52 compiled crcmod: True | 56 compiled crcmod: True |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 106 https://pypi.python.org/pypi/crcmod/1.7 | 110 https://pypi.python.org/pypi/crcmod/1.7 |
| 107 | 111 |
| 108 MSI installers are available for the 32-bit versions of Python 2.6 and 2.7. | 112 MSI installers are available for the 32-bit versions of Python 2.6 and 2.7. |
| 109 | 113 |
| 110 """) | 114 """) |
| 111 | 115 |
| 112 | 116 |
| 113 class CommandOptions(HelpProvider): | 117 class CommandOptions(HelpProvider): |
| 114 """Additional help about CRC32C and installing crcmod.""" | 118 """Additional help about CRC32C and installing crcmod.""" |
| 115 | 119 |
| 116 help_spec = { | 120 # Help specification. See help_provider.py for documentation. |
| 117 # Name of command or auxiliary help info for which this help applies. | 121 help_spec = HelpProvider.HelpSpec( |
| 118 HELP_NAME : 'crc32c', | 122 help_name='crc32c', |
| 119 # List of help name aliases. | 123 help_name_aliases=['crc32', 'crc', 'crcmod'], |
| 120 HELP_NAME_ALIASES : ['crc32', 'crc', 'crcmod'], | 124 help_type='additional_help', |
| 121 # Type of help: | 125 help_one_line_summary='CRC32C and Installing crcmod', |
| 122 HELP_TYPE : HelpType.ADDITIONAL_HELP, | 126 help_text=_DETAILED_HELP_TEXT, |
| 123 # One line summary of this help. | 127 subcommand_help_text={}, |
| 124 HELP_ONE_LINE_SUMMARY : 'CRC32C and Installing crcmod', | 128 ) |
| 125 # The full help text. | |
| 126 HELP_TEXT : _detailed_help_text, | |
| 127 } | |
| OLD | NEW |