| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # -*- coding: utf-8 -*- | |
| 3 # Copyright 2013 Google Inc. All Rights Reserved. | |
| 4 # | |
| 5 # Licensed under the Apache License, Version 2.0 (the "License"); | |
| 6 # you may not use this file except in compliance with the License. | |
| 7 # You may obtain a copy of the License at | |
| 8 # | |
| 9 # http://www.apache.org/licenses/LICENSE-2.0 | |
| 10 # | |
| 11 # Unless required by applicable law or agreed to in writing, software | |
| 12 # distributed under the License is distributed on an "AS IS" BASIS, | |
| 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 14 # See the License for the specific language governing permissions and | |
| 15 # limitations under the License. | |
| 16 """Main module for Google Cloud Storage command line tool.""" | |
| 17 | |
| 18 from __future__ import absolute_import | |
| 19 | |
| 20 import ConfigParser | |
| 21 import datetime | |
| 22 import errno | |
| 23 import getopt | |
| 24 import logging | |
| 25 import os | |
| 26 import re | |
| 27 import signal | |
| 28 import socket | |
| 29 import sys | |
| 30 import textwrap | |
| 31 import traceback | |
| 32 | |
| 33 # Load the gsutil version number and append it to boto.UserAgent so the value is | |
| 34 # set before anything instantiates boto. This has to run after THIRD_PARTY_DIR | |
| 35 # is modified (done in gsutil.py) but before any calls are made that would cause | |
| 36 # boto.s3.Connection to be loaded - otherwise the Connection class would end up | |
| 37 # with a static reference to the pre-modified version of the UserAgent field, | |
| 38 # so boto requests would not include gsutil/version# in the UserAgent string. | |
| 39 import boto | |
| 40 import gslib | |
| 41 # TODO: gsutil-beta: Cloud SDK scans for this string and performs | |
| 42 # substitution; ensure this works with both apitools and boto. | |
| 43 boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform) | |
| 44 if os.environ.get('CLOUDSDK_WRAPPER') == '1': | |
| 45 boto.UserAgent += ' Cloud SDK Command Line Tool' | |
| 46 if os.environ.get('CLOUDSDK_VERSION'): | |
| 47 boto.UserAgent += ' %s' % os.environ.get('CLOUDSDK_VERSION') | |
| 48 | |
| 49 # pylint: disable=g-bad-import-order | |
| 50 # pylint: disable=g-import-not-at-top | |
| 51 import httplib2 | |
| 52 import oauth2client | |
| 53 from gslib import wildcard_iterator | |
| 54 from gslib.cloud_api import AccessDeniedException | |
| 55 from gslib.cloud_api import ArgumentException | |
| 56 from gslib.cloud_api import BadRequestException | |
| 57 from gslib.cloud_api import ProjectIdException | |
| 58 from gslib.cloud_api import ServiceException | |
| 59 from gslib.command_runner import CommandRunner | |
| 60 import gslib.exception | |
| 61 from gslib.exception import CommandException | |
| 62 import apitools.base.py.exceptions as apitools_exceptions | |
| 63 from gslib.util import CreateLock | |
| 64 from gslib.util import GetBotoConfigFileList | |
| 65 from gslib.util import GetCertsFile | |
| 66 from gslib.util import GetCleanupFiles | |
| 67 from gslib.util import GsutilStreamHandler | |
| 68 from gslib.util import ProxyInfoFromEnvironmentVar | |
| 69 from gslib.sig_handling import GetCaughtSignals | |
| 70 from gslib.sig_handling import InitializeSignalHandling | |
| 71 from gslib.sig_handling import RegisterSignalHandler | |
| 72 | |
| 73 GSUTIL_CLIENT_ID = '909320924072.apps.googleusercontent.com' | |
| 74 # Google OAuth2 clients always have a secret, even if the client is an installed | |
| 75 # application/utility such as gsutil. Of course, in such cases the "secret" is | |
| 76 # actually publicly known; security depends entirely on the secrecy of refresh | |
| 77 # tokens, which effectively become bearer tokens. | |
| 78 GSUTIL_CLIENT_NOTSOSECRET = 'p3RlpR10xMFh9ZXBS/ZNLYUu' | |
| 79 if os.environ.get('CLOUDSDK_WRAPPER') == '1': | |
| 80 # Cloud SDK installs have a separate client ID / secret. | |
| 81 GSUTIL_CLIENT_ID = '32555940559.apps.googleusercontent.com' | |
| 82 GSUTIL_CLIENT_NOTSOSECRET = 'ZmssLNjJy2998hD4CTg2ejr2' | |
| 83 | |
| 84 CONFIG_KEYS_TO_REDACT = ['proxy', 'proxy_port', 'proxy_user', 'proxy_pass'] | |
| 85 | |
| 86 | |
| 87 # We don't use the oauth2 authentication plugin directly; importing it here | |
| 88 # ensures that it's loaded and available by default when an operation requiring | |
| 89 # authentication is performed. | |
| 90 try: | |
| 91 # pylint: disable=unused-import,g-import-not-at-top | |
| 92 import gcs_oauth2_boto_plugin | |
| 93 except ImportError: | |
| 94 pass | |
| 95 | |
| 96 DEBUG_WARNING = """ | |
| 97 ***************************** WARNING ***************************** | |
| 98 *** You are running gsutil with debug output enabled. | |
| 99 *** Be aware that debug output includes authentication credentials. | |
| 100 *** Make sure to remove the value of the Authorization header for | |
| 101 *** each HTTP request printed to the console prior to posting to | |
| 102 *** a public medium such as a forum post or Stack Overflow. | |
| 103 ***************************** WARNING ***************************** | |
| 104 """.lstrip() | |
| 105 | |
| 106 HTTP_WARNING = """ | |
| 107 ***************************** WARNING ***************************** | |
| 108 *** You are running gsutil with the "https_validate_certificates" config | |
| 109 *** variable set to False. This option should always be set to True in | |
| 110 *** production environments to protect against man-in-the-middle attacks, | |
| 111 *** and leaking of user data. | |
| 112 ***************************** WARNING ***************************** | |
| 113 """.lstrip() | |
| 114 | |
| 115 debug = 0 | |
| 116 test_exception_traces = False | |
| 117 | |
| 118 | |
| 119 # pylint: disable=unused-argument | |
| 120 def _CleanupSignalHandler(signal_num, cur_stack_frame): | |
| 121 """Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM.""" | |
| 122 _Cleanup() | |
| 123 | |
| 124 | |
| 125 def _Cleanup(): | |
| 126 for fname in GetCleanupFiles(): | |
| 127 try: | |
| 128 os.unlink(fname) | |
| 129 except: # pylint: disable=bare-except | |
| 130 pass | |
| 131 | |
| 132 | |
| 133 def _OutputAndExit(message): | |
| 134 """Outputs message and exists with code 1.""" | |
| 135 from gslib.util import UTF8 # pylint: disable=g-import-not-at-top | |
| 136 if debug >= 2 or test_exception_traces: | |
| 137 stack_trace = traceback.format_exc() | |
| 138 err = ('DEBUG: Exception stack trace:\n %s\n' % | |
| 139 re.sub('\\n', '\n ', stack_trace)) | |
| 140 else: | |
| 141 err = '%s\n' % message | |
| 142 try: | |
| 143 sys.stderr.write(err.encode(UTF8)) | |
| 144 except UnicodeDecodeError: | |
| 145 # Can happen when outputting invalid Unicode filenames. | |
| 146 sys.stderr.write(err) | |
| 147 sys.exit(1) | |
| 148 | |
| 149 | |
| 150 def _OutputUsageAndExit(command_runner): | |
| 151 command_runner.RunNamedCommand('help') | |
| 152 sys.exit(1) | |
| 153 | |
| 154 | |
| 155 class GsutilFormatter(logging.Formatter): | |
| 156 """A logging.Formatter that supports logging microseconds (%f).""" | |
| 157 | |
| 158 def formatTime(self, record, datefmt=None): | |
| 159 if datefmt: | |
| 160 return datetime.datetime.fromtimestamp(record.created).strftime(datefmt) | |
| 161 | |
| 162 # Use default implementation if datefmt is not specified. | |
| 163 return super(GsutilFormatter, self).formatTime(record, datefmt=datefmt) | |
| 164 | |
| 165 | |
| 166 def _ConfigureLogging(level=logging.INFO): | |
| 167 """Similar to logging.basicConfig() except it always adds a handler.""" | |
| 168 log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s' | |
| 169 date_format = '%m%d %H:%M:%S.%f' | |
| 170 formatter = GsutilFormatter(fmt=log_format, datefmt=date_format) | |
| 171 handler = GsutilStreamHandler() | |
| 172 handler.setFormatter(formatter) | |
| 173 root_logger = logging.getLogger() | |
| 174 root_logger.addHandler(handler) | |
| 175 root_logger.setLevel(level) | |
| 176 | |
| 177 | |
| 178 def main(): | |
| 179 InitializeSignalHandling() | |
| 180 # Any modules used in initializing multiprocessing variables must be | |
| 181 # imported after importing gslib.__main__. | |
| 182 # pylint: disable=redefined-outer-name,g-import-not-at-top | |
| 183 import gslib.boto_translation | |
| 184 import gslib.command | |
| 185 import gslib.util | |
| 186 from gslib.util import BOTO_IS_SECURE | |
| 187 from gslib.util import CERTIFICATE_VALIDATION_ENABLED | |
| 188 # pylint: disable=unused-variable | |
| 189 from gcs_oauth2_boto_plugin import oauth2_client | |
| 190 # pylint: enable=unused-variable | |
| 191 from gslib.util import MultiprocessingIsAvailable | |
| 192 if MultiprocessingIsAvailable()[0]: | |
| 193 # These setup methods must be called, and, on Windows, they can only be | |
| 194 # called from within an "if __name__ == '__main__':" block. | |
| 195 gslib.util.InitializeMultiprocessingVariables() | |
| 196 gslib.command.InitializeMultiprocessingVariables() | |
| 197 gslib.boto_translation.InitializeMultiprocessingVariables() | |
| 198 | |
| 199 # This needs to be done after gslib.util.InitializeMultiprocessingVariables(), | |
| 200 # since otherwise we can't call gslib.util.CreateLock. | |
| 201 try: | |
| 202 # pylint: disable=unused-import,g-import-not-at-top | |
| 203 import gcs_oauth2_boto_plugin | |
| 204 gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( | |
| 205 GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET) | |
| 206 gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock()) | |
| 207 except ImportError: | |
| 208 pass | |
| 209 | |
| 210 global debug | |
| 211 global test_exception_traces | |
| 212 | |
| 213 if not (2, 6) <= sys.version_info[:3] < (3,): | |
| 214 raise gslib.exception.CommandException( | |
| 215 'gsutil requires python 2.6 or 2.7.') | |
| 216 | |
| 217 # In gsutil 4.0 and beyond, we don't use the boto library for the JSON | |
| 218 # API. However, we still store gsutil configuration data in the .boto | |
| 219 # config file for compatibility with previous versions and user convenience. | |
| 220 # Many users have a .boto configuration file from previous versions, and it | |
| 221 # is useful to have all of the configuration for gsutil stored in one place. | |
| 222 command_runner = CommandRunner() | |
| 223 if not BOTO_IS_SECURE: | |
| 224 raise CommandException('\n'.join(textwrap.wrap( | |
| 225 'Your boto configuration has is_secure = False. Gsutil cannot be ' | |
| 226 'run this way, for security reasons.'))) | |
| 227 | |
| 228 headers = {} | |
| 229 parallel_operations = False | |
| 230 quiet = False | |
| 231 version = False | |
| 232 debug = 0 | |
| 233 test_exception_traces = False | |
| 234 | |
| 235 # If user enters no commands just print the usage info. | |
| 236 if len(sys.argv) == 1: | |
| 237 sys.argv.append('help') | |
| 238 | |
| 239 # Change the default of the 'https_validate_certificates' boto option to | |
| 240 # True (it is currently False in boto). | |
| 241 if not boto.config.has_option('Boto', 'https_validate_certificates'): | |
| 242 if not boto.config.has_section('Boto'): | |
| 243 boto.config.add_section('Boto') | |
| 244 boto.config.setbool('Boto', 'https_validate_certificates', True) | |
| 245 | |
| 246 gslib.util.certs_file_lock = CreateLock() | |
| 247 for signal_num in GetCaughtSignals(): | |
| 248 RegisterSignalHandler(signal_num, _CleanupSignalHandler) | |
| 249 GetCertsFile() | |
| 250 | |
| 251 try: | |
| 252 try: | |
| 253 opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq', | |
| 254 ['debug', 'detailedDebug', 'version', 'option', | |
| 255 'help', 'header', 'multithreaded', 'quiet', | |
| 256 'testexceptiontraces']) | |
| 257 except getopt.GetoptError as e: | |
| 258 _HandleCommandException(gslib.exception.CommandException(e.msg)) | |
| 259 for o, a in opts: | |
| 260 if o in ('-d', '--debug'): | |
| 261 # Passing debug=2 causes boto to include httplib header output. | |
| 262 debug = 3 | |
| 263 elif o in ('-D', '--detailedDebug'): | |
| 264 # We use debug level 3 to ask gsutil code to output more detailed | |
| 265 # debug output. This is a bit of a hack since it overloads the same | |
| 266 # flag that was originally implemented for boto use. And we use -DD | |
| 267 # to ask for really detailed debugging (i.e., including HTTP payload). | |
| 268 if debug == 3: | |
| 269 debug = 4 | |
| 270 else: | |
| 271 debug = 3 | |
| 272 elif o in ('-?', '--help'): | |
| 273 _OutputUsageAndExit(command_runner) | |
| 274 elif o in ('-h', '--header'): | |
| 275 (hdr_name, _, hdr_val) = a.partition(':') | |
| 276 if not hdr_name: | |
| 277 _OutputUsageAndExit(command_runner) | |
| 278 headers[hdr_name.lower()] = hdr_val | |
| 279 elif o in ('-m', '--multithreaded'): | |
| 280 parallel_operations = True | |
| 281 elif o in ('-q', '--quiet'): | |
| 282 quiet = True | |
| 283 elif o in ('-v', '--version'): | |
| 284 version = True | |
| 285 elif o == '--testexceptiontraces': # Hidden flag for integration tests. | |
| 286 test_exception_traces = True | |
| 287 elif o in ('-o', '--option'): | |
| 288 (opt_section_name, _, opt_value) = a.partition('=') | |
| 289 if not opt_section_name: | |
| 290 _OutputUsageAndExit(command_runner) | |
| 291 (opt_section, _, opt_name) = opt_section_name.partition(':') | |
| 292 if not opt_section or not opt_name: | |
| 293 _OutputUsageAndExit(command_runner) | |
| 294 if not boto.config.has_section(opt_section): | |
| 295 boto.config.add_section(opt_section) | |
| 296 boto.config.set(opt_section, opt_name, opt_value) | |
| 297 httplib2.debuglevel = debug | |
| 298 if debug > 1: | |
| 299 sys.stderr.write(DEBUG_WARNING) | |
| 300 if debug >= 2: | |
| 301 _ConfigureLogging(level=logging.DEBUG) | |
| 302 command_runner.RunNamedCommand('ver', ['-l']) | |
| 303 config_items = [] | |
| 304 try: | |
| 305 config_items.extend(boto.config.items('Boto')) | |
| 306 config_items.extend(boto.config.items('GSUtil')) | |
| 307 except ConfigParser.NoSectionError: | |
| 308 pass | |
| 309 for i in xrange(len(config_items)): | |
| 310 config_item_key = config_items[i][0] | |
| 311 if config_item_key in CONFIG_KEYS_TO_REDACT: | |
| 312 config_items[i] = (config_item_key, 'REDACTED') | |
| 313 sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) | |
| 314 sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList()) | |
| 315 sys.stderr.write('config: %s\n' % str(config_items)) | |
| 316 elif quiet: | |
| 317 _ConfigureLogging(level=logging.WARNING) | |
| 318 else: | |
| 319 _ConfigureLogging(level=logging.INFO) | |
| 320 # oauth2client uses info logging in places that would better | |
| 321 # correspond to gsutil's debug logging (e.g., when refreshing | |
| 322 # access tokens). | |
| 323 oauth2client.client.logger.setLevel(logging.WARNING) | |
| 324 | |
| 325 if not CERTIFICATE_VALIDATION_ENABLED: | |
| 326 sys.stderr.write(HTTP_WARNING) | |
| 327 | |
| 328 if version: | |
| 329 command_name = 'version' | |
| 330 elif not args: | |
| 331 command_name = 'help' | |
| 332 else: | |
| 333 command_name = args[0] | |
| 334 | |
| 335 _CheckAndWarnForProxyDifferences() | |
| 336 | |
| 337 if os.environ.get('_ARGCOMPLETE', '0') == '1': | |
| 338 return _PerformTabCompletion(command_runner) | |
| 339 | |
| 340 return _RunNamedCommandAndHandleExceptions( | |
| 341 command_runner, command_name, args=args[1:], headers=headers, | |
| 342 debug_level=debug, parallel_operations=parallel_operations) | |
| 343 finally: | |
| 344 _Cleanup() | |
| 345 | |
| 346 | |
| 347 def _CheckAndWarnForProxyDifferences(): | |
| 348 # If there are both boto config and environment variable config present for | |
| 349 # proxies, unset the environment variable and warn if it differs. | |
| 350 boto_port = boto.config.getint('Boto', 'proxy_port', 0) | |
| 351 if boto.config.get('Boto', 'proxy', None) or boto_port: | |
| 352 for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: | |
| 353 if proxy_env_var in os.environ and os.environ[proxy_env_var]: | |
| 354 differing_values = [] | |
| 355 proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) | |
| 356 if proxy_info.proxy_host != boto.config.get('Boto', 'proxy', None): | |
| 357 differing_values.append( | |
| 358 'Boto proxy host: "%s" differs from %s proxy host: "%s"' % | |
| 359 (boto.config.get('Boto', 'proxy', None), proxy_env_var, | |
| 360 proxy_info.proxy_host)) | |
| 361 if (proxy_info.proxy_user != | |
| 362 boto.config.get('Boto', 'proxy_user', None)): | |
| 363 differing_values.append( | |
| 364 'Boto proxy user: "%s" differs from %s proxy user: "%s"' % | |
| 365 (boto.config.get('Boto', 'proxy_user', None), proxy_env_var, | |
| 366 proxy_info.proxy_user)) | |
| 367 if (proxy_info.proxy_pass != | |
| 368 boto.config.get('Boto', 'proxy_pass', None)): | |
| 369 differing_values.append( | |
| 370 'Boto proxy password differs from %s proxy password' % | |
| 371 proxy_env_var) | |
| 372 # Only compare ports if at least one is present, since the | |
| 373 # boto logic for selecting default ports has not yet executed. | |
| 374 if ((proxy_info.proxy_port or boto_port) and | |
| 375 proxy_info.proxy_port != boto_port): | |
| 376 differing_values.append( | |
| 377 'Boto proxy port: "%s" differs from %s proxy port: "%s"' % | |
| 378 (boto_port, proxy_env_var, proxy_info.proxy_port)) | |
| 379 if differing_values: | |
| 380 sys.stderr.write('\n'.join(textwrap.wrap( | |
| 381 'WARNING: Proxy configuration is present in both the %s ' | |
| 382 'environment variable and boto configuration, but ' | |
| 383 'configuration differs. boto configuration proxy values will ' | |
| 384 'be used. Differences detected:' % proxy_env_var))) | |
| 385 sys.stderr.write('\n%s\n' % '\n'.join(differing_values)) | |
| 386 # Regardless of whether the proxy configuration values matched, | |
| 387 # delete the environment variable so as not to confuse boto. | |
| 388 del os.environ[proxy_env_var] | |
| 389 | |
| 390 | |
| 391 def _HandleUnknownFailure(e): | |
| 392 # Called if we fall through all known/handled exceptions. Allows us to | |
| 393 # print a stacktrace if -D option used. | |
| 394 if debug >= 2: | |
| 395 stack_trace = traceback.format_exc() | |
| 396 sys.stderr.write('DEBUG: Exception stack trace:\n %s\n' % | |
| 397 re.sub('\\n', '\n ', stack_trace)) | |
| 398 else: | |
| 399 _OutputAndExit('Failure: %s.' % e) | |
| 400 | |
| 401 | |
| 402 def _HandleCommandException(e): | |
| 403 if e.informational: | |
| 404 _OutputAndExit(e.reason) | |
| 405 else: | |
| 406 _OutputAndExit('CommandException: %s' % e.reason) | |
| 407 | |
| 408 | |
| 409 # pylint: disable=unused-argument | |
| 410 def _HandleControlC(signal_num, cur_stack_frame): | |
| 411 """Called when user hits ^C. | |
| 412 | |
| 413 This function prints a brief message instead of the normal Python stack trace | |
| 414 (unless -D option is used). | |
| 415 | |
| 416 Args: | |
| 417 signal_num: Signal that was caught. | |
| 418 cur_stack_frame: Unused. | |
| 419 """ | |
| 420 if debug >= 2: | |
| 421 stack_trace = ''.join(traceback.format_list(traceback.extract_stack())) | |
| 422 _OutputAndExit( | |
| 423 'DEBUG: Caught signal %d - Exception stack trace:\n' | |
| 424 ' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace))) | |
| 425 else: | |
| 426 _OutputAndExit('Caught signal %d - exiting' % signal_num) | |
| 427 | |
| 428 | |
| 429 def _HandleSigQuit(signal_num, cur_stack_frame): | |
| 430 """Called when user hits ^\\, so we can force breakpoint a running gsutil.""" | |
| 431 import pdb # pylint: disable=g-import-not-at-top | |
| 432 pdb.set_trace() | |
| 433 | |
| 434 | |
| 435 def _ConstructAccountProblemHelp(reason): | |
| 436 """Constructs a help string for an access control error. | |
| 437 | |
| 438 Args: | |
| 439 reason: e.reason string from caught exception. | |
| 440 | |
| 441 Returns: | |
| 442 Contructed help text. | |
| 443 """ | |
| 444 default_project_id = boto.config.get_value('GSUtil', 'default_project_id') | |
| 445 # pylint: disable=line-too-long, g-inconsistent-quotes | |
| 446 acct_help = ( | |
| 447 "Your request resulted in an AccountProblem (403) error. Usually this " | |
| 448 "happens if you attempt to create a bucket without first having " | |
| 449 "enabled billing for the project you are using. Please ensure billing is " | |
| 450 "enabled for your project by following the instructions at " | |
| 451 "`Google Developers Console<https://developers.google.com/console/help/bil
ling>`. ") | |
| 452 if default_project_id: | |
| 453 acct_help += ( | |
| 454 "In the project overview, ensure that the Project Number listed for " | |
| 455 "your project matches the project ID (%s) from your boto config file. " | |
| 456 % default_project_id) | |
| 457 acct_help += ( | |
| 458 "If the above doesn't resolve your AccountProblem, please send mail to " | |
| 459 "gs-team@google.com requesting assistance, noting the exact command you " | |
| 460 "ran, the fact that you received a 403 AccountProblem error, and your " | |
| 461 "project ID. Please do not post your project ID on StackOverflow. " | |
| 462 "Note: It's possible to use Google Cloud Storage without enabling " | |
| 463 "billing if you're only listing or reading objects for which you're " | |
| 464 "authorized, or if you're uploading objects to a bucket billed to a " | |
| 465 "project that has billing enabled. But if you're attempting to create " | |
| 466 "buckets or upload objects to a bucket owned by your own project, you " | |
| 467 "must first enable billing for that project.") | |
| 468 return acct_help | |
| 469 | |
| 470 | |
| 471 def _CheckAndHandleCredentialException(e, args): | |
| 472 # Provide detail to users who have no boto config file (who might previously | |
| 473 # have been using gsutil only for accessing publicly readable buckets and | |
| 474 # objects). | |
| 475 # pylint: disable=g-import-not-at-top | |
| 476 from gslib.util import HasConfiguredCredentials | |
| 477 if (not HasConfiguredCredentials() and | |
| 478 not boto.config.get_value('Tests', 'bypass_anonymous_access_warning', | |
| 479 False)): | |
| 480 # The check above allows tests to assert that we get a particular, | |
| 481 # expected failure, rather than always encountering this error message | |
| 482 # when there are no configured credentials. This allows tests to | |
| 483 # simulate a second user without permissions, without actually requiring | |
| 484 # two separate configured users. | |
| 485 if os.environ.get('CLOUDSDK_WRAPPER') == '1': | |
| 486 _OutputAndExit('\n'.join(textwrap.wrap( | |
| 487 'You are attempting to access protected data with no configured ' | |
| 488 'credentials. Please visit ' | |
| 489 'https://cloud.google.com/console#/project and sign up for an ' | |
| 490 'account, and then run the "gcloud auth login" command to ' | |
| 491 'configure gsutil to use these credentials.'))) | |
| 492 else: | |
| 493 _OutputAndExit('\n'.join(textwrap.wrap( | |
| 494 'You are attempting to access protected data with no configured ' | |
| 495 'credentials. Please visit ' | |
| 496 'https://cloud.google.com/console#/project and sign up for an ' | |
| 497 'account, and then run the "gsutil config" command to configure ' | |
| 498 'gsutil to use these credentials.'))) | |
| 499 elif (e.reason and | |
| 500 (e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or | |
| 501 'account for the specified project has been disabled' in e.reason) | |
| 502 and ','.join(args).find('gs://') != -1): | |
| 503 _OutputAndExit('\n'.join(textwrap.wrap( | |
| 504 _ConstructAccountProblemHelp(e.reason)))) | |
| 505 | |
| 506 | |
| 507 def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None, | |
| 508 headers=None, debug_level=0, | |
| 509 parallel_operations=False): | |
| 510 """Runs the command with the given command runner and arguments.""" | |
| 511 # pylint: disable=g-import-not-at-top | |
| 512 from gslib.util import GetConfigFilePath | |
| 513 from gslib.util import IS_WINDOWS | |
| 514 from gslib.util import IsRunningInteractively | |
| 515 try: | |
| 516 # Catch ^C so we can print a brief message instead of the normal Python | |
| 517 # stack trace. Register as a final signal handler because this handler kills | |
| 518 # the main gsutil process (so it must run last). | |
| 519 RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True) | |
| 520 # Catch ^\ so we can force a breakpoint in a running gsutil. | |
| 521 if not IS_WINDOWS: | |
| 522 RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit) | |
| 523 return command_runner.RunNamedCommand(command_name, args, headers, | |
| 524 debug_level, parallel_operations) | |
| 525 except AttributeError as e: | |
| 526 if str(e).find('secret_access_key') != -1: | |
| 527 _OutputAndExit('Missing credentials for the given URI(s). Does your ' | |
| 528 'boto config file contain all needed credentials?') | |
| 529 else: | |
| 530 _OutputAndExit(str(e)) | |
| 531 except gslib.exception.CommandException as e: | |
| 532 _HandleCommandException(e) | |
| 533 except getopt.GetoptError as e: | |
| 534 _HandleCommandException(gslib.exception.CommandException(e.msg)) | |
| 535 except boto.exception.InvalidUriError as e: | |
| 536 _OutputAndExit('InvalidUriError: %s.' % e.message) | |
| 537 except gslib.exception.InvalidUrlError as e: | |
| 538 _OutputAndExit('InvalidUrlError: %s.' % e.message) | |
| 539 except boto.auth_handler.NotReadyToAuthenticate: | |
| 540 _OutputAndExit('NotReadyToAuthenticate') | |
| 541 except OSError as e: | |
| 542 _OutputAndExit('OSError: %s.' % e.strerror) | |
| 543 except IOError as e: | |
| 544 if (e.errno == errno.EPIPE or (IS_WINDOWS and e.errno == errno.EINVAL) | |
| 545 and not IsRunningInteractively()): | |
| 546 # If we get a pipe error, this just means that the pipe to stdout or | |
| 547 # stderr is broken. This can happen if the user pipes gsutil to a command | |
| 548 # that doesn't use the entire output stream. Instead of raising an error, | |
| 549 # just swallow it up and exit cleanly. | |
| 550 sys.exit(0) | |
| 551 else: | |
| 552 raise | |
| 553 except wildcard_iterator.WildcardException as e: | |
| 554 _OutputAndExit(e.reason) | |
| 555 except ProjectIdException as e: | |
| 556 _OutputAndExit( | |
| 557 'You are attempting to perform an operation that requires a ' | |
| 558 'project id, with none configured. Please re-run ' | |
| 559 'gsutil config and make sure to follow the instructions for ' | |
| 560 'finding and entering your default project id.') | |
| 561 except BadRequestException as e: | |
| 562 if e.reason == 'MissingSecurityHeader': | |
| 563 _CheckAndHandleCredentialException(e, args) | |
| 564 _OutputAndExit(e) | |
| 565 except AccessDeniedException as e: | |
| 566 _CheckAndHandleCredentialException(e, args) | |
| 567 _OutputAndExit(e) | |
| 568 except ArgumentException as e: | |
| 569 _OutputAndExit(e) | |
| 570 except ServiceException as e: | |
| 571 _OutputAndExit(e) | |
| 572 except apitools_exceptions.HttpError as e: | |
| 573 # These should usually be retried by the underlying implementation or | |
| 574 # wrapped by CloudApi ServiceExceptions, but if we do get them, | |
| 575 # print something useful. | |
| 576 _OutputAndExit('HttpError: %s, %s' % (getattr(e.response, 'status', ''), | |
| 577 e.content or '')) | |
| 578 except socket.error as e: | |
| 579 if e.args[0] == errno.EPIPE: | |
| 580 # Retrying with a smaller file (per suggestion below) works because | |
| 581 # the library code send loop (in boto/s3/key.py) can get through the | |
| 582 # entire file and then request the HTTP response before the socket | |
| 583 # gets closed and the response lost. | |
| 584 _OutputAndExit( | |
| 585 'Got a "Broken pipe" error. This can happen to clients using Python ' | |
| 586 '2.x, when the server sends an error response and then closes the ' | |
| 587 'socket (see http://bugs.python.org/issue5542). If you are trying to ' | |
| 588 'upload a large object you might retry with a small (say 200k) ' | |
| 589 'object, and see if you get a more specific error code.' | |
| 590 ) | |
| 591 else: | |
| 592 _HandleUnknownFailure(e) | |
| 593 except Exception as e: | |
| 594 # Check for two types of errors related to service accounts. These errors | |
| 595 # appear to be the same except for their messages, but they are caused by | |
| 596 # different problems and both have unhelpful error messages. Moreover, | |
| 597 # the error type belongs to PyOpenSSL, which is not necessarily installed. | |
| 598 if 'mac verify failure' in str(e): | |
| 599 _OutputAndExit( | |
| 600 'Encountered an error while refreshing access token. ' | |
| 601 'If you are using a service account,\nplease verify that the ' | |
| 602 'gs_service_key_file_password field in your config file,' | |
| 603 '\n%s, is correct.' % GetConfigFilePath()) | |
| 604 elif 'asn1 encoding routines' in str(e): | |
| 605 _OutputAndExit( | |
| 606 'Encountered an error while refreshing access token. ' | |
| 607 'If you are using a service account,\nplease verify that the ' | |
| 608 'gs_service_key_file field in your config file,\n%s, is correct.' | |
| 609 % GetConfigFilePath()) | |
| 610 _HandleUnknownFailure(e) | |
| 611 | |
| 612 | |
| 613 def _PerformTabCompletion(command_runner): | |
| 614 """Performs gsutil-specific tab completion for the shell.""" | |
| 615 # argparse and argcomplete are bundled with the Google Cloud SDK. | |
| 616 # When gsutil is invoked from the Google Cloud SDK, both should be available. | |
| 617 try: | |
| 618 import argcomplete | |
| 619 import argparse | |
| 620 except ImportError as e: | |
| 621 _OutputAndExit('A library required for performing tab completion was' | |
| 622 ' not found.\nCause: %s' % e) | |
| 623 parser = argparse.ArgumentParser(add_help=False) | |
| 624 subparsers = parser.add_subparsers() | |
| 625 command_runner.ConfigureCommandArgumentParsers(subparsers) | |
| 626 argcomplete.autocomplete(parser, exit_method=sys.exit) | |
| 627 | |
| 628 return 0 | |
| 629 | |
| 630 if __name__ == '__main__': | |
| 631 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1260493004:
Revert "Add gsutil 4.13 to telemetry/third_party" (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master