Roll version of gsutil to 4.15.

third_party/gsutil/gslib/__main__.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright 2013 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
@@ skipping 85 matching lines @@
 DEBUG_WARNING = """
 ***************************** WARNING *****************************
 *** You are running gsutil with debug output enabled.
 *** Be aware that debug output includes authentication credentials.
 *** Make sure to remove the value of the Authorization header for
 *** each HTTP request printed to the console prior to posting to
 *** a public medium such as a forum post or Stack Overflow.
 ***************************** WARNING *****************************
 """.lstrip()
 
+TRACE_WARNING = """
+***************************** WARNING *****************************
+*** You are running gsutil with trace output enabled.
+*** Be aware that trace output includes authentication credentials
+*** and may include the contents of any files accessed during the trace.
+***************************** WARNING *****************************
+""".lstrip()
+
 HTTP_WARNING = """
 ***************************** WARNING *****************************
 *** You are running gsutil with the "https_validate_certificates" config
 *** variable set to False. This option should always be set to True in
 *** production environments to protect against man-in-the-middle attacks,
 *** and leaking of user data.
 ***************************** WARNING *****************************
 """.lstrip()
 
 debug = 0
@@ skipping 64 matching lines @@
   # Any modules used in initializing multiprocessing variables must be
   # imported after importing gslib.__main__.
   # pylint: disable=redefined-outer-name,g-import-not-at-top
   import gslib.boto_translation
   import gslib.command
   import gslib.util
   from gslib.util import BOTO_IS_SECURE
   from gslib.util import CERTIFICATE_VALIDATION_ENABLED
   # pylint: disable=unused-variable
   from gcs_oauth2_boto_plugin import oauth2_client
+  from apitools.base.py import credentials_lib
   # pylint: enable=unused-variable
-  from gslib.util import MultiprocessingIsAvailable
-  if MultiprocessingIsAvailable()[0]:
+  from gslib.util import CheckMultiprocessingAvailableAndInit
+  if CheckMultiprocessingAvailableAndInit().is_available:
     # These setup methods must be called, and, on Windows, they can only be
     # called from within an "if __name__ == '__main__':" block.
-    gslib.util.InitializeMultiprocessingVariables()
     gslib.command.InitializeMultiprocessingVariables()
     gslib.boto_translation.InitializeMultiprocessingVariables()
+  else:
+    gslib.command.InitializeThreadingVariables()
 
   # This needs to be done after gslib.util.InitializeMultiprocessingVariables(),
   # since otherwise we can't call gslib.util.CreateLock.
   try:
     # pylint: disable=unused-import,g-import-not-at-top
     import gcs_oauth2_boto_plugin
     gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
         GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET)
     gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock())
+    credentials_lib.SetCredentialsCacheFileLock(CreateLock())
   except ImportError:
     pass
 
   global debug
   global test_exception_traces
 
   if not (2, 6) <= sys.version_info[:3] < (3,):
     raise gslib.exception.CommandException(
         'gsutil requires python 2.6 or 2.7.')
 
   # In gsutil 4.0 and beyond, we don't use the boto library for the JSON
   # API. However, we still store gsutil configuration data in the .boto
   # config file for compatibility with previous versions and user convenience.
   # Many users have a .boto configuration file from previous versions, and it
   # is useful to have all of the configuration for gsutil stored in one place.
   command_runner = CommandRunner()
   if not BOTO_IS_SECURE:
     raise CommandException('\n'.join(textwrap.wrap(
         'Your boto configuration has is_secure = False. Gsutil cannot be '
         'run this way, for security reasons.')))
 
   headers = {}
   parallel_operations = False
   quiet = False
   version = False
   debug = 0
+  trace_token = None
   test_exception_traces = False
 
   # If user enters no commands just print the usage info.
   if len(sys.argv) == 1:
     sys.argv.append('help')
 
   # Change the default of the 'https_validate_certificates' boto option to
   # True (it is currently False in boto).
   if not boto.config.has_option('Boto', 'https_validate_certificates'):
     if not boto.config.has_section('Boto'):
       boto.config.add_section('Boto')
     boto.config.setbool('Boto', 'https_validate_certificates', True)
 
   gslib.util.certs_file_lock = CreateLock()
   for signal_num in GetCaughtSignals():
     RegisterSignalHandler(signal_num, _CleanupSignalHandler)
   GetCertsFile()
 
   try:
     try:
       opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq',
                                  ['debug', 'detailedDebug', 'version', 'option',
                                   'help', 'header', 'multithreaded', 'quiet',
-                                  'testexceptiontraces'])
+                                  'testexceptiontraces', 'trace-token='])
     except getopt.GetoptError as e:
       _HandleCommandException(gslib.exception.CommandException(e.msg))
     for o, a in opts:
       if o in ('-d', '--debug'):
         # Passing debug=2 causes boto to include httplib header output.
         debug = 3
       elif o in ('-D', '--detailedDebug'):
         # We use debug level 3 to ask gsutil code to output more detailed
         # debug output. This is a bit of a hack since it overloads the same
         # flag that was originally implemented for boto use. And we use -DD
         # to ask for really detailed debugging (i.e., including HTTP payload).
         if debug == 3:
           debug = 4
         else:
           debug = 3
       elif o in ('-?', '--help'):
         _OutputUsageAndExit(command_runner)
       elif o in ('-h', '--header'):
         (hdr_name, _, hdr_val) = a.partition(':')
         if not hdr_name:
           _OutputUsageAndExit(command_runner)
         headers[hdr_name.lower()] = hdr_val
       elif o in ('-m', '--multithreaded'):
         parallel_operations = True
       elif o in ('-q', '--quiet'):
         quiet = True
       elif o in ('-v', '--version'):
         version = True
+      elif o == '--trace-token':
+        trace_token = a
       elif o == '--testexceptiontraces':  # Hidden flag for integration tests.
         test_exception_traces = True
       elif o in ('-o', '--option'):
         (opt_section_name, _, opt_value) = a.partition('=')
         if not opt_section_name:
           _OutputUsageAndExit(command_runner)
         (opt_section, _, opt_name) = opt_section_name.partition(':')
         if not opt_section or not opt_name:
           _OutputUsageAndExit(command_runner)
         if not boto.config.has_section(opt_section):
           boto.config.add_section(opt_section)
         boto.config.set(opt_section, opt_name, opt_value)
     httplib2.debuglevel = debug
+    if trace_token:
+      sys.stderr.write(TRACE_WARNING)
     if debug > 1:
       sys.stderr.write(DEBUG_WARNING)
     if debug >= 2:
       _ConfigureLogging(level=logging.DEBUG)
       command_runner.RunNamedCommand('ver', ['-l'])
       config_items = []
       try:
         config_items.extend(boto.config.items('Boto'))
         config_items.extend(boto.config.items('GSUtil'))
       except ConfigParser.NoSectionError:
@@ skipping 24 matching lines @@
     else:
       command_name = args[0]
 
     _CheckAndWarnForProxyDifferences()
 
     if os.environ.get('_ARGCOMPLETE', '0') == '1':
       return _PerformTabCompletion(command_runner)
 
     return _RunNamedCommandAndHandleExceptions(
         command_runner, command_name, args=args[1:], headers=headers,
-        debug_level=debug, parallel_operations=parallel_operations)
+        debug_level=debug, trace_token=trace_token,
+        parallel_operations=parallel_operations)
   finally:
     _Cleanup()
 
 
 def _CheckAndWarnForProxyDifferences():
   # If there are both boto config and environment variable config present for
   # proxies, unset the environment variable and warn if it differs.
   boto_port = boto.config.getint('Boto', 'proxy_port', 0)
   if boto.config.get('Boto', 'proxy', None) or boto_port:
     for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
@@ skipping 29 matching lines @@
               'environment variable and boto configuration, but '
               'configuration differs. boto configuration proxy values will '
               'be used. Differences detected:' % proxy_env_var)))
           sys.stderr.write('\n%s\n' % '\n'.join(differing_values))
         # Regardless of whether the proxy configuration values matched,
         # delete the environment variable so as not to confuse boto.
         del os.environ[proxy_env_var]
 
 
 def _HandleUnknownFailure(e):
-  # Called if we fall through all known/handled exceptions. Allows us to
-  # print a stacktrace if -D option used.
-  if debug >= 2:
-    stack_trace = traceback.format_exc()
-    sys.stderr.write('DEBUG: Exception stack trace:\n    %s\n' %
-                     re.sub('\\n', '\n    ', stack_trace))
-  else:
-    _OutputAndExit('Failure: %s.' % e)
+  # Called if we fall through all known/handled exceptions.
+  _OutputAndExit('Failure: %s.' % e)
 
 
 def _HandleCommandException(e):
   if e.informational:
     _OutputAndExit(e.reason)
   else:
     _OutputAndExit('CommandException: %s' % e.reason)
 
 
 # pylint: disable=unused-argument
@@ skipping 89 matching lines @@
   elif (e.reason and
         (e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or
          'account for the specified project has been disabled' in e.reason)
         and ','.join(args).find('gs://') != -1):
     _OutputAndExit('\n'.join(textwrap.wrap(
         _ConstructAccountProblemHelp(e.reason))))
 
 
 def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
                                         headers=None, debug_level=0,
+                                        trace_token=None,
                                         parallel_operations=False):
   """Runs the command with the given command runner and arguments."""
   # pylint: disable=g-import-not-at-top
   from gslib.util import GetConfigFilePath
   from gslib.util import IS_WINDOWS
   from gslib.util import IsRunningInteractively
   try:
     # Catch ^C so we can print a brief message instead of the normal Python
     # stack trace. Register as a final signal handler because this handler kills
     # the main gsutil process (so it must run last).
     RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True)
     # Catch ^\ so we can force a breakpoint in a running gsutil.
     if not IS_WINDOWS:
       RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit)
     return command_runner.RunNamedCommand(command_name, args, headers,
-                                          debug_level, parallel_operations)
+                                          debug_level, trace_token,
+                                          parallel_operations)
   except AttributeError as e:
     if str(e).find('secret_access_key') != -1:
       _OutputAndExit('Missing credentials for the given URI(s). Does your '
                      'boto config file contain all needed credentials?')
     else:
       _OutputAndExit(str(e))
   except gslib.exception.CommandException as e:
     _HandleCommandException(e)
   except getopt.GetoptError as e:
     _HandleCommandException(gslib.exception.CommandException(e.msg))
@@ skipping 87 matching lines @@
                    ' not found.\nCause: %s' % e)
   parser = argparse.ArgumentParser(add_help=False)
   subparsers = parser.add_subparsers()
   command_runner.ConfigureCommandArgumentParsers(subparsers)
   argcomplete.autocomplete(parser, exit_method=sys.exit)
 
   return 0
 
 if __name__ == '__main__':
   sys.exit(main())