| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- coding: utf-8 -*- | |
| 2 # Copyright 2011 Google Inc. All Rights Reserved. | |
| 3 # | |
| 4 # Licensed under the Apache License, Version 2.0 (the "License"); | |
| 5 # you may not use this file except in compliance with the License. | |
| 6 # You may obtain a copy of the License at | |
| 7 # | |
| 8 # http://www.apache.org/licenses/LICENSE-2.0 | |
| 9 # | |
| 10 # Unless required by applicable law or agreed to in writing, software | |
| 11 # distributed under the License is distributed on an "AS IS" BASIS, | |
| 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 13 # See the License for the specific language governing permissions and | |
| 14 # limitations under the License. | |
| 15 """Class that runs a named gsutil command.""" | |
| 16 | |
| 17 from __future__ import absolute_import | |
| 18 | |
| 19 import difflib | |
| 20 import logging | |
| 21 import os | |
| 22 import pkgutil | |
| 23 import sys | |
| 24 import textwrap | |
| 25 import time | |
| 26 | |
| 27 import boto | |
| 28 from boto.storage_uri import BucketStorageUri | |
| 29 import gslib | |
| 30 from gslib.cloud_api_delegator import CloudApiDelegator | |
| 31 from gslib.command import Command | |
| 32 from gslib.command import CreateGsutilLogger | |
| 33 from gslib.command import GetFailureCount | |
| 34 from gslib.command import OLD_ALIAS_MAP | |
| 35 from gslib.command import ShutDownGsutil | |
| 36 import gslib.commands | |
| 37 from gslib.cs_api_map import ApiSelector | |
| 38 from gslib.cs_api_map import GsutilApiClassMapFactory | |
| 39 from gslib.cs_api_map import GsutilApiMapFactory | |
| 40 from gslib.exception import CommandException | |
| 41 from gslib.gcs_json_api import GcsJsonApi | |
| 42 from gslib.no_op_credentials import NoOpCredentials | |
| 43 from gslib.tab_complete import MakeCompleter | |
| 44 from gslib.util import CompareVersions | |
| 45 from gslib.util import GetGsutilVersionModifiedTime | |
| 46 from gslib.util import GSUTIL_PUB_TARBALL | |
| 47 from gslib.util import IsRunningInteractively | |
| 48 from gslib.util import LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE | |
| 49 from gslib.util import LookUpGsutilVersion | |
| 50 from gslib.util import MultiprocessingIsAvailable | |
| 51 from gslib.util import RELEASE_NOTES_URL | |
| 52 from gslib.util import SECONDS_PER_DAY | |
| 53 from gslib.util import UTF8 | |
| 54 | |
| 55 | |
| 56 def HandleArgCoding(args): | |
| 57 """Handles coding of command-line args. | |
| 58 | |
| 59 Args: | |
| 60 args: array of command-line args. | |
| 61 | |
| 62 Returns: | |
| 63 array of command-line args. | |
| 64 | |
| 65 Raises: | |
| 66 CommandException: if errors encountered. | |
| 67 """ | |
| 68 # Python passes arguments from the command line as byte strings. To | |
| 69 # correctly interpret them, we decode ones other than -h and -p args (which | |
| 70 # will be passed as headers, and thus per HTTP spec should not be encoded) as | |
| 71 # utf-8. The exception is x-goog-meta-* headers, which are allowed to contain | |
| 72 # non-ASCII content (and hence, should be decoded), per | |
| 73 # https://developers.google.com/storage/docs/gsutil/addlhelp/WorkingWithObject
Metadata | |
| 74 processing_header = False | |
| 75 for i in range(len(args)): | |
| 76 arg = args[i] | |
| 77 # Commands like mv can run this function twice; don't decode twice. | |
| 78 try: | |
| 79 decoded = arg if isinstance(arg, unicode) else arg.decode(UTF8) | |
| 80 except UnicodeDecodeError: | |
| 81 raise CommandException('\n'.join(textwrap.wrap( | |
| 82 'Invalid encoding for argument (%s). Arguments must be decodable as ' | |
| 83 'Unicode. NOTE: the argument printed above replaces the problematic ' | |
| 84 'characters with a hex-encoded printable representation. For more ' | |
| 85 'details (including how to convert to a gsutil-compatible encoding) ' | |
| 86 'see `gsutil help encoding`.' % repr(arg)))) | |
| 87 if processing_header: | |
| 88 if arg.lower().startswith('x-goog-meta'): | |
| 89 args[i] = decoded | |
| 90 else: | |
| 91 try: | |
| 92 # Try to encode as ASCII to check for invalid header values (which | |
| 93 # can't be sent over HTTP). | |
| 94 decoded.encode('ascii') | |
| 95 except UnicodeEncodeError: | |
| 96 # Raise the CommandException using the decoded value because | |
| 97 # _OutputAndExit function re-encodes at the end. | |
| 98 raise CommandException( | |
| 99 'Invalid non-ASCII header value (%s).\nOnly ASCII characters are ' | |
| 100 'allowed in headers other than x-goog-meta- headers' % decoded) | |
| 101 else: | |
| 102 args[i] = decoded | |
| 103 processing_header = (arg in ('-h', '-p')) | |
| 104 return args | |
| 105 | |
| 106 | |
| 107 class CommandRunner(object): | |
| 108 """Runs gsutil commands and does some top-level argument handling.""" | |
| 109 | |
| 110 def __init__(self, bucket_storage_uri_class=BucketStorageUri, | |
| 111 gsutil_api_class_map_factory=GsutilApiClassMapFactory, | |
| 112 command_map=None): | |
| 113 """Instantiates a CommandRunner. | |
| 114 | |
| 115 Args: | |
| 116 bucket_storage_uri_class: Class to instantiate for cloud StorageUris. | |
| 117 Settable for testing/mocking. | |
| 118 gsutil_api_class_map_factory: Creates map of cloud storage interfaces. | |
| 119 Settable for testing/mocking. | |
| 120 command_map: Map of command names to their implementations for | |
| 121 testing/mocking. If not set, the map is built dynamically. | |
| 122 """ | |
| 123 self.bucket_storage_uri_class = bucket_storage_uri_class | |
| 124 self.gsutil_api_class_map_factory = gsutil_api_class_map_factory | |
| 125 if command_map: | |
| 126 self.command_map = command_map | |
| 127 else: | |
| 128 self.command_map = self._LoadCommandMap() | |
| 129 | |
| 130 def _LoadCommandMap(self): | |
| 131 """Returns dict mapping each command_name to implementing class.""" | |
| 132 # Import all gslib.commands submodules. | |
| 133 for _, module_name, _ in pkgutil.iter_modules(gslib.commands.__path__): | |
| 134 __import__('gslib.commands.%s' % module_name) | |
| 135 | |
| 136 command_map = {} | |
| 137 # Only include Command subclasses in the dict. | |
| 138 for command in Command.__subclasses__(): | |
| 139 command_map[command.command_spec.command_name] = command | |
| 140 for command_name_aliases in command.command_spec.command_name_aliases: | |
| 141 command_map[command_name_aliases] = command | |
| 142 return command_map | |
| 143 | |
| 144 def _ConfigureCommandArgumentParserArguments( | |
| 145 self, parser, arguments, gsutil_api): | |
| 146 """Configures an argument parser with the given arguments. | |
| 147 | |
| 148 Args: | |
| 149 parser: argparse parser object. | |
| 150 arguments: array of CommandArgument objects. | |
| 151 gsutil_api: gsutil Cloud API instance to use. | |
| 152 Raises: | |
| 153 RuntimeError: if argument is configured with unsupported completer | |
| 154 """ | |
| 155 for command_argument in arguments: | |
| 156 action = parser.add_argument( | |
| 157 *command_argument.args, **command_argument.kwargs) | |
| 158 if command_argument.completer: | |
| 159 action.completer = MakeCompleter(command_argument.completer, gsutil_api) | |
| 160 | |
| 161 def ConfigureCommandArgumentParsers(self, subparsers): | |
| 162 """Configures argparse arguments and argcomplete completers for commands. | |
| 163 | |
| 164 Args: | |
| 165 subparsers: argparse object that can be used to add parsers for | |
| 166 subcommands (called just 'commands' in gsutil) | |
| 167 """ | |
| 168 | |
| 169 # This should match the support map for the "ls" command. | |
| 170 support_map = { | |
| 171 'gs': [ApiSelector.XML, ApiSelector.JSON], | |
| 172 's3': [ApiSelector.XML] | |
| 173 } | |
| 174 default_map = { | |
| 175 'gs': ApiSelector.JSON, | |
| 176 's3': ApiSelector.XML | |
| 177 } | |
| 178 gsutil_api_map = GsutilApiMapFactory.GetApiMap( | |
| 179 self.gsutil_api_class_map_factory, support_map, default_map) | |
| 180 | |
| 181 logger = CreateGsutilLogger('tab_complete') | |
| 182 gsutil_api = CloudApiDelegator( | |
| 183 self.bucket_storage_uri_class, gsutil_api_map, | |
| 184 logger, debug=0) | |
| 185 | |
| 186 for command in set(self.command_map.values()): | |
| 187 command_parser = subparsers.add_parser( | |
| 188 command.command_spec.command_name, add_help=False) | |
| 189 if isinstance(command.command_spec.argparse_arguments, dict): | |
| 190 subcommand_parsers = command_parser.add_subparsers() | |
| 191 subcommand_argument_dict = command.command_spec.argparse_arguments | |
| 192 for subcommand, arguments in subcommand_argument_dict.iteritems(): | |
| 193 subcommand_parser = subcommand_parsers.add_parser( | |
| 194 subcommand, add_help=False) | |
| 195 self._ConfigureCommandArgumentParserArguments( | |
| 196 subcommand_parser, arguments, gsutil_api) | |
| 197 else: | |
| 198 self._ConfigureCommandArgumentParserArguments( | |
| 199 command_parser, command.command_spec.argparse_arguments, gsutil_api) | |
| 200 | |
| 201 def RunNamedCommand(self, command_name, args=None, headers=None, debug=0, | |
| 202 parallel_operations=False, test_method=None, | |
| 203 skip_update_check=False, logging_filters=None, | |
| 204 do_shutdown=True): | |
| 205 """Runs the named command. | |
| 206 | |
| 207 Used by gsutil main, commands built atop other commands, and tests. | |
| 208 | |
| 209 Args: | |
| 210 command_name: The name of the command being run. | |
| 211 args: Command-line args (arg0 = actual arg, not command name ala bash). | |
| 212 headers: Dictionary containing optional HTTP headers to pass to boto. | |
| 213 debug: Debug level to pass in to boto connection (range 0..3). | |
| 214 parallel_operations: Should command operations be executed in parallel? | |
| 215 test_method: Optional general purpose method for testing purposes. | |
| 216 Application and semantics of this method will vary by | |
| 217 command and test type. | |
| 218 skip_update_check: Set to True to disable checking for gsutil updates. | |
| 219 logging_filters: Optional list of logging.Filters to apply to this | |
| 220 command's logger. | |
| 221 do_shutdown: Stop all parallelism framework workers iff this is True. | |
| 222 | |
| 223 Raises: | |
| 224 CommandException: if errors encountered. | |
| 225 | |
| 226 Returns: | |
| 227 Return value(s) from Command that was run. | |
| 228 """ | |
| 229 if (not skip_update_check and | |
| 230 self.MaybeCheckForAndOfferSoftwareUpdate(command_name, debug)): | |
| 231 command_name = 'update' | |
| 232 args = ['-n'] | |
| 233 | |
| 234 if not args: | |
| 235 args = [] | |
| 236 | |
| 237 # Include api_version header in all commands. | |
| 238 api_version = boto.config.get_value('GSUtil', 'default_api_version', '1') | |
| 239 if not headers: | |
| 240 headers = {} | |
| 241 headers['x-goog-api-version'] = api_version | |
| 242 | |
| 243 if command_name not in self.command_map: | |
| 244 close_matches = difflib.get_close_matches( | |
| 245 command_name, self.command_map.keys(), n=1) | |
| 246 if close_matches: | |
| 247 # Instead of suggesting a deprecated command alias, suggest the new | |
| 248 # name for that command. | |
| 249 translated_command_name = ( | |
| 250 OLD_ALIAS_MAP.get(close_matches[0], close_matches)[0]) | |
| 251 print >> sys.stderr, 'Did you mean this?' | |
| 252 print >> sys.stderr, '\t%s' % translated_command_name | |
| 253 elif command_name == 'update' and gslib.IS_PACKAGE_INSTALL: | |
| 254 sys.stderr.write( | |
| 255 'Update command is not supported for package installs; ' | |
| 256 'please instead update using your package manager.') | |
| 257 | |
| 258 raise CommandException('Invalid command "%s".' % command_name) | |
| 259 if '--help' in args: | |
| 260 new_args = [command_name] | |
| 261 original_command_class = self.command_map[command_name] | |
| 262 subcommands = original_command_class.help_spec.subcommand_help_text.keys() | |
| 263 for arg in args: | |
| 264 if arg in subcommands: | |
| 265 new_args.append(arg) | |
| 266 break # Take the first match and throw away the rest. | |
| 267 args = new_args | |
| 268 command_name = 'help' | |
| 269 | |
| 270 args = HandleArgCoding(args) | |
| 271 | |
| 272 command_class = self.command_map[command_name] | |
| 273 command_inst = command_class( | |
| 274 self, args, headers, debug, parallel_operations, | |
| 275 self.bucket_storage_uri_class, self.gsutil_api_class_map_factory, | |
| 276 test_method, logging_filters, command_alias_used=command_name) | |
| 277 return_code = command_inst.RunCommand() | |
| 278 | |
| 279 if MultiprocessingIsAvailable()[0] and do_shutdown: | |
| 280 ShutDownGsutil() | |
| 281 if GetFailureCount() > 0: | |
| 282 return_code = 1 | |
| 283 return return_code | |
| 284 | |
| 285 def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug): | |
| 286 """Checks the last time we checked for an update and offers one if needed. | |
| 287 | |
| 288 Offer is made if the time since the last update check is longer | |
| 289 than the configured threshold offers the user to update gsutil. | |
| 290 | |
| 291 Args: | |
| 292 command_name: The name of the command being run. | |
| 293 debug: Debug level to pass in to boto connection (range 0..3). | |
| 294 | |
| 295 Returns: | |
| 296 True if the user decides to update. | |
| 297 """ | |
| 298 # Don't try to interact with user if: | |
| 299 # - gsutil is not connected to a tty (e.g., if being run from cron); | |
| 300 # - user is running gsutil -q | |
| 301 # - user is running the config command (which could otherwise attempt to | |
| 302 # check for an update for a user running behind a proxy, who has not yet | |
| 303 # configured gsutil to go through the proxy; for such users we need the | |
| 304 # first connection attempt to be made by the gsutil config command). | |
| 305 # - user is running the version command (which gets run when using | |
| 306 # gsutil -D, which would prevent users with proxy config problems from | |
| 307 # sending us gsutil -D output). | |
| 308 # - user is running the update command (which could otherwise cause an | |
| 309 # additional note that an update is available when user is already trying | |
| 310 # to perform an update); | |
| 311 # - user specified gs_host (which could be a non-production different | |
| 312 # service instance, in which case credentials won't work for checking | |
| 313 # gsutil tarball). | |
| 314 # - user is using a Cloud SDK install (which should only be updated via | |
| 315 # gcloud components update) | |
| 316 logger = logging.getLogger() | |
| 317 gs_host = boto.config.get('Credentials', 'gs_host', None) | |
| 318 if (not IsRunningInteractively() | |
| 319 or command_name in ('config', 'update', 'ver', 'version') | |
| 320 or not logger.isEnabledFor(logging.INFO) | |
| 321 or gs_host | |
| 322 or os.environ.get('CLOUDSDK_WRAPPER') == '1'): | |
| 323 return False | |
| 324 | |
| 325 software_update_check_period = boto.config.getint( | |
| 326 'GSUtil', 'software_update_check_period', 30) | |
| 327 # Setting software_update_check_period to 0 means periodic software | |
| 328 # update checking is disabled. | |
| 329 if software_update_check_period == 0: | |
| 330 return False | |
| 331 | |
| 332 cur_ts = int(time.time()) | |
| 333 if not os.path.isfile(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE): | |
| 334 # Set last_checked_ts from date of VERSION file, so if the user installed | |
| 335 # an old copy of gsutil it will get noticed (and an update offered) the | |
| 336 # first time they try to run it. | |
| 337 last_checked_ts = GetGsutilVersionModifiedTime() | |
| 338 with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f: | |
| 339 f.write(str(last_checked_ts)) | |
| 340 else: | |
| 341 try: | |
| 342 with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'r') as f: | |
| 343 last_checked_ts = int(f.readline()) | |
| 344 except (TypeError, ValueError): | |
| 345 return False | |
| 346 | |
| 347 if (cur_ts - last_checked_ts | |
| 348 > software_update_check_period * SECONDS_PER_DAY): | |
| 349 # Create a credential-less gsutil API to check for the public | |
| 350 # update tarball. | |
| 351 gsutil_api = GcsJsonApi(self.bucket_storage_uri_class, logger, | |
| 352 credentials=NoOpCredentials(), debug=debug) | |
| 353 | |
| 354 cur_ver = LookUpGsutilVersion(gsutil_api, GSUTIL_PUB_TARBALL) | |
| 355 with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f: | |
| 356 f.write(str(cur_ts)) | |
| 357 (g, m) = CompareVersions(cur_ver, gslib.VERSION) | |
| 358 if m: | |
| 359 print '\n'.join(textwrap.wrap( | |
| 360 'A newer version of gsutil (%s) is available than the version you ' | |
| 361 'are running (%s). NOTE: This is a major new version, so it is ' | |
| 362 'strongly recommended that you review the release note details at ' | |
| 363 '%s before updating to this version, especially if you use gsutil ' | |
| 364 'in scripts.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))) | |
| 365 if gslib.IS_PACKAGE_INSTALL: | |
| 366 return False | |
| 367 print | |
| 368 answer = raw_input('Would you like to update [y/N]? ') | |
| 369 return answer and answer.lower()[0] == 'y' | |
| 370 elif g: | |
| 371 print '\n'.join(textwrap.wrap( | |
| 372 'A newer version of gsutil (%s) is available than the version you ' | |
| 373 'are running (%s). A detailed log of gsutil release changes is ' | |
| 374 'available at %s if you would like to read them before updating.' | |
| 375 % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))) | |
| 376 if gslib.IS_PACKAGE_INSTALL: | |
| 377 return False | |
| 378 print | |
| 379 answer = raw_input('Would you like to update [Y/n]? ') | |
| 380 return not answer or answer.lower()[0] != 'n' | |
| 381 return False | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1260493004:
Revert "Add gsutil 4.13 to telemetry/third_party" (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master