| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- coding: utf-8 -*- | |
| 2 # Copyright 2011 Google Inc. All Rights Reserved. | |
| 3 # | |
| 4 # Licensed under the Apache License, Version 2.0 (the "License"); | |
| 5 # you may not use this file except in compliance with the License. | |
| 6 # You may obtain a copy of the License at | |
| 7 # | |
| 8 # http://www.apache.org/licenses/LICENSE-2.0 | |
| 9 # | |
| 10 # Unless required by applicable law or agreed to in writing, software | |
| 11 # distributed under the License is distributed on an "AS IS" BASIS, | |
| 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 13 # See the License for the specific language governing permissions and | |
| 14 # limitations under the License. | |
| 15 """Implementation of config command for creating a gsutil configuration file.""" | |
| 16 | |
| 17 from __future__ import absolute_import | |
| 18 | |
| 19 import datetime | |
| 20 from httplib import ResponseNotReady | |
| 21 import json | |
| 22 import multiprocessing | |
| 23 import os | |
| 24 import platform | |
| 25 import signal | |
| 26 import socket | |
| 27 import stat | |
| 28 import sys | |
| 29 import textwrap | |
| 30 import time | |
| 31 import webbrowser | |
| 32 | |
| 33 import boto | |
| 34 from boto.provider import Provider | |
| 35 from httplib2 import ServerNotFoundError | |
| 36 from oauth2client.client import HAS_CRYPTO | |
| 37 | |
| 38 import gslib | |
| 39 from gslib.command import Command | |
| 40 from gslib.commands.compose import MAX_COMPONENT_COUNT | |
| 41 from gslib.cred_types import CredTypes | |
| 42 from gslib.exception import AbortException | |
| 43 from gslib.exception import CommandException | |
| 44 from gslib.hashing_helper import CHECK_HASH_ALWAYS | |
| 45 from gslib.hashing_helper import CHECK_HASH_IF_FAST_ELSE_FAIL | |
| 46 from gslib.hashing_helper import CHECK_HASH_IF_FAST_ELSE_SKIP | |
| 47 from gslib.hashing_helper import CHECK_HASH_NEVER | |
| 48 from gslib.sig_handling import RegisterSignalHandler | |
| 49 from gslib.util import EIGHT_MIB | |
| 50 from gslib.util import IS_WINDOWS | |
| 51 | |
| 52 | |
| 53 _SYNOPSIS = """ | |
| 54 gsutil [-D] config [-a] [-b] [-e] [-f] [-o <file>] [-r] [-s <scope>] [-w] | |
| 55 """ | |
| 56 | |
| 57 _DETAILED_HELP_TEXT = (""" | |
| 58 <B>SYNOPSIS</B> | |
| 59 """ + _SYNOPSIS + """ | |
| 60 | |
| 61 | |
| 62 <B>DESCRIPTION</B> | |
| 63 The gsutil config command obtains access credentials for Google Cloud | |
| 64 Storage and writes a boto/gsutil configuration file containing the obtained | |
| 65 credentials along with a number of other configuration-controllable values. | |
| 66 | |
| 67 Unless specified otherwise (see OPTIONS), the configuration file is written | |
| 68 to ~/.boto (i.e., the file .boto under the user's home directory). If the | |
| 69 default file already exists, an attempt is made to rename the existing file | |
| 70 to ~/.boto.bak; if that attempt fails the command will exit. A different | |
| 71 destination file can be specified with the -o option (see OPTIONS). | |
| 72 | |
| 73 Because the boto configuration file contains your credentials you should | |
| 74 keep its file permissions set so no one but you has read access. (The file | |
| 75 is created read-only when you run gsutil config.) | |
| 76 | |
| 77 | |
| 78 <B>CREDENTIALS</B> | |
| 79 By default gsutil config obtains OAuth2 credentials, and writes them | |
| 80 to the [Credentials] section of the configuration file. The -r, -w, | |
| 81 -f options (see OPTIONS below) cause gsutil config to request a token | |
| 82 with restricted scope; the resulting token will be restricted to read-only | |
| 83 operations, read-write operations, or all operations (including acl get/set, | |
| 84 defacl get/set, and logging get/'set on'/'set off' operations). In | |
| 85 addition, -s <scope> can be used to request additional (non-Google-Storage) | |
| 86 scopes. | |
| 87 | |
| 88 If you want to use credentials based on access key and secret (the older | |
| 89 authentication method before OAuth2 was supported) instead of OAuth2, | |
| 90 see help about the -a option in the OPTIONS section. | |
| 91 | |
| 92 If you wish to use gsutil with other providers (or to copy data back and | |
| 93 forth between multiple providers) you can edit their credentials into the | |
| 94 [Credentials] section after creating the initial configuration file. | |
| 95 | |
| 96 | |
| 97 <B>CONFIGURING SERVICE ACCOUNT CREDENTIALS</B> | |
| 98 You can configure credentials for service accounts using the gsutil config -e | |
| 99 option. Service accounts are useful for authenticating on behalf of a service | |
| 100 or application (as opposed to a user). | |
| 101 | |
| 102 When you run gsutil config -e, you will be prompted for your service account | |
| 103 email address and the path to your private key file. To get these data, visit | |
| 104 the `Google Developers Console <https://cloud.google.com/console#/project>`_, | |
| 105 click on the project you are using, then click "APIs & auth", then click | |
| 106 "Credentials", then click "Create new Client ID"; on the pop-up dialog box | |
| 107 select "Service account" and click "Create Client ID". This will download | |
| 108 a private key file, which you should move to somewhere | |
| 109 accessible from the machine where you run gsutil. Make sure to set its | |
| 110 protection so only the users you want to be able to authenticate have | |
| 111 access. | |
| 112 | |
| 113 Note that your service account will NOT be considered an Owner for the | |
| 114 purposes of API access (see "gsutil help creds" for more information about | |
| 115 this). See https://developers.google.com/accounts/docs/OAuth2ServiceAccount | |
| 116 for further information on service account authentication. | |
| 117 | |
| 118 | |
| 119 <B>CONFIGURATION FILE SELECTION PROCEDURE</B> | |
| 120 By default, gsutil will look for the configuration file in /etc/boto.cfg and | |
| 121 ~/.boto. You can override this choice by setting the BOTO_CONFIG environment | |
| 122 variable. This is also useful if you have several different identities or | |
| 123 cloud storage environments: By setting up the credentials and any additional | |
| 124 configuration in separate files for each, you can switch environments by | |
| 125 changing environment variables. | |
| 126 | |
| 127 You can also set up a path of configuration files, by setting the BOTO_PATH | |
| 128 environment variable to contain a ":" delimited path. For example setting | |
| 129 the BOTO_PATH environment variable to: | |
| 130 | |
| 131 /etc/projects/my_group_project.boto.cfg:/home/mylogin/.boto | |
| 132 | |
| 133 will cause gsutil to load each configuration file found in the path in | |
| 134 order. This is useful if you want to set up some shared configuration | |
| 135 state among many users: The shared state can go in the central shared file | |
| 136 ( /etc/projects/my_group_project.boto.cfg) and each user's individual | |
| 137 credentials can be placed in the configuration file in each of their home | |
| 138 directories. (For security reasons users should never share credentials | |
| 139 via a shared configuration file.) | |
| 140 | |
| 141 | |
| 142 <B>CONFIGURATION FILE STRUCTURE</B> | |
| 143 The configuration file contains a number of sections: [Credentials], | |
| 144 [Boto], [GSUtil], and [OAuth2]. If you edit the file make sure to edit the | |
| 145 appropriate section (discussed below), and to be careful not to mis-edit | |
| 146 any of the setting names (like "gs_access_key_id") and not to remove the | |
| 147 section delimiters (like "[Credentials]"). | |
| 148 | |
| 149 | |
| 150 <B>ADDITIONAL CONFIGURATION-CONTROLLABLE FEATURES</B> | |
| 151 With the exception of setting up gsutil to work through a proxy (see | |
| 152 below), most users won't need to edit values in the boto configuration file; | |
| 153 values found in there tend to be of more specialized use than command line | |
| 154 option-controllable features. | |
| 155 | |
| 156 The following are the currently defined configuration settings, broken | |
| 157 down by section. Their use is documented in comments preceding each, in | |
| 158 the configuration file. If you see a setting you want to change that's not | |
| 159 listed in your current file, see the section below on Updating to the Latest | |
| 160 Configuration File. | |
| 161 | |
| 162 The currently supported settings, are, by section: | |
| 163 | |
| 164 [Credentials] | |
| 165 aws_access_key_id | |
| 166 aws_secret_access_key | |
| 167 gs_access_key_id | |
| 168 gs_host | |
| 169 gs_json_host | |
| 170 gs_json_port | |
| 171 gs_oauth2_refresh_token | |
| 172 gs_port | |
| 173 gs_secret_access_key | |
| 174 s3_host | |
| 175 s3_port | |
| 176 | |
| 177 [Boto] | |
| 178 proxy | |
| 179 proxy_port | |
| 180 proxy_user | |
| 181 proxy_pass | |
| 182 proxy_rdns | |
| 183 http_socket_timeout | |
| 184 https_validate_certificates | |
| 185 debug | |
| 186 max_retry_delay | |
| 187 num_retries | |
| 188 | |
| 189 [GSUtil] | |
| 190 check_hashes | |
| 191 content_language | |
| 192 default_api_version | |
| 193 default_project_id | |
| 194 json_api_version | |
| 195 parallel_composite_upload_component_size | |
| 196 parallel_composite_upload_threshold | |
| 197 parallel_process_count | |
| 198 parallel_thread_count | |
| 199 prefer_api | |
| 200 resumable_threshold | |
| 201 resumable_tracker_dir (deprecated in 4.6, use state_dir) | |
| 202 rsync_buffer_lines | |
| 203 software_update_check_period | |
| 204 state_dir | |
| 205 tab_completion_time_logs | |
| 206 tab_completion_timeout | |
| 207 use_magicfile | |
| 208 | |
| 209 [OAuth2] | |
| 210 client_id | |
| 211 client_secret | |
| 212 oauth2_refresh_retries | |
| 213 provider_authorization_uri | |
| 214 provider_label | |
| 215 provider_token_uri | |
| 216 token_cache | |
| 217 | |
| 218 | |
| 219 <B>UPDATING TO THE LATEST CONFIGURATION FILE</B> | |
| 220 We add new configuration controllable features to the boto configuration file | |
| 221 over time, but most gsutil users create a configuration file once and then | |
| 222 keep it for a long time, so new features aren't apparent when you update | |
| 223 to a newer version of gsutil. If you want to get the latest configuration | |
| 224 file (which includes all the latest settings and documentation about each) | |
| 225 you can rename your current file (e.g., to '.boto_old'), run gsutil config, | |
| 226 and then edit any configuration settings you wanted from your old file | |
| 227 into the newly created file. Note, however, that if you're using OAuth2 | |
| 228 credentials and you go back through the OAuth2 configuration dialog it will | |
| 229 invalidate your previous OAuth2 credentials. | |
| 230 | |
| 231 If no explicit scope option is given, -f (full control) is assumed by default. | |
| 232 | |
| 233 | |
| 234 <B>OPTIONS</B> | |
| 235 -a Prompt for Google Cloud Storage access key and secret (the older | |
| 236 authentication method before OAuth2 was supported) instead of | |
| 237 obtaining an OAuth2 token. | |
| 238 | |
| 239 -b Causes gsutil config to launch a browser to obtain OAuth2 approval | |
| 240 and the project ID instead of showing the URL for each and asking | |
| 241 the user to open the browser. This will probably not work as | |
| 242 expected if you are running gsutil from an ssh window, or using | |
| 243 gsutil on Windows. | |
| 244 | |
| 245 -e Prompt for service account credentials. This option requires that | |
| 246 -a is not set. | |
| 247 | |
| 248 -f Request token with full-control access (default). | |
| 249 | |
| 250 -o <file> Write the configuration to <file> instead of ~/.boto. | |
| 251 Use '-' for stdout. | |
| 252 | |
| 253 -r Request token restricted to read-only access. | |
| 254 | |
| 255 -s <scope> Request additional OAuth2 <scope>. | |
| 256 | |
| 257 -w Request token restricted to read-write access. | |
| 258 """) | |
| 259 | |
| 260 | |
| 261 try: | |
| 262 from gcs_oauth2_boto_plugin import oauth2_helper # pylint: disable=g-import-n
ot-at-top | |
| 263 except ImportError: | |
| 264 pass | |
| 265 | |
| 266 GOOG_CLOUD_CONSOLE_URI = 'https://cloud.google.com/console#/project' | |
| 267 | |
| 268 SCOPE_FULL_CONTROL = 'https://www.googleapis.com/auth/devstorage.full_control' | |
| 269 SCOPE_READ_WRITE = 'https://www.googleapis.com/auth/devstorage.read_write' | |
| 270 SCOPE_READ_ONLY = 'https://www.googleapis.com/auth/devstorage.read_only' | |
| 271 | |
| 272 CONFIG_PRELUDE_CONTENT = """ | |
| 273 # This file contains credentials and other configuration information needed | |
| 274 # by the boto library, used by gsutil. You can edit this file (e.g., to add | |
| 275 # credentials) but be careful not to mis-edit any of the variable names (like | |
| 276 # "gs_access_key_id") or remove important markers (like the "[Credentials]" and | |
| 277 # "[Boto]" section delimiters). | |
| 278 # | |
| 279 """ | |
| 280 | |
| 281 # Default number of OS processes and Python threads for parallel operations. | |
| 282 # On Linux systems we automatically scale the number of processes to match | |
| 283 # the underlying CPU/core count. Given we'll be running multiple concurrent | |
| 284 # processes on a typical multi-core Linux computer, to avoid being too | |
| 285 # aggressive with resources, the default number of threads is reduced from | |
| 286 # the previous value of 24 to 10. | |
| 287 # On Windows and Mac systems parallel multi-processing and multi-threading | |
| 288 # in Python presents various challenges so we retain compatibility with | |
| 289 # the established parallel mode operation, i.e. one process and 24 threads. | |
| 290 if platform.system() == 'Linux': | |
| 291 DEFAULT_PARALLEL_PROCESS_COUNT = multiprocessing.cpu_count() | |
| 292 DEFAULT_PARALLEL_THREAD_COUNT = 10 | |
| 293 else: | |
| 294 DEFAULT_PARALLEL_PROCESS_COUNT = 1 | |
| 295 DEFAULT_PARALLEL_THREAD_COUNT = 24 | |
| 296 | |
| 297 # TODO: Once compiled crcmod is being distributed by major Linux distributions | |
| 298 # revert DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD value to '150M'. | |
| 299 DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD = '0' | |
| 300 DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE = '50M' | |
| 301 | |
| 302 CONFIG_BOTO_SECTION_CONTENT = """ | |
| 303 [Boto] | |
| 304 | |
| 305 # http_socket_timeout specifies the timeout (in seconds) used to tell httplib | |
| 306 # how long to wait for socket timeouts. The default is 70 seconds. Note that | |
| 307 # this timeout only applies to httplib, not to httplib2 (which is used for | |
| 308 # OAuth2 refresh/access token exchanges). | |
| 309 #http_socket_timeout = 70 | |
| 310 | |
| 311 # The following two options control the use of a secure transport for requests | |
| 312 # to S3 and Google Cloud Storage. It is highly recommended to set both options | |
| 313 # to True in production environments, especially when using OAuth2 bearer token | |
| 314 # authentication with Google Cloud Storage. | |
| 315 | |
| 316 # Set 'https_validate_certificates' to False to disable server certificate | |
| 317 # checking. The default for this option in the boto library is currently | |
| 318 # 'False' (to avoid breaking apps that depend on invalid certificates); it is | |
| 319 # therefore strongly recommended to always set this option explicitly to True | |
| 320 # in configuration files, to protect against "man-in-the-middle" attacks. | |
| 321 https_validate_certificates = True | |
| 322 | |
| 323 # 'debug' controls the level of debug messages printed: 0 for none, 1 | |
| 324 # for basic boto debug, 2 for all boto debug plus HTTP requests/responses. | |
| 325 # Note: 'gsutil -d' sets debug to 2 for that one command run. | |
| 326 #debug = <0, 1, or 2> | |
| 327 | |
| 328 # 'num_retries' controls the number of retry attempts made when errors occur | |
| 329 # during data transfers. The default is 6. | |
| 330 # Note 1: You can cause gsutil to retry failures effectively infinitely by | |
| 331 # setting this value to a large number (like 10000). Doing that could be useful | |
| 332 # in cases where your network connection occasionally fails and is down for an | |
| 333 # extended period of time, because when it comes back up gsutil will continue | |
| 334 # retrying. However, in general we recommend not setting the value above 10, | |
| 335 # because otherwise gsutil could appear to "hang" due to excessive retries | |
| 336 # (since unless you run gsutil -D you won't see any logged evidence that gsutil | |
| 337 # is retrying). | |
| 338 # Note 2: Don't set this value to 0, as it will cause boto to fail when reusing | |
| 339 # HTTP connections. | |
| 340 #num_retries = <integer value> | |
| 341 | |
| 342 # 'max_retry_delay' controls the max delay (in seconds) between retries. The | |
| 343 # default value is 60, so the backoff sequence will be 1 seconds, 2 seconds, 4, | |
| 344 # 8, 16, 32, and then 60 for all subsequent retries for a given HTTP request. | |
| 345 # Note: At present this value only impacts the XML API and the JSON API uses a | |
| 346 # fixed value of 60. | |
| 347 #max_retry_delay = <integer value> | |
| 348 """ | |
| 349 | |
| 350 CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT = """ | |
| 351 [GSUtil] | |
| 352 | |
| 353 # 'resumable_threshold' specifies the smallest file size [bytes] for which | |
| 354 # resumable Google Cloud Storage uploads are attempted. The default is 8388608 | |
| 355 # (8 MiB). | |
| 356 #resumable_threshold = %(resumable_threshold)d | |
| 357 | |
| 358 # 'rsync_buffer_lines' specifies the number of lines of bucket or directory | |
| 359 # listings saved in each temp file during sorting. (The complete set is | |
| 360 # split across temp files and separately sorted/merged, to avoid needing to | |
| 361 # fit everything in memory at once.) If you are trying to synchronize very | |
| 362 # large directories/buckets (e.g., containing millions or more objects), | |
| 363 # having too small a value here can cause gsutil to run out of open file | |
| 364 # handles. If that happens, you can try to increase the number of open file | |
| 365 # handles your system allows (e.g., see 'man ulimit' on Linux; see also | |
| 366 # http://docs.python.org/2/library/resource.html). If you can't do that (or | |
| 367 # if you're already at the upper limit), increasing rsync_buffer_lines will | |
| 368 # cause gsutil to use fewer file handles, but at the cost of more memory. With | |
| 369 # rsync_buffer_lines set to 32000 and assuming a typical URL is 100 bytes | |
| 370 # long, gsutil will require approximately 10 MiB of memory while building | |
| 371 # the synchronization state, and will require approximately 60 open file | |
| 372 # descriptors to build the synchronization state over all 1M source and 1M | |
| 373 # destination URLs. Memory and file descriptors are only consumed while | |
| 374 # building the state; once the state is built, it resides in two temp files that | |
| 375 # are read and processed incrementally during the actual copy/delete | |
| 376 # operations. | |
| 377 #rsync_buffer_lines = 32000 | |
| 378 | |
| 379 # 'state_dir' specifies the base location where files that | |
| 380 # need a static location are stored, such as pointers to credentials, | |
| 381 # resumable transfer tracker files, and the last software update check. | |
| 382 # By default these files are stored in ~/.gsutil | |
| 383 #state_dir = <file_path> | |
| 384 # gsutil periodically checks whether a new version of the gsutil software is | |
| 385 # available. 'software_update_check_period' specifies the number of days | |
| 386 # between such checks. The default is 30. Setting the value to 0 disables | |
| 387 # periodic software update checks. | |
| 388 #software_update_check_period = 30 | |
| 389 | |
| 390 # 'tab_completion_timeout' controls the timeout (in seconds) for tab | |
| 391 # completions that involve remote requests (such as bucket or object names). | |
| 392 # If tab completion does not succeed within this timeout, no tab completion | |
| 393 # suggestions will be returned. | |
| 394 # A value of 0 will disable completions that involve remote requests. | |
| 395 #tab_completion_timeout = 5 | |
| 396 | |
| 397 # 'parallel_process_count' and 'parallel_thread_count' specify the number | |
| 398 # of OS processes and Python threads, respectively, to use when executing | |
| 399 # operations in parallel. The default settings should work well as configured, | |
| 400 # however, to enhance performance for transfers involving large numbers of | |
| 401 # files, you may experiment with hand tuning these values to optimize | |
| 402 # performance for your particular system configuration. | |
| 403 # MacOS and Windows users should see | |
| 404 # https://github.com/GoogleCloudPlatform/gsutil/issues/77 before attempting | |
| 405 # to experiment with these values. | |
| 406 #parallel_process_count = %(parallel_process_count)d | |
| 407 #parallel_thread_count = %(parallel_thread_count)d | |
| 408 | |
| 409 # 'parallel_composite_upload_threshold' specifies the maximum size of a file to | |
| 410 # upload in a single stream. Files larger than this threshold will be | |
| 411 # partitioned into component parts and uploaded in parallel and then composed | |
| 412 # into a single object. | |
| 413 # The number of components will be the smaller of | |
| 414 # ceil(file_size / parallel_composite_upload_component_size) and | |
| 415 # MAX_COMPONENT_COUNT. The current value of MAX_COMPONENT_COUNT is | |
| 416 # %(max_component_count)d. | |
| 417 # If 'parallel_composite_upload_threshold' is set to 0, then automatic parallel | |
| 418 # uploads will never occur. | |
| 419 # Setting an extremely low threshold is unadvisable. The vast majority of | |
| 420 # environments will see degraded performance for thresholds below 80M, and it | |
| 421 # is almost never advantageous to have a threshold below 20M. | |
| 422 # 'parallel_composite_upload_component_size' specifies the ideal size of a | |
| 423 # component in bytes, which will act as an upper bound to the size of the | |
| 424 # components if ceil(file_size / parallel_composite_upload_component_size) is | |
| 425 # less than MAX_COMPONENT_COUNT. | |
| 426 # Values can be provided either in bytes or as human-readable values | |
| 427 # (e.g., "150M" to represent 150 mebibytes) | |
| 428 # | |
| 429 # Note: At present parallel composite uploads are disabled by default, because | |
| 430 # using composite objects requires a compiled crcmod (see "gsutil help crcmod"), | |
| 431 # and for operating systems that don't already have this package installed this | |
| 432 # makes gsutil harder to use. Google is actively working with a number of the | |
| 433 # Linux distributions to get crcmod included with the stock distribution. Once | |
| 434 # that is done we will re-enable parallel composite uploads by default in | |
| 435 # gsutil. | |
| 436 #parallel_composite_upload_threshold = %(parallel_composite_upload_threshold)s | |
| 437 #parallel_composite_upload_component_size = %(parallel_composite_upload_componen
t_size)s | |
| 438 | |
| 439 # 'use_magicfile' specifies if the 'file --mime-type <filename>' command should | |
| 440 # be used to guess content types instead of the default filename extension-based | |
| 441 # mechanism. Available on UNIX and MacOS (and possibly on Windows, if you're | |
| 442 # running Cygwin or some other package that provides implementations of | |
| 443 # UNIX-like commands). When available and enabled use_magicfile should be more | |
| 444 # robust because it analyzes file contents in addition to extensions. | |
| 445 #use_magicfile = False | |
| 446 | |
| 447 # 'content_language' specifies the ISO 639-1 language code of the content, to be | |
| 448 # passed in the Content-Language header. By default no Content-Language is sent. | |
| 449 # See the ISO 639-1 column of | |
| 450 # http://www.loc.gov/standards/iso639-2/php/code_list.php for a list of | |
| 451 # language codes. | |
| 452 content_language = en | |
| 453 | |
| 454 # 'check_hashes' specifies how strictly to require integrity checking for | |
| 455 # downloaded data. Legal values are: | |
| 456 # '%(hash_fast_else_fail)s' - (default) Only integrity check if the digest | |
| 457 # will run efficiently (using compiled code), else fail the download. | |
| 458 # '%(hash_fast_else_skip)s' - Only integrity check if the server supplies a | |
| 459 # hash and the local digest computation will run quickly, else skip the | |
| 460 # check. | |
| 461 # '%(hash_always)s' - Always check download integrity regardless of possible | |
| 462 # performance costs. | |
| 463 # '%(hash_never)s' - Don't perform download integrity checks. This setting is | |
| 464 # not recommended except for special cases such as measuring download | |
| 465 # performance excluding time for integrity checking. | |
| 466 # This option exists to assist users who wish to download a GCS composite object | |
| 467 # and are unable to install crcmod with the C-extension. CRC32c is the only | |
| 468 # available integrity check for composite objects, and without the C-extension, | |
| 469 # download performance can be significantly degraded by the digest computation. | |
| 470 # This option is ignored for daisy-chain copies, which don't compute hashes but | |
| 471 # instead (inexpensively) compare the cloud source and destination hashes. | |
| 472 #check_hashes = if_fast_else_fail | |
| 473 | |
| 474 # The ability to specify an alternative JSON API version is primarily for cloud | |
| 475 # storage service developers. | |
| 476 #json_api_version = v1 | |
| 477 | |
| 478 # Specifies the API to use when interacting with cloud storage providers. If | |
| 479 # the gsutil command supports this API for the provider, it will be used | |
| 480 # instead of the default. | |
| 481 # Commands typically default to XML for S3 and JSON for GCS. | |
| 482 #prefer_api = json | |
| 483 #prefer_api = xml | |
| 484 | |
| 485 """ % {'hash_fast_else_fail': CHECK_HASH_IF_FAST_ELSE_FAIL, | |
| 486 'hash_fast_else_skip': CHECK_HASH_IF_FAST_ELSE_SKIP, | |
| 487 'hash_always': CHECK_HASH_ALWAYS, | |
| 488 'hash_never': CHECK_HASH_NEVER, | |
| 489 'resumable_threshold': EIGHT_MIB, | |
| 490 'parallel_process_count': DEFAULT_PARALLEL_PROCESS_COUNT, | |
| 491 'parallel_thread_count': DEFAULT_PARALLEL_THREAD_COUNT, | |
| 492 'parallel_composite_upload_threshold': ( | |
| 493 DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD), | |
| 494 'parallel_composite_upload_component_size': ( | |
| 495 DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE), | |
| 496 'max_component_count': MAX_COMPONENT_COUNT} | |
| 497 | |
| 498 CONFIG_OAUTH2_CONFIG_CONTENT = """ | |
| 499 [OAuth2] | |
| 500 # This section specifies options used with OAuth2 authentication. | |
| 501 | |
| 502 # 'token_cache' specifies how the OAuth2 client should cache access tokens. | |
| 503 # Valid values are: | |
| 504 # 'in_memory': an in-memory cache is used. This is only useful if the boto | |
| 505 # client instance (and with it the OAuth2 plugin instance) persists | |
| 506 # across multiple requests. | |
| 507 # 'file_system' : access tokens will be cached in the file system, in files | |
| 508 # whose names include a key derived from the refresh token the access token | |
| 509 # based on. | |
| 510 # The default is 'file_system'. | |
| 511 #token_cache = file_system | |
| 512 #token_cache = in_memory | |
| 513 | |
| 514 # 'token_cache_path_pattern' specifies a path pattern for token cache files. | |
| 515 # This option is only relevant if token_cache = file_system. | |
| 516 # The value of this option should be a path, with place-holders '%(key)s' (which | |
| 517 # will be replaced with a key derived from the refresh token the cached access | |
| 518 # token was based on), and (optionally), %(uid)s (which will be replaced with | |
| 519 # the UID of the current user, if available via os.getuid()). | |
| 520 # Note that the config parser itself interpolates '%' placeholders, and hence | |
| 521 # the above placeholders need to be escaped as '%%(key)s'. | |
| 522 # The default value of this option is | |
| 523 # token_cache_path_pattern = <tmpdir>/oauth2client-tokencache.%%(uid)s.%%(key)s | |
| 524 # where <tmpdir> is the system-dependent default temp directory. | |
| 525 | |
| 526 # The following options specify the OAuth2 client identity and secret that is | |
| 527 # used when requesting and using OAuth2 tokens. If not specified, a default | |
| 528 # OAuth2 client for the gsutil tool is used; for uses of the boto library (with | |
| 529 # OAuth2 authentication plugin) in other client software, it is recommended to | |
| 530 # use a tool/client-specific OAuth2 client. For more information on OAuth2, see | |
| 531 # http://code.google.com/apis/accounts/docs/OAuth2.html | |
| 532 #client_id = <OAuth2 client id> | |
| 533 #client_secret = <OAuth2 client secret> | |
| 534 | |
| 535 # The following options specify the label and endpoint URIs for the OAUth2 | |
| 536 # authorization provider being used. Primarily useful for tool developers. | |
| 537 #provider_label = Google | |
| 538 #provider_authorization_uri = https://accounts.google.com/o/oauth2/auth | |
| 539 #provider_token_uri = https://accounts.google.com/o/oauth2/token | |
| 540 | |
| 541 # 'oauth2_refresh_retries' controls the number of retry attempts made when | |
| 542 # rate limiting errors occur for OAuth2 requests to retrieve an access token. | |
| 543 # The default value is 6. | |
| 544 #oauth2_refresh_retries = <integer value> | |
| 545 """ | |
| 546 | |
| 547 | |
| 548 class ConfigCommand(Command): | |
| 549 """Implementation of gsutil config command.""" | |
| 550 | |
| 551 # Command specification. See base class for documentation. | |
| 552 command_spec = Command.CreateCommandSpec( | |
| 553 'config', | |
| 554 command_name_aliases=['cfg', 'conf', 'configure'], | |
| 555 usage_synopsis=_SYNOPSIS, | |
| 556 min_args=0, | |
| 557 max_args=0, | |
| 558 supported_sub_args='habefwrs:o:', | |
| 559 file_url_ok=False, | |
| 560 provider_url_ok=False, | |
| 561 urls_start_arg=0, | |
| 562 ) | |
| 563 # Help specification. See help_provider.py for documentation. | |
| 564 help_spec = Command.HelpSpec( | |
| 565 help_name='config', | |
| 566 help_name_aliases=['cfg', 'conf', 'configure', 'aws', 's3'], | |
| 567 help_type='command_help', | |
| 568 help_one_line_summary=( | |
| 569 'Obtain credentials and create configuration file'), | |
| 570 help_text=_DETAILED_HELP_TEXT, | |
| 571 subcommand_help_text={}, | |
| 572 ) | |
| 573 | |
| 574 def _OpenConfigFile(self, file_path): | |
| 575 """Creates and opens a configuration file for writing. | |
| 576 | |
| 577 The file is created with mode 0600, and attempts to open existing files will | |
| 578 fail (the latter is important to prevent symlink attacks). | |
| 579 | |
| 580 It is the caller's responsibility to close the file. | |
| 581 | |
| 582 Args: | |
| 583 file_path: Path of the file to be created. | |
| 584 | |
| 585 Returns: | |
| 586 A writable file object for the opened file. | |
| 587 | |
| 588 Raises: | |
| 589 CommandException: if an error occurred when opening the file (including | |
| 590 when the file already exists). | |
| 591 """ | |
| 592 flags = os.O_RDWR | os.O_CREAT | os.O_EXCL | |
| 593 # Accommodate Windows; copied from python2.6/tempfile.py. | |
| 594 if hasattr(os, 'O_NOINHERIT'): | |
| 595 flags |= os.O_NOINHERIT | |
| 596 try: | |
| 597 fd = os.open(file_path, flags, 0600) | |
| 598 except (OSError, IOError), e: | |
| 599 raise CommandException('Failed to open %s for writing: %s' % | |
| 600 (file_path, e)) | |
| 601 return os.fdopen(fd, 'w') | |
| 602 | |
| 603 def _CheckPrivateKeyFilePermissions(self, file_path): | |
| 604 """Checks that the file has reasonable permissions for a private key. | |
| 605 | |
| 606 In particular, check that the filename provided by the user is not | |
| 607 world- or group-readable. If either of these are true, we issue a warning | |
| 608 and offer to fix the permissions. | |
| 609 | |
| 610 Args: | |
| 611 file_path: The name of the private key file. | |
| 612 """ | |
| 613 if IS_WINDOWS: | |
| 614 # For Windows, this check doesn't work (it actually just checks whether | |
| 615 # the file is read-only). Since Windows files have a complicated ACL | |
| 616 # system, this check doesn't make much sense on Windows anyway, so we | |
| 617 # just don't do it. | |
| 618 return | |
| 619 | |
| 620 st = os.stat(file_path) | |
| 621 if bool((stat.S_IRGRP | stat.S_IROTH) & st.st_mode): | |
| 622 self.logger.warn( | |
| 623 '\nYour private key file is readable by people other than yourself.\n' | |
| 624 'This is a security risk, since anyone with this information can use ' | |
| 625 'your service account.\n') | |
| 626 fix_it = raw_input('Would you like gsutil to change the file ' | |
| 627 'permissions for you? (y/N) ') | |
| 628 if fix_it in ('y', 'Y'): | |
| 629 try: | |
| 630 os.chmod(file_path, 0400) | |
| 631 self.logger.info( | |
| 632 '\nThe permissions on your file have been successfully ' | |
| 633 'modified.' | |
| 634 '\nThe only access allowed is readability by the user ' | |
| 635 '(permissions 0400 in chmod).') | |
| 636 except Exception, _: # pylint: disable=broad-except | |
| 637 self.logger.warn( | |
| 638 '\nWe were unable to modify the permissions on your file.\n' | |
| 639 'If you would like to fix this yourself, consider running:\n' | |
| 640 '"sudo chmod 400 </path/to/key>" for improved security.') | |
| 641 else: | |
| 642 self.logger.info( | |
| 643 '\nYou have chosen to allow this file to be readable by others.\n' | |
| 644 'If you would like to fix this yourself, consider running:\n' | |
| 645 '"sudo chmod 400 </path/to/key>" for improved security.') | |
| 646 | |
| 647 def _PromptForProxyConfigVarAndMaybeSaveToBotoConfig(self, varname, prompt, | |
| 648 convert_to_bool=False): | |
| 649 """Prompts for one proxy config line, saves to boto.config if not empty. | |
| 650 | |
| 651 Args: | |
| 652 varname: The config variable name. | |
| 653 prompt: The prompt to output to the user. | |
| 654 convert_to_bool: Whether to convert "y/n" to True/False. | |
| 655 """ | |
| 656 value = raw_input(prompt) | |
| 657 if value: | |
| 658 if convert_to_bool: | |
| 659 if value == 'y' or value == 'Y': | |
| 660 value = 'True' | |
| 661 else: | |
| 662 value = 'False' | |
| 663 boto.config.set('Boto', varname, value) | |
| 664 | |
| 665 def _PromptForProxyConfig(self): | |
| 666 """Prompts for proxy config data, loads non-empty values into boto.config. | |
| 667 """ | |
| 668 self._PromptForProxyConfigVarAndMaybeSaveToBotoConfig( | |
| 669 'proxy', 'What is your proxy host? ') | |
| 670 self._PromptForProxyConfigVarAndMaybeSaveToBotoConfig( | |
| 671 'proxy_port', 'What is your proxy port? ') | |
| 672 self._PromptForProxyConfigVarAndMaybeSaveToBotoConfig( | |
| 673 'proxy_user', 'What is your proxy user (leave blank if not used)? ') | |
| 674 self._PromptForProxyConfigVarAndMaybeSaveToBotoConfig( | |
| 675 'proxy_pass', 'What is your proxy pass (leave blank if not used)? ') | |
| 676 self._PromptForProxyConfigVarAndMaybeSaveToBotoConfig( | |
| 677 'proxy_rdns', | |
| 678 'Should DNS lookups be resolved by your proxy? (Y if your site ' | |
| 679 'disallows client DNS lookups)? ', | |
| 680 convert_to_bool=True) | |
| 681 | |
| 682 def _WriteConfigLineMaybeCommented(self, config_file, name, value, desc): | |
| 683 """Writes proxy name/value pair or comment line to config file. | |
| 684 | |
| 685 Writes proxy name/value pair if value is not None. Otherwise writes | |
| 686 comment line. | |
| 687 | |
| 688 Args: | |
| 689 config_file: File object to which the resulting config file will be | |
| 690 written. | |
| 691 name: The config variable name. | |
| 692 value: The value, or None. | |
| 693 desc: Human readable description (for comment). | |
| 694 """ | |
| 695 if not value: | |
| 696 name = '#%s' % name | |
| 697 value = '<%s>' % desc | |
| 698 config_file.write('%s = %s\n' % (name, value)) | |
| 699 | |
| 700 def _WriteProxyConfigFileSection(self, config_file): | |
| 701 """Writes proxy section of configuration file. | |
| 702 | |
| 703 Args: | |
| 704 config_file: File object to which the resulting config file will be | |
| 705 written. | |
| 706 """ | |
| 707 config = boto.config | |
| 708 config_file.write( | |
| 709 '# To use a proxy, edit and uncomment the proxy and proxy_port lines.\n' | |
| 710 '# If you need a user/password with this proxy, edit and uncomment\n' | |
| 711 '# those lines as well. If your organization also disallows DNS\n' | |
| 712 '# lookups by client machines set proxy_rdns = True\n' | |
| 713 '# If proxy_host and proxy_port are not specified in this file and\n' | |
| 714 '# one of the OS environment variables http_proxy, https_proxy, or\n' | |
| 715 '# HTTPS_PROXY is defined, gsutil will use the proxy server specified\n' | |
| 716 '# in these environment variables, in order of precedence according\n' | |
| 717 '# to how they are listed above.\n') | |
| 718 self._WriteConfigLineMaybeCommented( | |
| 719 config_file, 'proxy', config.get_value('Boto', 'proxy', None), | |
| 720 'proxy host') | |
| 721 self._WriteConfigLineMaybeCommented( | |
| 722 config_file, 'proxy_port', config.get_value('Boto', 'proxy_port', None), | |
| 723 'proxy port') | |
| 724 self._WriteConfigLineMaybeCommented( | |
| 725 config_file, 'proxy_user', config.get_value('Boto', 'proxy_user', None), | |
| 726 'proxy user') | |
| 727 self._WriteConfigLineMaybeCommented( | |
| 728 config_file, 'proxy_pass', config.get_value('Boto', 'proxy_pass', None), | |
| 729 'proxy password') | |
| 730 self._WriteConfigLineMaybeCommented( | |
| 731 config_file, 'proxy_rdns', | |
| 732 config.get_value('Boto', 'proxy_rdns', False), | |
| 733 'let proxy server perform DNS lookups') | |
| 734 | |
| 735 # pylint: disable=dangerous-default-value,too-many-statements | |
| 736 def _WriteBotoConfigFile(self, config_file, launch_browser=True, | |
| 737 oauth2_scopes=[SCOPE_FULL_CONTROL], | |
| 738 cred_type=CredTypes.OAUTH2_USER_ACCOUNT): | |
| 739 """Creates a boto config file interactively. | |
| 740 | |
| 741 Needed credentials are obtained interactively, either by asking the user for | |
| 742 access key and secret, or by walking the user through the OAuth2 approval | |
| 743 flow. | |
| 744 | |
| 745 Args: | |
| 746 config_file: File object to which the resulting config file will be | |
| 747 written. | |
| 748 launch_browser: In the OAuth2 approval flow, attempt to open a browser | |
| 749 window and navigate to the approval URL. | |
| 750 oauth2_scopes: A list of OAuth2 scopes to request authorization for, when | |
| 751 using OAuth2. | |
| 752 cred_type: There are three options: | |
| 753 - for HMAC, ask the user for access key and secret | |
| 754 - for OAUTH2_USER_ACCOUNT, walk the user through OAuth2 approval flow | |
| 755 and produce a config with an oauth2_refresh_token credential. | |
| 756 - for OAUTH2_SERVICE_ACCOUNT, prompt the user for OAuth2 for service | |
| 757 account email address and private key file (and if the file is a .p12 | |
| 758 file, the password for that file). | |
| 759 """ | |
| 760 # Collect credentials | |
| 761 provider_map = {'aws': 'aws', 'google': 'gs'} | |
| 762 uri_map = {'aws': 's3', 'google': 'gs'} | |
| 763 key_ids = {} | |
| 764 sec_keys = {} | |
| 765 service_account_key_is_json = False | |
| 766 if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: | |
| 767 gs_service_key_file = raw_input('What is the full path to your private ' | |
| 768 'key file? ') | |
| 769 # JSON files have the email address built-in and don't require a password. | |
| 770 try: | |
| 771 with open(gs_service_key_file, 'rb') as key_file_fp: | |
| 772 json.loads(key_file_fp.read()) | |
| 773 service_account_key_is_json = True | |
| 774 except ValueError: | |
| 775 if not HAS_CRYPTO: | |
| 776 raise CommandException( | |
| 777 'Service account authentication via a .p12 file requires ' | |
| 778 'either\nPyOpenSSL or PyCrypto 2.6 or later. Please install ' | |
| 779 'either of these\nto proceed, use a JSON-format key file, or ' | |
| 780 'configure a different type of credentials.') | |
| 781 | |
| 782 if not service_account_key_is_json: | |
| 783 gs_service_client_id = raw_input('What is your service account email ' | |
| 784 'address? ') | |
| 785 gs_service_key_file_password = raw_input( | |
| 786 '\n'.join(textwrap.wrap( | |
| 787 'What is the password for your service key file [if you ' | |
| 788 'haven\'t set one explicitly, leave this line blank]?')) + ' ') | |
| 789 self._CheckPrivateKeyFilePermissions(gs_service_key_file) | |
| 790 elif cred_type == CredTypes.OAUTH2_USER_ACCOUNT: | |
| 791 oauth2_client = oauth2_helper.OAuth2ClientFromBotoConfig(boto.config, | |
| 792 cred_type) | |
| 793 try: | |
| 794 oauth2_refresh_token = oauth2_helper.OAuth2ApprovalFlow( | |
| 795 oauth2_client, oauth2_scopes, launch_browser) | |
| 796 except (ResponseNotReady, ServerNotFoundError, socket.error): | |
| 797 # TODO: Determine condition to check for in the ResponseNotReady | |
| 798 # exception so we only run proxy config flow if failure was caused by | |
| 799 # request being blocked because it wasn't sent through proxy. (This | |
| 800 # error could also happen if gsutil or the oauth2 client had a bug that | |
| 801 # attempted to incorrectly reuse an HTTP connection, for example.) | |
| 802 sys.stdout.write('\n'.join(textwrap.wrap( | |
| 803 "Unable to connect to accounts.google.com during OAuth2 flow. This " | |
| 804 "can happen if your site uses a proxy. If you are using gsutil " | |
| 805 "through a proxy, please enter the proxy's information; otherwise " | |
| 806 "leave the following fields blank.")) + '\n') | |
| 807 self._PromptForProxyConfig() | |
| 808 oauth2_client = oauth2_helper.OAuth2ClientFromBotoConfig(boto.config, | |
| 809 cred_type) | |
| 810 oauth2_refresh_token = oauth2_helper.OAuth2ApprovalFlow( | |
| 811 oauth2_client, oauth2_scopes, launch_browser) | |
| 812 elif cred_type == CredTypes.HMAC: | |
| 813 got_creds = False | |
| 814 for provider in provider_map: | |
| 815 if provider == 'google': | |
| 816 key_ids[provider] = raw_input('What is your %s access key ID? ' % | |
| 817 provider) | |
| 818 sec_keys[provider] = raw_input('What is your %s secret access key? ' % | |
| 819 provider) | |
| 820 got_creds = True | |
| 821 if not key_ids[provider] or not sec_keys[provider]: | |
| 822 raise CommandException( | |
| 823 'Incomplete credentials provided. Please try again.') | |
| 824 if not got_creds: | |
| 825 raise CommandException('No credentials provided. Please try again.') | |
| 826 | |
| 827 # Write the config file prelude. | |
| 828 config_file.write(CONFIG_PRELUDE_CONTENT.lstrip()) | |
| 829 config_file.write( | |
| 830 '# This file was created by gsutil version %s at %s.\n' | |
| 831 % (gslib.VERSION, | |
| 832 datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) | |
| 833 config_file.write( | |
| 834 '#\n# You can create additional configuration files by ' | |
| 835 'running\n# gsutil config [options] [-o <config-file>]\n\n\n') | |
| 836 | |
| 837 # Write the config file Credentials section. | |
| 838 config_file.write('[Credentials]\n\n') | |
| 839 if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: | |
| 840 config_file.write('# Google OAuth2 service account credentials ' | |
| 841 '(for "gs://" URIs):\n') | |
| 842 config_file.write('gs_service_key_file = %s\n' % gs_service_key_file) | |
| 843 if not service_account_key_is_json: | |
| 844 config_file.write('gs_service_client_id = %s\n' | |
| 845 % gs_service_client_id) | |
| 846 | |
| 847 if not gs_service_key_file_password: | |
| 848 config_file.write( | |
| 849 '# If you would like to set your password, you can do so using\n' | |
| 850 '# the following commands (replaced with your information):\n' | |
| 851 '# "openssl pkcs12 -in cert1.p12 -out temp_cert.pem"\n' | |
| 852 '# "openssl pkcs12 -export -in temp_cert.pem -out cert2.p12"\n' | |
| 853 '# "rm -f temp_cert.pem"\n' | |
| 854 '# Your initial password is "notasecret" - for more information,' | |
| 855 '\n# please see http://www.openssl.org/docs/apps/pkcs12.html.\n') | |
| 856 config_file.write('#gs_service_key_file_password =\n\n') | |
| 857 else: | |
| 858 config_file.write('gs_service_key_file_password = %s\n\n' | |
| 859 % gs_service_key_file_password) | |
| 860 elif cred_type == CredTypes.OAUTH2_USER_ACCOUNT: | |
| 861 config_file.write( | |
| 862 '# Google OAuth2 credentials (for "gs://" URIs):\n' | |
| 863 '# The following OAuth2 account is authorized for scope(s):\n') | |
| 864 for scope in oauth2_scopes: | |
| 865 config_file.write('# %s\n' % scope) | |
| 866 config_file.write( | |
| 867 'gs_oauth2_refresh_token = %s\n\n' % oauth2_refresh_token) | |
| 868 else: | |
| 869 config_file.write( | |
| 870 '# To add Google OAuth2 credentials ("gs://" URIs), ' | |
| 871 'edit and uncomment the\n# following line:\n' | |
| 872 '#gs_oauth2_refresh_token = <your OAuth2 refresh token>\n\n') | |
| 873 | |
| 874 for provider in provider_map: | |
| 875 key_prefix = provider_map[provider] | |
| 876 uri_scheme = uri_map[provider] | |
| 877 if provider in key_ids and provider in sec_keys: | |
| 878 config_file.write('# %s credentials ("%s://" URIs):\n' % | |
| 879 (provider, uri_scheme)) | |
| 880 config_file.write('%s_access_key_id = %s\n' % | |
| 881 (key_prefix, key_ids[provider])) | |
| 882 config_file.write('%s_secret_access_key = %s\n' % | |
| 883 (key_prefix, sec_keys[provider])) | |
| 884 else: | |
| 885 config_file.write( | |
| 886 '# To add %s credentials ("%s://" URIs), edit and ' | |
| 887 'uncomment the\n# following two lines:\n' | |
| 888 '#%s_access_key_id = <your %s access key ID>\n' | |
| 889 '#%s_secret_access_key = <your %s secret access key>\n' % | |
| 890 (provider, uri_scheme, key_prefix, provider, key_prefix, | |
| 891 provider)) | |
| 892 host_key = Provider.HostKeyMap[provider] | |
| 893 config_file.write( | |
| 894 '# The ability to specify an alternate storage host and port\n' | |
| 895 '# is primarily for cloud storage service developers.\n' | |
| 896 '# Setting a non-default gs_host only works if prefer_api=xml.\n' | |
| 897 '#%s_host = <alternate storage host address>\n' | |
| 898 '#%s_port = <alternate storage host port>\n' | |
| 899 % (host_key, host_key)) | |
| 900 if host_key == 'gs': | |
| 901 config_file.write( | |
| 902 '#%s_json_host = <alternate JSON API storage host address>\n' | |
| 903 '#%s_json_port = <alternate JSON API storage host port>\n\n' | |
| 904 % (host_key, host_key)) | |
| 905 config_file.write('\n') | |
| 906 | |
| 907 # Write the config file Boto section. | |
| 908 config_file.write('%s\n' % CONFIG_BOTO_SECTION_CONTENT) | |
| 909 self._WriteProxyConfigFileSection(config_file) | |
| 910 | |
| 911 # Write the config file GSUtil section that doesn't depend on user input. | |
| 912 config_file.write(CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT) | |
| 913 | |
| 914 # Write the default API version. | |
| 915 config_file.write(""" | |
| 916 # 'default_api_version' specifies the default Google Cloud Storage XML API | |
| 917 # version to use. If not set below gsutil defaults to API version 1. | |
| 918 """) | |
| 919 api_version = 2 | |
| 920 if cred_type == CredTypes.HMAC: api_version = 1 | |
| 921 | |
| 922 config_file.write('default_api_version = %d\n' % api_version) | |
| 923 | |
| 924 # Write the config file GSUtil section that includes the default | |
| 925 # project ID input from the user. | |
| 926 if launch_browser: | |
| 927 sys.stdout.write( | |
| 928 'Attempting to launch a browser to open the Google Cloud Console at ' | |
| 929 'URL: %s\n\n' | |
| 930 '[Note: due to a Python bug, you may see a spurious error message ' | |
| 931 '"object is not\ncallable [...] in [...] Popen.__del__" which can ' | |
| 932 'be ignored.]\n\n' % GOOG_CLOUD_CONSOLE_URI) | |
| 933 sys.stdout.write( | |
| 934 'In your browser you should see the Cloud Console. Find the project ' | |
| 935 'you will\nuse, and then copy the Project ID string from the second ' | |
| 936 'column. Older projects do\nnot have Project ID strings. For such ' | |
| 937 'projects, click the project and then copy the\nProject Number ' | |
| 938 'listed under that project.\n\n') | |
| 939 if not webbrowser.open(GOOG_CLOUD_CONSOLE_URI, new=1, autoraise=True): | |
| 940 sys.stdout.write( | |
| 941 'Launching browser appears to have failed; please navigate a ' | |
| 942 'browser to the following URL:\n%s\n' % GOOG_CLOUD_CONSOLE_URI) | |
| 943 # Short delay; webbrowser.open on linux insists on printing out a message | |
| 944 # which we don't want to run into the prompt for the auth code. | |
| 945 time.sleep(2) | |
| 946 else: | |
| 947 sys.stdout.write( | |
| 948 '\nPlease navigate your browser to %s,\nthen find the project you ' | |
| 949 'will use, and copy the Project ID string from the\nsecond column. ' | |
| 950 'Older projects do not have Project ID strings. For such projects,\n' | |
| 951 'click the project and then copy the Project Number listed under ' | |
| 952 'that project.\n\n' % GOOG_CLOUD_CONSOLE_URI) | |
| 953 default_project_id = raw_input('What is your project-id? ').strip() | |
| 954 project_id_section_prelude = """ | |
| 955 # 'default_project_id' specifies the default Google Cloud Storage project ID to | |
| 956 # use with the 'mb' and 'ls' commands. This default can be overridden by | |
| 957 # specifying the -p option to the 'mb' and 'ls' commands. | |
| 958 """ | |
| 959 if not default_project_id: | |
| 960 raise CommandException( | |
| 961 'No default project ID entered. The default project ID is needed by ' | |
| 962 'the\nls and mb commands; please try again.') | |
| 963 config_file.write('%sdefault_project_id = %s\n\n\n' % | |
| 964 (project_id_section_prelude, default_project_id)) | |
| 965 | |
| 966 # Write the config file OAuth2 section. | |
| 967 config_file.write(CONFIG_OAUTH2_CONFIG_CONTENT) | |
| 968 | |
| 969 def RunCommand(self): | |
| 970 """Command entry point for the config command.""" | |
| 971 scopes = [] | |
| 972 cred_type = CredTypes.OAUTH2_USER_ACCOUNT | |
| 973 launch_browser = False | |
| 974 output_file_name = None | |
| 975 has_a = False | |
| 976 has_e = False | |
| 977 for opt, opt_arg in self.sub_opts: | |
| 978 if opt == '-a': | |
| 979 cred_type = CredTypes.HMAC | |
| 980 has_a = True | |
| 981 elif opt == '-b': | |
| 982 launch_browser = True | |
| 983 elif opt == '-e': | |
| 984 cred_type = CredTypes.OAUTH2_SERVICE_ACCOUNT | |
| 985 has_e = True | |
| 986 elif opt == '-f': | |
| 987 scopes.append(SCOPE_FULL_CONTROL) | |
| 988 elif opt == '-o': | |
| 989 output_file_name = opt_arg | |
| 990 elif opt == '-r': | |
| 991 scopes.append(SCOPE_READ_ONLY) | |
| 992 elif opt == '-s': | |
| 993 scopes.append(opt_arg) | |
| 994 elif opt == '-w': | |
| 995 scopes.append(SCOPE_READ_WRITE) | |
| 996 else: | |
| 997 self.RaiseInvalidArgumentException() | |
| 998 | |
| 999 if has_e and has_a: | |
| 1000 raise CommandException('Both -a and -e cannot be specified. Please see ' | |
| 1001 '"gsutil help config" for more information.') | |
| 1002 | |
| 1003 if not scopes: | |
| 1004 scopes.append(SCOPE_FULL_CONTROL) | |
| 1005 | |
| 1006 default_config_path_bak = None | |
| 1007 if not output_file_name: | |
| 1008 # Check to see if a default config file name is requested via | |
| 1009 # environment variable. If so, use it, otherwise use the hard-coded | |
| 1010 # default file. Then use the default config file name, if it doesn't | |
| 1011 # exist or can be moved out of the way without clobbering an existing | |
| 1012 # backup file. | |
| 1013 boto_config_from_env = os.environ.get('BOTO_CONFIG', None) | |
| 1014 if boto_config_from_env: | |
| 1015 default_config_path = boto_config_from_env | |
| 1016 else: | |
| 1017 default_config_path = os.path.expanduser(os.path.join('~', '.boto')) | |
| 1018 if not os.path.exists(default_config_path): | |
| 1019 output_file_name = default_config_path | |
| 1020 else: | |
| 1021 default_config_path_bak = default_config_path + '.bak' | |
| 1022 if os.path.exists(default_config_path_bak): | |
| 1023 raise CommandException( | |
| 1024 'Cannot back up existing config ' | |
| 1025 'file "%s": backup file exists ("%s").' | |
| 1026 % (default_config_path, default_config_path_bak)) | |
| 1027 else: | |
| 1028 try: | |
| 1029 sys.stderr.write( | |
| 1030 'Backing up existing config file "%s" to "%s"...\n' | |
| 1031 % (default_config_path, default_config_path_bak)) | |
| 1032 os.rename(default_config_path, default_config_path_bak) | |
| 1033 except Exception, e: | |
| 1034 raise CommandException( | |
| 1035 'Failed to back up existing config ' | |
| 1036 'file ("%s" -> "%s"): %s.' | |
| 1037 % (default_config_path, default_config_path_bak, e)) | |
| 1038 output_file_name = default_config_path | |
| 1039 | |
| 1040 if output_file_name == '-': | |
| 1041 output_file = sys.stdout | |
| 1042 else: | |
| 1043 output_file = self._OpenConfigFile(output_file_name) | |
| 1044 sys.stderr.write('\n'.join(textwrap.wrap( | |
| 1045 'This command will create a boto config file at %s containing your ' | |
| 1046 'credentials, based on your responses to the following questions.' | |
| 1047 % output_file_name)) + '\n') | |
| 1048 | |
| 1049 # Catch ^C so we can restore the backup. | |
| 1050 RegisterSignalHandler(signal.SIGINT, _CleanupHandler) | |
| 1051 try: | |
| 1052 self._WriteBotoConfigFile(output_file, launch_browser=launch_browser, | |
| 1053 oauth2_scopes=scopes, cred_type=cred_type) | |
| 1054 except Exception as e: | |
| 1055 user_aborted = isinstance(e, AbortException) | |
| 1056 if user_aborted: | |
| 1057 sys.stderr.write('\nCaught ^C; cleaning up\n') | |
| 1058 # If an error occurred during config file creation, remove the invalid | |
| 1059 # config file and restore the backup file. | |
| 1060 if output_file_name != '-': | |
| 1061 output_file.close() | |
| 1062 os.unlink(output_file_name) | |
| 1063 try: | |
| 1064 if default_config_path_bak: | |
| 1065 sys.stderr.write('Restoring previous backed up file (%s)\n' % | |
| 1066 default_config_path_bak) | |
| 1067 os.rename(default_config_path_bak, output_file_name) | |
| 1068 except Exception as e: | |
| 1069 # Raise the original exception so that we can see what actually went | |
| 1070 # wrong, rather than just finding out that we died before assigning | |
| 1071 # a value to default_config_path_bak. | |
| 1072 raise e | |
| 1073 raise | |
| 1074 | |
| 1075 if output_file_name != '-': | |
| 1076 output_file.close() | |
| 1077 if not boto.config.has_option('Boto', 'proxy'): | |
| 1078 sys.stderr.write('\n' + '\n'.join(textwrap.wrap( | |
| 1079 'Boto config file "%s" created.\nIf you need to use a proxy to ' | |
| 1080 'access the Internet please see the instructions in that file.' | |
| 1081 % output_file_name)) + '\n') | |
| 1082 | |
| 1083 return 0 | |
| 1084 | |
| 1085 | |
| 1086 def _CleanupHandler(unused_signalnum, unused_handler): | |
| 1087 raise AbortException('User interrupted config command') | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1260493004:
Revert "Add gsutil 4.13 to telemetry/third_party" (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master