OLD | NEW |
| (Empty) |
1 # Copyright 2010 Google Inc. All Rights Reserved. | |
2 # | |
3 # Licensed under the Apache License, Version 2.0 (the "License"); | |
4 # you may not use this file except in compliance with the License. | |
5 # You may obtain a copy of the License at | |
6 # | |
7 # http://www.apache.org/licenses/LICENSE-2.0 | |
8 # | |
9 # Unless required by applicable law or agreed to in writing, software | |
10 # distributed under the License is distributed on an "AS IS" BASIS, | |
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
12 # See the License for the specific language governing permissions and | |
13 # limitations under the License. | |
14 | |
15 """Static data and helper functions.""" | |
16 | |
17 import math | |
18 import re | |
19 import os | |
20 import sys | |
21 import time | |
22 | |
23 import boto | |
24 from third_party.retry_decorator.decorators import retry | |
25 | |
26 # We don't use the oauth2 authentication plugin directly; importing it here | |
27 # ensures that it's loaded and available by default. Note: we made this static | |
28 # state instead of Command instance state because the top-level gsutil code | |
29 # needs to check it. | |
30 HAVE_OAUTH2 = False | |
31 try: | |
32 from oauth2_plugin import oauth2_helper | |
33 HAVE_OAUTH2 = True | |
34 except ImportError: | |
35 pass | |
36 | |
37 TWO_MB = 2 * 1024 * 1024 | |
38 | |
39 NO_MAX = sys.maxint | |
40 | |
41 # Binary exponentiation strings. | |
42 _EXP_STRINGS = [ | |
43 (0, 'B', 'bit'), | |
44 (10, 'KB', 'kbit'), | |
45 (20, 'MB', 'Mbit'), | |
46 (30, 'GB', 'Gbit'), | |
47 (40, 'TB', 'Tbit'), | |
48 (50, 'PB', 'Pbit'), | |
49 ] | |
50 | |
51 # Detect platform types. | |
52 IS_WINDOWS = 'win32' in str(sys.platform).lower() | |
53 IS_LINUX = 'linux' in str(sys.platform).lower() | |
54 IS_OSX = 'darwin' in str(sys.platform).lower() | |
55 | |
56 Retry = retry | |
57 | |
58 # Enum class for specifying listing style. | |
59 class ListingStyle(object): | |
60 SHORT = 'SHORT' | |
61 LONG = 'LONG' | |
62 LONG_LONG = 'LONG_LONG' | |
63 | |
64 | |
65 def HasConfiguredCredentials(bypass_prodaccess): | |
66 """Determines if boto credential/config file exists.""" | |
67 config = boto.config | |
68 has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and | |
69 config.has_option('Credentials', 'gs_secret_access_key')) | |
70 has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and | |
71 config.has_option('Credentials', 'aws_secret_access_key')) | |
72 has_oauth_creds = (HAVE_OAUTH2 and | |
73 config.has_option('Credentials', 'gs_oauth2_refresh_token')) | |
74 has_auth_plugins = config.has_option('Plugin', 'plugin_directory') | |
75 # Pretend prodaccess doesn't exist if --bypass_prodaccess is passed in. | |
76 has_prodaccess = HasExecutable('prodaccess') and not bypass_prodaccess | |
77 return (has_goog_creds or has_amzn_creds or has_oauth_creds | |
78 or has_auth_plugins or has_prodaccess) | |
79 | |
80 | |
81 def _RoundToNearestExponent(num): | |
82 i = 0 | |
83 while i+1 < len(_EXP_STRINGS) and num >= (2 ** _EXP_STRINGS[i+1][0]): | |
84 i += 1 | |
85 return i, round(float(num) / 2 ** _EXP_STRINGS[i][0], 2) | |
86 | |
87 def MakeHumanReadable(num): | |
88 """Generates human readable string for a number of bytes. | |
89 | |
90 Args: | |
91 num: The number, in bytes. | |
92 | |
93 Returns: | |
94 A string form of the number using size abbreviations (KB, MB, etc.). | |
95 """ | |
96 i, rounded_val = _RoundToNearestExponent(num) | |
97 return '%s %s' % (rounded_val, _EXP_STRINGS[i][1]) | |
98 | |
99 def MakeBitsHumanReadable(num): | |
100 """Generates human readable string for a number of bits. | |
101 | |
102 Args: | |
103 num: The number, in bits. | |
104 | |
105 Returns: | |
106 A string form of the number using bit size abbreviations (kbit, Mbit, etc.) | |
107 """ | |
108 i, rounded_val = _RoundToNearestExponent(num) | |
109 return '%s %s' % (rounded_val, _EXP_STRINGS[i][2]) | |
110 | |
111 def Percentile(values, percent, key=lambda x:x): | |
112 """Find the percentile of a list of values. | |
113 | |
114 Taken from: http://code.activestate.com/recipes/511478/ | |
115 | |
116 Args: | |
117 values: a list of numeric values. Note that the values MUST BE already | |
118 sorted. | |
119 percent: a float value from 0.0 to 1.0. | |
120 key: optional key function to compute value from each element of the list | |
121 of values. | |
122 | |
123 Returns: | |
124 The percentile of the values. | |
125 """ | |
126 if not values: | |
127 return None | |
128 k = (len(values) - 1) * percent | |
129 f = math.floor(k) | |
130 c = math.ceil(k) | |
131 if f == c: | |
132 return key(values[int(k)]) | |
133 d0 = key(values[int(f)]) * (c-k) | |
134 d1 = key(values[int(c)]) * (k-f) | |
135 return d0 + d1 | |
136 | |
137 def ExtractErrorDetail(e): | |
138 """Extract <Details> text from XML content. | |
139 | |
140 Args: | |
141 e: The GSResponseError that includes XML to be parsed. | |
142 | |
143 Returns: | |
144 (exception_name, d), where d is <Details> text or None if not found. | |
145 """ | |
146 exc_name_parts = re.split("[\.']", str(type(e))) | |
147 if len(exc_name_parts) < 2: | |
148 # Shouldn't happen, but have fallback in case. | |
149 exc_name = str(type(e)) | |
150 else: | |
151 exc_name = exc_name_parts[-2] | |
152 if not hasattr(e, 'body'): | |
153 return (exc_name, None) | |
154 detail_start = e.body.find('<Details>') | |
155 detail_end = e.body.find('</Details>') | |
156 if detail_start != -1 and detail_end != -1: | |
157 return (exc_name, e.body[detail_start+9:detail_end]) | |
158 return (exc_name, None) | |
159 | |
160 | |
161 def HasExecutable(filename): | |
162 """Determines if an executable is available on the system.""" | |
163 for path in os.environ['PATH'].split(os.pathsep): | |
164 exe_file = os.path.join(path, filename) | |
165 if os.path.exists(exe_file) and os.access(exe_file, os.X_OK): | |
166 return True | |
167 return False | |
OLD | NEW |