OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """This module contains functions for fetching and extracting archived builds. | 5 """This module contains functions for fetching and extracting archived builds. |
6 | 6 |
7 The builds may be stored in different places by different types of builders; | 7 The builds may be stored in different places by different types of builders; |
8 for example, builders on tryserver.chromium.perf stores builds in one place, | 8 for example, builders on tryserver.chromium.perf stores builds in one place, |
9 while builders on chromium.linux store builds in another. | 9 while builders on chromium.linux store builds in another. |
10 | 10 |
(...skipping 19 matching lines...) Expand all Loading... |
30 # Possible builder types. | 30 # Possible builder types. |
31 PERF_BUILDER = 'perf' | 31 PERF_BUILDER = 'perf' |
32 FULL_BUILDER = 'full' | 32 FULL_BUILDER = 'full' |
33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf' | 33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf' |
34 | 34 |
35 # Maximum time in seconds to wait after posting build request to the try server. | 35 # Maximum time in seconds to wait after posting build request to the try server. |
36 MAX_MAC_BUILD_TIME = 14400 | 36 MAX_MAC_BUILD_TIME = 14400 |
37 MAX_WIN_BUILD_TIME = 14400 | 37 MAX_WIN_BUILD_TIME = 14400 |
38 MAX_LINUX_BUILD_TIME = 14400 | 38 MAX_LINUX_BUILD_TIME = 14400 |
39 | 39 |
| 40 # Try server status page URLs, used to get build status. |
| 41 PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf' |
| 42 LINUX_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.linux' |
| 43 |
40 | 44 |
41 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER, | 45 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER, |
42 target_arch='ia32', target_platform='chromium', | 46 target_arch='ia32', target_platform='chromium', |
43 deps_patch_sha=None, extra_src=None): | 47 deps_patch_sha=None, extra_src=None): |
44 """Returns the location where a build archive is expected to be. | 48 """Returns the location where a build archive is expected to be. |
45 | 49 |
46 Args: | 50 Args: |
47 revision: Revision string, e.g. a git commit hash or SVN revision. | 51 revision: Revision string, e.g. a git commit hash or SVN revision. |
48 builder_type: Type of build archive. | 52 builder_type: Type of build archive. |
49 target_arch: Architecture, e.g. "ia32". | 53 target_arch: Architecture, e.g. "ia32". |
(...skipping 20 matching lines...) Expand all Loading... |
70 target_platform='chromium', extra_src=None): | 74 target_platform='chromium', extra_src=None): |
71 """Gets builder bot name and build time in seconds based on platform.""" | 75 """Gets builder bot name and build time in seconds based on platform.""" |
72 logging.info('Getting builder name for builder "%s", "%s", "%s".', | 76 logging.info('Getting builder name for builder "%s", "%s", "%s".', |
73 builder_type, target_arch, target_platform) | 77 builder_type, target_arch, target_platform) |
74 build_archive = BuildArchive.Create( | 78 build_archive = BuildArchive.Create( |
75 builder_type, target_arch=target_arch, target_platform=target_platform, | 79 builder_type, target_arch=target_arch, target_platform=target_platform, |
76 extra_src=extra_src) | 80 extra_src=extra_src) |
77 return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime() | 81 return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime() |
78 | 82 |
79 | 83 |
| 84 def GetBuildBotUrl(builder_type=PERF_BUILDER, target_arch='ia32', |
| 85 target_platform='chromium', extra_src=None): |
| 86 """Gets buildbot URL for a given builder type.""" |
| 87 logging.info('Getting buildbot URL for "%s", "%s", "%s".', |
| 88 builder_type, target_arch, target_platform) |
| 89 build_archive = BuildArchive.Create( |
| 90 builder_type, target_arch=target_arch, target_platform=target_platform, |
| 91 extra_src=extra_src) |
| 92 return build_archive.GetBuildBotUrl() |
| 93 |
| 94 |
80 class BuildArchive(object): | 95 class BuildArchive(object): |
81 """Represents a place where builds of some type are stored. | 96 """Represents a place where builds of some type are stored. |
82 | 97 |
83 There are two pieces of information required to locate a file in Google | 98 There are two pieces of information required to locate a file in Google |
84 Cloud Storage, bucket name and file path. Subclasses of this class contain | 99 Cloud Storage, bucket name and file path. Subclasses of this class contain |
85 specific logic about which bucket names and paths should be used to fetch | 100 specific logic about which bucket names and paths should be used to fetch |
86 a build. | 101 a build. |
87 """ | 102 """ |
88 | 103 |
89 @staticmethod | 104 @staticmethod |
(...skipping 12 matching lines...) Expand all Loading... |
102 target_arch, target_platform, loaded_extra_src) | 117 target_arch, target_platform, loaded_extra_src) |
103 except (IOError, TypeError, ImportError): | 118 except (IOError, TypeError, ImportError): |
104 raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src) | 119 raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src) |
105 raise NotImplementedError('Builder type "%s" not supported.' % builder_type) | 120 raise NotImplementedError('Builder type "%s" not supported.' % builder_type) |
106 | 121 |
107 def __init__(self, target_arch='ia32', target_platform='chromium', | 122 def __init__(self, target_arch='ia32', target_platform='chromium', |
108 extra_src=None): | 123 extra_src=None): |
109 self._extra_src = extra_src | 124 self._extra_src = extra_src |
110 if bisect_utils.IsLinuxHost() and target_platform == 'android': | 125 if bisect_utils.IsLinuxHost() and target_platform == 'android': |
111 self._platform = 'android' | 126 self._platform = 'android' |
| 127 elif bisect_utils.IsLinuxHost() and target_platform == 'android-chrome': |
| 128 self._platform = 'android-chrome' |
112 elif bisect_utils.IsLinuxHost(): | 129 elif bisect_utils.IsLinuxHost(): |
113 self._platform = 'linux' | 130 self._platform = 'linux' |
114 elif bisect_utils.IsMacHost(): | 131 elif bisect_utils.IsMacHost(): |
115 self._platform = 'mac' | 132 self._platform = 'mac' |
116 elif bisect_utils.Is64BitWindows() and target_arch == 'x64': | 133 elif bisect_utils.Is64BitWindows() and target_arch == 'x64': |
117 self._platform = 'win64' | 134 self._platform = 'win64' |
118 elif bisect_utils.IsWindowsHost(): | 135 elif bisect_utils.IsWindowsHost(): |
119 self._platform = 'win' | 136 self._platform = 'win' |
120 else: | 137 else: |
121 raise NotImplementedError('Unknown platform "%s".' % sys.platform) | 138 raise NotImplementedError('Unknown platform "%s".' % sys.platform) |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
168 return 'mac' | 185 return 'mac' |
169 raise NotImplementedError('Unknown platform "%s".' % sys.platform) | 186 raise NotImplementedError('Unknown platform "%s".' % sys.platform) |
170 | 187 |
171 def GetBuilderName(self): | 188 def GetBuilderName(self): |
172 raise NotImplementedError() | 189 raise NotImplementedError() |
173 | 190 |
174 def GetBuilderBuildTime(self): | 191 def GetBuilderBuildTime(self): |
175 """Returns the time to wait for a build after requesting one.""" | 192 """Returns the time to wait for a build after requesting one.""" |
176 if self._platform in ('win', 'win64'): | 193 if self._platform in ('win', 'win64'): |
177 return MAX_WIN_BUILD_TIME | 194 return MAX_WIN_BUILD_TIME |
178 if self._platform in ('linux', 'android'): | 195 if self._platform in ('linux', 'android', 'android-chrome'): |
179 return MAX_LINUX_BUILD_TIME | 196 return MAX_LINUX_BUILD_TIME |
180 if self._platform == 'mac': | 197 if self._platform == 'mac': |
181 return MAX_MAC_BUILD_TIME | 198 return MAX_MAC_BUILD_TIME |
182 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) | 199 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) |
183 | 200 |
| 201 def GetBuildBotUrl(self): |
| 202 raise NotImplementedError() |
| 203 |
184 | 204 |
185 class PerfBuildArchive(BuildArchive): | 205 class PerfBuildArchive(BuildArchive): |
186 | 206 |
187 def BucketName(self): | 207 def BucketName(self): |
188 return 'chrome-perf' | 208 return 'chrome-perf' |
189 | 209 |
190 def FilePath(self, revision, deps_patch_sha=None): | 210 def FilePath(self, revision, deps_patch_sha=None): |
191 return '%s/%s' % (self._ArchiveDirectory(), | 211 return '%s/%s' % (self._ArchiveDirectory(), |
192 self._ZipFileName(revision, deps_patch_sha)) | 212 self._ZipFileName(revision, deps_patch_sha)) |
193 | 213 |
(...skipping 16 matching lines...) Expand all Loading... |
210 elif self._platform == 'win': | 230 elif self._platform == 'win': |
211 return 'win_perf_bisect_builder' | 231 return 'win_perf_bisect_builder' |
212 elif self._platform == 'linux': | 232 elif self._platform == 'linux': |
213 return 'linux_perf_bisect_builder' | 233 return 'linux_perf_bisect_builder' |
214 elif self._platform == 'android': | 234 elif self._platform == 'android': |
215 return 'android_perf_bisect_builder' | 235 return 'android_perf_bisect_builder' |
216 elif self._platform == 'mac': | 236 elif self._platform == 'mac': |
217 return 'mac_perf_bisect_builder' | 237 return 'mac_perf_bisect_builder' |
218 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) | 238 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) |
219 | 239 |
| 240 def GetBuildBotUrl(self): |
| 241 """Returns buildbot URL for fetching build info.""" |
| 242 return PERF_TRY_SERVER_URL |
| 243 |
220 | 244 |
221 class FullBuildArchive(BuildArchive): | 245 class FullBuildArchive(BuildArchive): |
222 | 246 |
223 def BucketName(self): | 247 def BucketName(self): |
224 platform_to_bucket = { | 248 platform_to_bucket = { |
225 'android': 'chromium-android', | 249 'android': 'chromium-android', |
226 'linux': 'chromium-linux-archive', | 250 'linux': 'chromium-linux-archive', |
227 'mac': 'chromium-mac-archive', | 251 'mac': 'chromium-mac-archive', |
228 'win64': 'chromium-win-archive', | 252 'win64': 'chromium-win-archive', |
229 'win': 'chromium-win-archive', | 253 'win': 'chromium-win-archive', |
(...skipping 16 matching lines...) Expand all Loading... |
246 } | 270 } |
247 assert self._platform in platform_to_directory | 271 assert self._platform in platform_to_directory |
248 return platform_to_directory.get(self._platform) | 272 return platform_to_directory.get(self._platform) |
249 | 273 |
250 def GetBuilderName(self): | 274 def GetBuilderName(self): |
251 """Gets builder bot name based on platform.""" | 275 """Gets builder bot name based on platform.""" |
252 if self._platform == 'linux': | 276 if self._platform == 'linux': |
253 return 'linux_full_bisect_builder' | 277 return 'linux_full_bisect_builder' |
254 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) | 278 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) |
255 | 279 |
| 280 def GetBuildBotUrl(self): |
| 281 """Returns buildbot URL for fetching build info.""" |
| 282 return LINUX_TRY_SERVER_URL |
| 283 |
| 284 |
256 class AndroidChromeBuildArchive(BuildArchive): | 285 class AndroidChromeBuildArchive(BuildArchive): |
257 """Represents a place where builds of android-chrome type are stored. | 286 """Represents a place where builds of android-chrome type are stored. |
258 | 287 |
259 If AndroidChromeBuildArchive is used, it is assumed that the --extra_src | 288 If AndroidChromeBuildArchive is used, it is assumed that the --extra_src |
260 is a valid Python module which contains the module-level functions | 289 is a valid Python module which contains the module-level functions |
261 GetBucketName and GetArchiveDirectory. | 290 GetBucketName and GetArchiveDirectory. |
262 """ | 291 """ |
263 | 292 |
264 def BucketName(self): | 293 def BucketName(self): |
265 return self._extra_src.GetBucketName() | 294 return self._extra_src.GetBucketName() |
(...skipping 20 matching lines...) Expand all Loading... |
286 self._ZipFileName(revision, deps_patch_sha)) | 315 self._ZipFileName(revision, deps_patch_sha)) |
287 | 316 |
288 def _ArchiveDirectory(self): | 317 def _ArchiveDirectory(self): |
289 """Returns the directory name to download builds from.""" | 318 """Returns the directory name to download builds from.""" |
290 return self._extra_src.GetArchiveDirectory() | 319 return self._extra_src.GetArchiveDirectory() |
291 | 320 |
292 def GetBuilderName(self): | 321 def GetBuilderName(self): |
293 """Returns the builder name extra source.""" | 322 """Returns the builder name extra source.""" |
294 return self._extra_src.GetBuilderName() | 323 return self._extra_src.GetBuilderName() |
295 | 324 |
| 325 def GetBuildBotUrl(self): |
| 326 """Returns buildbot URL for fetching build info.""" |
| 327 return self._extra_src.GetBuildBotUrl() |
| 328 |
296 | 329 |
297 def BuildIsAvailable(bucket_name, remote_path): | 330 def BuildIsAvailable(bucket_name, remote_path): |
298 """Checks whether a build is currently archived at some place.""" | 331 """Checks whether a build is currently archived at some place.""" |
299 logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path)) | 332 logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path)) |
300 try: | 333 try: |
301 exists = cloud_storage.Exists(bucket_name, remote_path) | 334 exists = cloud_storage.Exists(bucket_name, remote_path) |
302 logging.info('Exists? %s' % exists) | 335 logging.info('Exists? %s' % exists) |
303 return exists | 336 return exists |
304 except cloud_storage.CloudStorageError: | 337 except cloud_storage.CloudStorageError: |
305 return False | 338 return False |
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
461 print 'Build is not available.' | 494 print 'Build is not available.' |
462 return 1 | 495 return 1 |
463 | 496 |
464 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir) | 497 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir) |
465 print 'Build has been downloaded to and extracted in %s.' % args.output_dir | 498 print 'Build has been downloaded to and extracted in %s.' % args.output_dir |
466 return 0 | 499 return 0 |
467 | 500 |
468 | 501 |
469 if __name__ == '__main__': | 502 if __name__ == '__main__': |
470 sys.exit(Main(sys.argv)) | 503 sys.exit(Main(sys.argv)) |
OLD | NEW |