| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """This module contains functions for fetching and extracting archived builds. | 5 """This module contains functions for fetching and extracting archived builds. |
| 6 | 6 |
| 7 The builds may be stored in different places by different types of builders; | 7 The builds may be stored in different places by different types of builders; |
| 8 for example, builders on tryserver.chromium.perf stores builds in one place, | 8 for example, builders on tryserver.chromium.perf stores builds in one place, |
| 9 while builders on chromium.linux store builds in another. | 9 while builders on chromium.linux store builds in another. |
| 10 | 10 |
| (...skipping 19 matching lines...) Expand all Loading... |
| 30 # Possible builder types. | 30 # Possible builder types. |
| 31 PERF_BUILDER = 'perf' | 31 PERF_BUILDER = 'perf' |
| 32 FULL_BUILDER = 'full' | 32 FULL_BUILDER = 'full' |
| 33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf' | 33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf' |
| 34 | 34 |
| 35 # Maximum time in seconds to wait after posting build request to the try server. | 35 # Maximum time in seconds to wait after posting build request to the try server. |
| 36 MAX_MAC_BUILD_TIME = 14400 | 36 MAX_MAC_BUILD_TIME = 14400 |
| 37 MAX_WIN_BUILD_TIME = 14400 | 37 MAX_WIN_BUILD_TIME = 14400 |
| 38 MAX_LINUX_BUILD_TIME = 14400 | 38 MAX_LINUX_BUILD_TIME = 14400 |
| 39 | 39 |
| 40 # Try server status page URLs, used to get build status. |
| 41 PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf' |
| 42 LINUX_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.linux' |
| 43 |
| 40 | 44 |
| 41 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER, | 45 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER, |
| 42 target_arch='ia32', target_platform='chromium', | 46 target_arch='ia32', target_platform='chromium', |
| 43 deps_patch_sha=None, extra_src=None): | 47 deps_patch_sha=None, extra_src=None): |
| 44 """Returns the location where a build archive is expected to be. | 48 """Returns the location where a build archive is expected to be. |
| 45 | 49 |
| 46 Args: | 50 Args: |
| 47 revision: Revision string, e.g. a git commit hash or SVN revision. | 51 revision: Revision string, e.g. a git commit hash or SVN revision. |
| 48 builder_type: Type of build archive. | 52 builder_type: Type of build archive. |
| 49 target_arch: Architecture, e.g. "ia32". | 53 target_arch: Architecture, e.g. "ia32". |
| (...skipping 20 matching lines...) Expand all Loading... |
| 70 target_platform='chromium', extra_src=None): | 74 target_platform='chromium', extra_src=None): |
| 71 """Gets builder bot name and build time in seconds based on platform.""" | 75 """Gets builder bot name and build time in seconds based on platform.""" |
| 72 logging.info('Getting builder name for builder "%s", "%s", "%s".', | 76 logging.info('Getting builder name for builder "%s", "%s", "%s".', |
| 73 builder_type, target_arch, target_platform) | 77 builder_type, target_arch, target_platform) |
| 74 build_archive = BuildArchive.Create( | 78 build_archive = BuildArchive.Create( |
| 75 builder_type, target_arch=target_arch, target_platform=target_platform, | 79 builder_type, target_arch=target_arch, target_platform=target_platform, |
| 76 extra_src=extra_src) | 80 extra_src=extra_src) |
| 77 return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime() | 81 return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime() |
| 78 | 82 |
| 79 | 83 |
| 84 def GetBuildBotUrl(builder_type=PERF_BUILDER, target_arch='ia32', |
| 85 target_platform='chromium', extra_src=None): |
| 86 """Gets buildbot URL for a given builder type.""" |
| 87 logging.info('Getting buildbot URL for "%s", "%s", "%s".', |
| 88 builder_type, target_arch, target_platform) |
| 89 build_archive = BuildArchive.Create( |
| 90 builder_type, target_arch=target_arch, target_platform=target_platform, |
| 91 extra_src=extra_src) |
| 92 return build_archive.GetBuildBotUrl() |
| 93 |
| 94 |
| 80 class BuildArchive(object): | 95 class BuildArchive(object): |
| 81 """Represents a place where builds of some type are stored. | 96 """Represents a place where builds of some type are stored. |
| 82 | 97 |
| 83 There are two pieces of information required to locate a file in Google | 98 There are two pieces of information required to locate a file in Google |
| 84 Cloud Storage, bucket name and file path. Subclasses of this class contain | 99 Cloud Storage, bucket name and file path. Subclasses of this class contain |
| 85 specific logic about which bucket names and paths should be used to fetch | 100 specific logic about which bucket names and paths should be used to fetch |
| 86 a build. | 101 a build. |
| 87 """ | 102 """ |
| 88 | 103 |
| 89 @staticmethod | 104 @staticmethod |
| (...skipping 12 matching lines...) Expand all Loading... |
| 102 target_arch, target_platform, loaded_extra_src) | 117 target_arch, target_platform, loaded_extra_src) |
| 103 except (IOError, TypeError, ImportError): | 118 except (IOError, TypeError, ImportError): |
| 104 raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src) | 119 raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src) |
| 105 raise NotImplementedError('Builder type "%s" not supported.' % builder_type) | 120 raise NotImplementedError('Builder type "%s" not supported.' % builder_type) |
| 106 | 121 |
| 107 def __init__(self, target_arch='ia32', target_platform='chromium', | 122 def __init__(self, target_arch='ia32', target_platform='chromium', |
| 108 extra_src=None): | 123 extra_src=None): |
| 109 self._extra_src = extra_src | 124 self._extra_src = extra_src |
| 110 if bisect_utils.IsLinuxHost() and target_platform == 'android': | 125 if bisect_utils.IsLinuxHost() and target_platform == 'android': |
| 111 self._platform = 'android' | 126 self._platform = 'android' |
| 127 elif bisect_utils.IsLinuxHost() and target_platform == 'android-chrome': |
| 128 self._platform = 'android-chrome' |
| 112 elif bisect_utils.IsLinuxHost(): | 129 elif bisect_utils.IsLinuxHost(): |
| 113 self._platform = 'linux' | 130 self._platform = 'linux' |
| 114 elif bisect_utils.IsMacHost(): | 131 elif bisect_utils.IsMacHost(): |
| 115 self._platform = 'mac' | 132 self._platform = 'mac' |
| 116 elif bisect_utils.Is64BitWindows() and target_arch == 'x64': | 133 elif bisect_utils.Is64BitWindows() and target_arch == 'x64': |
| 117 self._platform = 'win64' | 134 self._platform = 'win64' |
| 118 elif bisect_utils.IsWindowsHost(): | 135 elif bisect_utils.IsWindowsHost(): |
| 119 self._platform = 'win' | 136 self._platform = 'win' |
| 120 else: | 137 else: |
| 121 raise NotImplementedError('Unknown platform "%s".' % sys.platform) | 138 raise NotImplementedError('Unknown platform "%s".' % sys.platform) |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 168 return 'mac' | 185 return 'mac' |
| 169 raise NotImplementedError('Unknown platform "%s".' % sys.platform) | 186 raise NotImplementedError('Unknown platform "%s".' % sys.platform) |
| 170 | 187 |
| 171 def GetBuilderName(self): | 188 def GetBuilderName(self): |
| 172 raise NotImplementedError() | 189 raise NotImplementedError() |
| 173 | 190 |
| 174 def GetBuilderBuildTime(self): | 191 def GetBuilderBuildTime(self): |
| 175 """Returns the time to wait for a build after requesting one.""" | 192 """Returns the time to wait for a build after requesting one.""" |
| 176 if self._platform in ('win', 'win64'): | 193 if self._platform in ('win', 'win64'): |
| 177 return MAX_WIN_BUILD_TIME | 194 return MAX_WIN_BUILD_TIME |
| 178 if self._platform in ('linux', 'android'): | 195 if self._platform in ('linux', 'android', 'android-chrome'): |
| 179 return MAX_LINUX_BUILD_TIME | 196 return MAX_LINUX_BUILD_TIME |
| 180 if self._platform == 'mac': | 197 if self._platform == 'mac': |
| 181 return MAX_MAC_BUILD_TIME | 198 return MAX_MAC_BUILD_TIME |
| 182 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) | 199 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) |
| 183 | 200 |
| 201 def GetBuildBotUrl(self): |
| 202 raise NotImplementedError() |
| 203 |
| 184 | 204 |
| 185 class PerfBuildArchive(BuildArchive): | 205 class PerfBuildArchive(BuildArchive): |
| 186 | 206 |
| 187 def BucketName(self): | 207 def BucketName(self): |
| 188 return 'chrome-perf' | 208 return 'chrome-perf' |
| 189 | 209 |
| 190 def FilePath(self, revision, deps_patch_sha=None): | 210 def FilePath(self, revision, deps_patch_sha=None): |
| 191 return '%s/%s' % (self._ArchiveDirectory(), | 211 return '%s/%s' % (self._ArchiveDirectory(), |
| 192 self._ZipFileName(revision, deps_patch_sha)) | 212 self._ZipFileName(revision, deps_patch_sha)) |
| 193 | 213 |
| (...skipping 16 matching lines...) Expand all Loading... |
| 210 elif self._platform == 'win': | 230 elif self._platform == 'win': |
| 211 return 'win_perf_bisect_builder' | 231 return 'win_perf_bisect_builder' |
| 212 elif self._platform == 'linux': | 232 elif self._platform == 'linux': |
| 213 return 'linux_perf_bisect_builder' | 233 return 'linux_perf_bisect_builder' |
| 214 elif self._platform == 'android': | 234 elif self._platform == 'android': |
| 215 return 'android_perf_bisect_builder' | 235 return 'android_perf_bisect_builder' |
| 216 elif self._platform == 'mac': | 236 elif self._platform == 'mac': |
| 217 return 'mac_perf_bisect_builder' | 237 return 'mac_perf_bisect_builder' |
| 218 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) | 238 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) |
| 219 | 239 |
| 240 def GetBuildBotUrl(self): |
| 241 """Returns buildbot URL for fetching build info.""" |
| 242 return PERF_TRY_SERVER_URL |
| 243 |
| 220 | 244 |
| 221 class FullBuildArchive(BuildArchive): | 245 class FullBuildArchive(BuildArchive): |
| 222 | 246 |
| 223 def BucketName(self): | 247 def BucketName(self): |
| 224 platform_to_bucket = { | 248 platform_to_bucket = { |
| 225 'android': 'chromium-android', | 249 'android': 'chromium-android', |
| 226 'linux': 'chromium-linux-archive', | 250 'linux': 'chromium-linux-archive', |
| 227 'mac': 'chromium-mac-archive', | 251 'mac': 'chromium-mac-archive', |
| 228 'win64': 'chromium-win-archive', | 252 'win64': 'chromium-win-archive', |
| 229 'win': 'chromium-win-archive', | 253 'win': 'chromium-win-archive', |
| (...skipping 16 matching lines...) Expand all Loading... |
| 246 } | 270 } |
| 247 assert self._platform in platform_to_directory | 271 assert self._platform in platform_to_directory |
| 248 return platform_to_directory.get(self._platform) | 272 return platform_to_directory.get(self._platform) |
| 249 | 273 |
| 250 def GetBuilderName(self): | 274 def GetBuilderName(self): |
| 251 """Gets builder bot name based on platform.""" | 275 """Gets builder bot name based on platform.""" |
| 252 if self._platform == 'linux': | 276 if self._platform == 'linux': |
| 253 return 'linux_full_bisect_builder' | 277 return 'linux_full_bisect_builder' |
| 254 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) | 278 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) |
| 255 | 279 |
| 280 def GetBuildBotUrl(self): |
| 281 """Returns buildbot URL for fetching build info.""" |
| 282 return LINUX_TRY_SERVER_URL |
| 283 |
| 284 |
| 256 class AndroidChromeBuildArchive(BuildArchive): | 285 class AndroidChromeBuildArchive(BuildArchive): |
| 257 """Represents a place where builds of android-chrome type are stored. | 286 """Represents a place where builds of android-chrome type are stored. |
| 258 | 287 |
| 259 If AndroidChromeBuildArchive is used, it is assumed that the --extra_src | 288 If AndroidChromeBuildArchive is used, it is assumed that the --extra_src |
| 260 is a valid Python module which contains the module-level functions | 289 is a valid Python module which contains the module-level functions |
| 261 GetBucketName and GetArchiveDirectory. | 290 GetBucketName and GetArchiveDirectory. |
| 262 """ | 291 """ |
| 263 | 292 |
| 264 def BucketName(self): | 293 def BucketName(self): |
| 265 return self._extra_src.GetBucketName() | 294 return self._extra_src.GetBucketName() |
| (...skipping 20 matching lines...) Expand all Loading... |
| 286 self._ZipFileName(revision, deps_patch_sha)) | 315 self._ZipFileName(revision, deps_patch_sha)) |
| 287 | 316 |
| 288 def _ArchiveDirectory(self): | 317 def _ArchiveDirectory(self): |
| 289 """Returns the directory name to download builds from.""" | 318 """Returns the directory name to download builds from.""" |
| 290 return self._extra_src.GetArchiveDirectory() | 319 return self._extra_src.GetArchiveDirectory() |
| 291 | 320 |
| 292 def GetBuilderName(self): | 321 def GetBuilderName(self): |
| 293 """Returns the builder name extra source.""" | 322 """Returns the builder name extra source.""" |
| 294 return self._extra_src.GetBuilderName() | 323 return self._extra_src.GetBuilderName() |
| 295 | 324 |
| 325 def GetBuildBotUrl(self): |
| 326 """Returns buildbot URL for fetching build info.""" |
| 327 return self._extra_src.GetBuildBotUrl() |
| 328 |
| 296 | 329 |
| 297 def BuildIsAvailable(bucket_name, remote_path): | 330 def BuildIsAvailable(bucket_name, remote_path): |
| 298 """Checks whether a build is currently archived at some place.""" | 331 """Checks whether a build is currently archived at some place.""" |
| 299 logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path)) | 332 logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path)) |
| 300 try: | 333 try: |
| 301 exists = cloud_storage.Exists(bucket_name, remote_path) | 334 exists = cloud_storage.Exists(bucket_name, remote_path) |
| 302 logging.info('Exists? %s' % exists) | 335 logging.info('Exists? %s' % exists) |
| 303 return exists | 336 return exists |
| 304 except cloud_storage.CloudStorageError: | 337 except cloud_storage.CloudStorageError: |
| 305 return False | 338 return False |
| (...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 461 print 'Build is not available.' | 494 print 'Build is not available.' |
| 462 return 1 | 495 return 1 |
| 463 | 496 |
| 464 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir) | 497 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir) |
| 465 print 'Build has been downloaded to and extracted in %s.' % args.output_dir | 498 print 'Build has been downloaded to and extracted in %s.' % args.output_dir |
| 466 return 0 | 499 return 0 |
| 467 | 500 |
| 468 | 501 |
| 469 if __name__ == '__main__': | 502 if __name__ == '__main__': |
| 470 sys.exit(Main(sys.argv)) | 503 sys.exit(Main(sys.argv)) |
| OLD | NEW |