Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(45)

Side by Side Diff: tools/auto_bisect/fetch_build.py

Issue 994183002: Make android-chrome bisect work with archives. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """This module contains functions for fetching and extracting archived builds. 5 """This module contains functions for fetching and extracting archived builds.
6 6
7 The builds may be stored in different places by different types of builders; 7 The builds may be stored in different places by different types of builders;
8 for example, builders on tryserver.chromium.perf stores builds in one place, 8 for example, builders on tryserver.chromium.perf stores builds in one place,
9 while builders on chromium.linux store builds in another. 9 while builders on chromium.linux store builds in another.
10 10
(...skipping 19 matching lines...) Expand all
30 # Possible builder types. 30 # Possible builder types.
31 PERF_BUILDER = 'perf' 31 PERF_BUILDER = 'perf'
32 FULL_BUILDER = 'full' 32 FULL_BUILDER = 'full'
33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf' 33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf'
34 34
35 # Maximum time in seconds to wait after posting build request to the try server. 35 # Maximum time in seconds to wait after posting build request to the try server.
36 MAX_MAC_BUILD_TIME = 14400 36 MAX_MAC_BUILD_TIME = 14400
37 MAX_WIN_BUILD_TIME = 14400 37 MAX_WIN_BUILD_TIME = 14400
38 MAX_LINUX_BUILD_TIME = 14400 38 MAX_LINUX_BUILD_TIME = 14400
39 39
40 # Try server status page URLs, used to get build status.
41 PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf'
42 LINUX_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.linux'
43
40 44
41 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER, 45 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER,
42 target_arch='ia32', target_platform='chromium', 46 target_arch='ia32', target_platform='chromium',
43 deps_patch_sha=None, extra_src=None): 47 deps_patch_sha=None, extra_src=None):
44 """Returns the location where a build archive is expected to be. 48 """Returns the location where a build archive is expected to be.
45 49
46 Args: 50 Args:
47 revision: Revision string, e.g. a git commit hash or SVN revision. 51 revision: Revision string, e.g. a git commit hash or SVN revision.
48 builder_type: Type of build archive. 52 builder_type: Type of build archive.
49 target_arch: Architecture, e.g. "ia32". 53 target_arch: Architecture, e.g. "ia32".
(...skipping 20 matching lines...) Expand all
70 target_platform='chromium', extra_src=None): 74 target_platform='chromium', extra_src=None):
71 """Gets builder bot name and build time in seconds based on platform.""" 75 """Gets builder bot name and build time in seconds based on platform."""
72 logging.info('Getting builder name for builder "%s", "%s", "%s".', 76 logging.info('Getting builder name for builder "%s", "%s", "%s".',
73 builder_type, target_arch, target_platform) 77 builder_type, target_arch, target_platform)
74 build_archive = BuildArchive.Create( 78 build_archive = BuildArchive.Create(
75 builder_type, target_arch=target_arch, target_platform=target_platform, 79 builder_type, target_arch=target_arch, target_platform=target_platform,
76 extra_src=extra_src) 80 extra_src=extra_src)
77 return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime() 81 return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime()
78 82
79 83
84 def GetBuildBotUrl(builder_type=PERF_BUILDER, target_arch='ia32',
85 target_platform='chromium', extra_src=None):
86 """Gets buildbot URL for a given builder type."""
87 logging.info('Getting buildbot URL for "%s", "%s", "%s".',
88 builder_type, target_arch, target_platform)
89 build_archive = BuildArchive.Create(
90 builder_type, target_arch=target_arch, target_platform=target_platform,
91 extra_src=extra_src)
92 return build_archive.GetBuildBotUrl()
93
94
80 class BuildArchive(object): 95 class BuildArchive(object):
81 """Represents a place where builds of some type are stored. 96 """Represents a place where builds of some type are stored.
82 97
83 There are two pieces of information required to locate a file in Google 98 There are two pieces of information required to locate a file in Google
84 Cloud Storage, bucket name and file path. Subclasses of this class contain 99 Cloud Storage, bucket name and file path. Subclasses of this class contain
85 specific logic about which bucket names and paths should be used to fetch 100 specific logic about which bucket names and paths should be used to fetch
86 a build. 101 a build.
87 """ 102 """
88 103
89 @staticmethod 104 @staticmethod
(...skipping 12 matching lines...) Expand all
102 target_arch, target_platform, loaded_extra_src) 117 target_arch, target_platform, loaded_extra_src)
103 except (IOError, TypeError, ImportError): 118 except (IOError, TypeError, ImportError):
104 raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src) 119 raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src)
105 raise NotImplementedError('Builder type "%s" not supported.' % builder_type) 120 raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
106 121
107 def __init__(self, target_arch='ia32', target_platform='chromium', 122 def __init__(self, target_arch='ia32', target_platform='chromium',
108 extra_src=None): 123 extra_src=None):
109 self._extra_src = extra_src 124 self._extra_src = extra_src
110 if bisect_utils.IsLinuxHost() and target_platform == 'android': 125 if bisect_utils.IsLinuxHost() and target_platform == 'android':
111 self._platform = 'android' 126 self._platform = 'android'
127 elif bisect_utils.IsLinuxHost() and target_platform == 'android-chrome':
128 self._platform = 'android-chrome'
112 elif bisect_utils.IsLinuxHost(): 129 elif bisect_utils.IsLinuxHost():
113 self._platform = 'linux' 130 self._platform = 'linux'
114 elif bisect_utils.IsMacHost(): 131 elif bisect_utils.IsMacHost():
115 self._platform = 'mac' 132 self._platform = 'mac'
116 elif bisect_utils.Is64BitWindows() and target_arch == 'x64': 133 elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
117 self._platform = 'win64' 134 self._platform = 'win64'
118 elif bisect_utils.IsWindowsHost(): 135 elif bisect_utils.IsWindowsHost():
119 self._platform = 'win' 136 self._platform = 'win'
120 else: 137 else:
121 raise NotImplementedError('Unknown platform "%s".' % sys.platform) 138 raise NotImplementedError('Unknown platform "%s".' % sys.platform)
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after
168 return 'mac' 185 return 'mac'
169 raise NotImplementedError('Unknown platform "%s".' % sys.platform) 186 raise NotImplementedError('Unknown platform "%s".' % sys.platform)
170 187
171 def GetBuilderName(self): 188 def GetBuilderName(self):
172 raise NotImplementedError() 189 raise NotImplementedError()
173 190
174 def GetBuilderBuildTime(self): 191 def GetBuilderBuildTime(self):
175 """Returns the time to wait for a build after requesting one.""" 192 """Returns the time to wait for a build after requesting one."""
176 if self._platform in ('win', 'win64'): 193 if self._platform in ('win', 'win64'):
177 return MAX_WIN_BUILD_TIME 194 return MAX_WIN_BUILD_TIME
178 if self._platform in ('linux', 'android'): 195 if self._platform in ('linux', 'android', 'android-chrome'):
179 return MAX_LINUX_BUILD_TIME 196 return MAX_LINUX_BUILD_TIME
180 if self._platform == 'mac': 197 if self._platform == 'mac':
181 return MAX_MAC_BUILD_TIME 198 return MAX_MAC_BUILD_TIME
182 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) 199 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
183 200
201 def GetBuildBotUrl(self):
202 raise NotImplementedError()
203
184 204
185 class PerfBuildArchive(BuildArchive): 205 class PerfBuildArchive(BuildArchive):
186 206
187 def BucketName(self): 207 def BucketName(self):
188 return 'chrome-perf' 208 return 'chrome-perf'
189 209
190 def FilePath(self, revision, deps_patch_sha=None): 210 def FilePath(self, revision, deps_patch_sha=None):
191 return '%s/%s' % (self._ArchiveDirectory(), 211 return '%s/%s' % (self._ArchiveDirectory(),
192 self._ZipFileName(revision, deps_patch_sha)) 212 self._ZipFileName(revision, deps_patch_sha))
193 213
(...skipping 16 matching lines...) Expand all
210 elif self._platform == 'win': 230 elif self._platform == 'win':
211 return 'win_perf_bisect_builder' 231 return 'win_perf_bisect_builder'
212 elif self._platform == 'linux': 232 elif self._platform == 'linux':
213 return 'linux_perf_bisect_builder' 233 return 'linux_perf_bisect_builder'
214 elif self._platform == 'android': 234 elif self._platform == 'android':
215 return 'android_perf_bisect_builder' 235 return 'android_perf_bisect_builder'
216 elif self._platform == 'mac': 236 elif self._platform == 'mac':
217 return 'mac_perf_bisect_builder' 237 return 'mac_perf_bisect_builder'
218 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) 238 raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
219 239
240 def GetBuildBotUrl(self):
241 """Returns buildbot URL for fetching build info."""
242 return PERF_TRY_SERVER_URL
243
220 244
221 class FullBuildArchive(BuildArchive): 245 class FullBuildArchive(BuildArchive):
222 246
223 def BucketName(self): 247 def BucketName(self):
224 platform_to_bucket = { 248 platform_to_bucket = {
225 'android': 'chromium-android', 249 'android': 'chromium-android',
226 'linux': 'chromium-linux-archive', 250 'linux': 'chromium-linux-archive',
227 'mac': 'chromium-mac-archive', 251 'mac': 'chromium-mac-archive',
228 'win64': 'chromium-win-archive', 252 'win64': 'chromium-win-archive',
229 'win': 'chromium-win-archive', 253 'win': 'chromium-win-archive',
(...skipping 16 matching lines...) Expand all
246 } 270 }
247 assert self._platform in platform_to_directory 271 assert self._platform in platform_to_directory
248 return platform_to_directory.get(self._platform) 272 return platform_to_directory.get(self._platform)
249 273
250 def GetBuilderName(self): 274 def GetBuilderName(self):
251 """Gets builder bot name based on platform.""" 275 """Gets builder bot name based on platform."""
252 if self._platform == 'linux': 276 if self._platform == 'linux':
253 return 'linux_full_bisect_builder' 277 return 'linux_full_bisect_builder'
254 raise NotImplementedError('Unsupported platform "%s".' % sys.platform) 278 raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
255 279
280 def GetBuildBotUrl(self):
281 """Returns buildbot URL for fetching build info."""
282 return LINUX_TRY_SERVER_URL
283
284
256 class AndroidChromeBuildArchive(BuildArchive): 285 class AndroidChromeBuildArchive(BuildArchive):
257 """Represents a place where builds of android-chrome type are stored. 286 """Represents a place where builds of android-chrome type are stored.
258 287
259 If AndroidChromeBuildArchive is used, it is assumed that the --extra_src 288 If AndroidChromeBuildArchive is used, it is assumed that the --extra_src
260 is a valid Python module which contains the module-level functions 289 is a valid Python module which contains the module-level functions
261 GetBucketName and GetArchiveDirectory. 290 GetBucketName and GetArchiveDirectory.
262 """ 291 """
263 292
264 def BucketName(self): 293 def BucketName(self):
265 return self._extra_src.GetBucketName() 294 return self._extra_src.GetBucketName()
(...skipping 20 matching lines...) Expand all
286 self._ZipFileName(revision, deps_patch_sha)) 315 self._ZipFileName(revision, deps_patch_sha))
287 316
288 def _ArchiveDirectory(self): 317 def _ArchiveDirectory(self):
289 """Returns the directory name to download builds from.""" 318 """Returns the directory name to download builds from."""
290 return self._extra_src.GetArchiveDirectory() 319 return self._extra_src.GetArchiveDirectory()
291 320
292 def GetBuilderName(self): 321 def GetBuilderName(self):
293 """Returns the builder name extra source.""" 322 """Returns the builder name extra source."""
294 return self._extra_src.GetBuilderName() 323 return self._extra_src.GetBuilderName()
295 324
325 def GetBuildBotUrl(self):
326 """Returns buildbot URL for fetching build info."""
327 return self._extra_src.GetBuildBotUrl()
328
296 329
297 def BuildIsAvailable(bucket_name, remote_path): 330 def BuildIsAvailable(bucket_name, remote_path):
298 """Checks whether a build is currently archived at some place.""" 331 """Checks whether a build is currently archived at some place."""
299 logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path)) 332 logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path))
300 try: 333 try:
301 exists = cloud_storage.Exists(bucket_name, remote_path) 334 exists = cloud_storage.Exists(bucket_name, remote_path)
302 logging.info('Exists? %s' % exists) 335 logging.info('Exists? %s' % exists)
303 return exists 336 return exists
304 except cloud_storage.CloudStorageError: 337 except cloud_storage.CloudStorageError:
305 return False 338 return False
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after
461 print 'Build is not available.' 494 print 'Build is not available.'
462 return 1 495 return 1
463 496
464 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir) 497 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir)
465 print 'Build has been downloaded to and extracted in %s.' % args.output_dir 498 print 'Build has been downloaded to and extracted in %s.' % args.output_dir
466 return 0 499 return 0
467 500
468 501
469 if __name__ == '__main__': 502 if __name__ == '__main__':
470 sys.exit(Main(sys.argv)) 503 sys.exit(Main(sys.argv))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698