OLD | NEW |
---|---|
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """This module contains functions for fetching and extracting archived builds. | 5 """This module contains functions for fetching and extracting archived builds. |
6 | 6 |
7 The builds may be stored in different places by different types of builders; | 7 The builds may be stored in different places by different types of builders; |
8 for example, builders on tryserver.chromium.perf stores builds in one place, | 8 for example, builders on tryserver.chromium.perf stores builds in one place, |
9 while builders on chromium.linux store builds in another. | 9 while builders on chromium.linux store builds in another. |
10 | 10 |
(...skipping 12 matching lines...) Expand all Loading... | |
23 import zipfile | 23 import zipfile |
24 | 24 |
25 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH. | 25 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH. |
26 from telemetry.util import cloud_storage | 26 from telemetry.util import cloud_storage |
27 | 27 |
28 import bisect_utils | 28 import bisect_utils |
29 | 29 |
30 # Possible builder types. | 30 # Possible builder types. |
31 PERF_BUILDER = 'perf' | 31 PERF_BUILDER = 'perf' |
32 FULL_BUILDER = 'full' | 32 FULL_BUILDER = 'full' |
33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf' | |
33 | 34 |
34 | 35 |
35 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER, | 36 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER, |
36 target_arch='ia32', target_platform='chromium', | 37 target_arch='ia32', target_platform='chromium', |
37 deps_patch_sha=None): | 38 deps_patch_sha=None, extra_src=None): |
38 """Returns the location where a build archive is expected to be. | 39 """Returns the location where a build archive is expected to be. |
39 | 40 |
40 Args: | 41 Args: |
41 revision: Revision string, e.g. a git commit hash or SVN revision. | 42 revision: Revision string, e.g. a git commit hash or SVN revision. |
42 builder_type: Type of build archive. | 43 builder_type: Type of build archive. |
43 target_arch: Architecture, e.g. "ia32". | 44 target_arch: Architecture, e.g. "ia32". |
44 target_platform: Platform name, e.g. "chromium" or "android". | 45 target_platform: Platform name, e.g. "chromium" or "android". |
45 deps_patch_sha: SHA1 hash which identifies a particular combination of | 46 deps_patch_sha: SHA1 hash which identifies a particular combination of |
46 custom revisions for dependency repositories. | 47 custom revisions for dependency repositories. |
48 extra_src: Path to a script which can be used to modify the bisect script's | |
49 behavior. | |
47 | 50 |
48 Returns: | 51 Returns: |
49 A pair of strings (bucket, path), where the archive is expected to be. | 52 A pair of strings (bucket, path), where the archive is expected to be. |
50 """ | 53 """ |
51 build_archive = BuildArchive.Create( | 54 build_archive = BuildArchive.Create( |
52 builder_type, target_arch=target_arch, target_platform=target_platform) | 55 builder_type, target_arch=target_arch, target_platform=target_platform, |
56 extra_src=extra_src) | |
53 bucket = build_archive.BucketName() | 57 bucket = build_archive.BucketName() |
54 remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha) | 58 remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha) |
55 return bucket, remote_path | 59 return bucket, remote_path |
56 | 60 |
57 | 61 |
58 class BuildArchive(object): | 62 class BuildArchive(object): |
59 """Represents a place where builds of some type are stored. | 63 """Represents a place where builds of some type are stored. |
60 | 64 |
61 There are two pieces of information required to locate a file in Google | 65 There are two pieces of information required to locate a file in Google |
62 Cloud Storage, bucket name and file path. Subclasses of this class contain | 66 Cloud Storage, bucket name and file path. Subclasses of this class contain |
63 specific logic about which bucket names and paths should be used to fetch | 67 specific logic about which bucket names and paths should be used to fetch |
64 a build. | 68 a build. |
65 """ | 69 """ |
66 | 70 |
67 @staticmethod | 71 @staticmethod |
68 def Create(builder_type, target_arch='ia32', target_platform='chromium'): | 72 def Create(builder_type, target_arch='ia32', target_platform='chromium', |
73 extra_src=None): | |
69 if builder_type == PERF_BUILDER: | 74 if builder_type == PERF_BUILDER: |
70 return PerfBuildArchive(target_arch, target_platform) | 75 return PerfBuildArchive(target_arch, target_platform) |
71 if builder_type == FULL_BUILDER: | 76 if builder_type == FULL_BUILDER: |
72 return FullBuildArchive(target_arch, target_platform) | 77 return FullBuildArchive(target_arch, target_platform) |
78 if builder_type == ANDROID_CHROME_PERF_BUILDER: | |
79 try: | |
80 extra_src = bisect_utils.LoadExtraSrc(extra_src) | |
RobertoCN
2015/01/21 23:15:19
I think it would be more readable if we used an ad
prasadv
2015/01/23 18:05:57
Done.
| |
81 return AndroidChromeBuildArchive( | |
82 target_arch, target_platform, extra_src) | |
83 except (IOError, TypeError, ImportError): | |
84 raise RuntimeError('Invalid or missing --extra_src.') | |
qyearsley
2015/01/26 17:50:32
[Optional] In the case of a failure, it may be use
prasadv
2015/01/26 18:58:26
Done.
| |
73 raise NotImplementedError('Builder type "%s" not supported.' % builder_type) | 85 raise NotImplementedError('Builder type "%s" not supported.' % builder_type) |
74 | 86 |
75 def __init__(self, target_arch='ia32', target_platform='chromium'): | 87 def __init__(self, target_arch='ia32', target_platform='chromium', |
88 extra_src=None): | |
89 self._extra_src = extra_src | |
76 if bisect_utils.IsLinuxHost() and target_platform == 'android': | 90 if bisect_utils.IsLinuxHost() and target_platform == 'android': |
77 self._platform = 'android' | 91 self._platform = 'android' |
78 elif bisect_utils.IsLinuxHost(): | 92 elif bisect_utils.IsLinuxHost(): |
79 self._platform = 'linux' | 93 self._platform = 'linux' |
80 elif bisect_utils.IsMacHost(): | 94 elif bisect_utils.IsMacHost(): |
81 self._platform = 'mac' | 95 self._platform = 'mac' |
82 elif bisect_utils.Is64BitWindows() and target_arch == 'x64': | 96 elif bisect_utils.Is64BitWindows() and target_arch == 'x64': |
83 self._platform = 'win64' | 97 self._platform = 'win64' |
84 elif bisect_utils.IsWindowsHost(): | 98 elif bisect_utils.IsWindowsHost(): |
85 self._platform = 'win' | 99 self._platform = 'win' |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
180 'android': 'android_main_rel', | 194 'android': 'android_main_rel', |
181 'linux': 'chromium.linux/Linux Builder', | 195 'linux': 'chromium.linux/Linux Builder', |
182 'mac': 'chromium.mac/Mac Builder', | 196 'mac': 'chromium.mac/Mac Builder', |
183 'win64': 'chromium.win/Win x64 Builder', | 197 'win64': 'chromium.win/Win x64 Builder', |
184 'win': 'chromium.win/Win Builder', | 198 'win': 'chromium.win/Win Builder', |
185 } | 199 } |
186 assert self._platform in platform_to_directory | 200 assert self._platform in platform_to_directory |
187 return platform_to_directory.get(self._platform) | 201 return platform_to_directory.get(self._platform) |
188 | 202 |
189 | 203 |
204 class AndroidChromeBuildArchive(BuildArchive): | |
qyearsley
2015/01/26 17:50:32
[Optional] It can be seen from the code below, but
prasadv
2015/01/26 18:58:26
Done.
| |
205 | |
206 def BucketName(self): | |
207 return self._extra_src.GetBucketName() | |
208 | |
209 def _ZipFileName(self, revision, deps_patch_sha=None): | |
210 """Gets the file name of a zip archive on android-chrome. | |
211 | |
212 This returns a file name of the form build_product_<revision>.zip, | |
213 which is a format used by android-chrome. | |
214 | |
215 Args: | |
216 revision: A git commit hash or other revision string. | |
217 deps_patch_sha: SHA1 hash of a DEPS file patch. | |
218 | |
219 Returns: | |
220 The archive file name. | |
221 """ | |
222 if deps_patch_sha: | |
223 revision = '%s_%s' % (revision, deps_patch_sha) | |
224 return 'build_product_%s.zip' % revision | |
225 | |
226 def FilePath(self, revision, deps_patch_sha=None): | |
227 return '%s/%s' % (self._ArchiveDirectory(), | |
228 self._ZipFileName(revision, deps_patch_sha)) | |
229 | |
230 def _ArchiveDirectory(self): | |
231 """Returns the directory name to download builds from.""" | |
232 return self._extra_src.GetArchiveDirectory() | |
233 | |
234 | |
190 def BuildIsAvailable(bucket_name, remote_path): | 235 def BuildIsAvailable(bucket_name, remote_path): |
191 """Checks whether a build is currently archived at some place.""" | 236 """Checks whether a build is currently archived at some place.""" |
192 logging.info('Checking existance: gs://%s/%s' % (bucket_name, remote_path)) | 237 logging.info('Checking existance: gs://%s/%s' % (bucket_name, remote_path)) |
193 try: | 238 try: |
194 exists = cloud_storage.Exists(bucket_name, remote_path) | 239 exists = cloud_storage.Exists(bucket_name, remote_path) |
195 logging.info('Exists? %s' % exists) | 240 logging.info('Exists? %s' % exists) |
196 return exists | 241 return exists |
197 except cloud_storage.CloudStorageError: | 242 except cloud_storage.CloudStorageError: |
198 return False | 243 return False |
199 | 244 |
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
355 return 1 | 400 return 1 |
356 | 401 |
357 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir) | 402 FetchFromCloudStorage(bucket_name, remote_path, args.output_dir) |
358 print 'Build has been downloaded to and extracted in %s.' % args.output_dir | 403 print 'Build has been downloaded to and extracted in %s.' % args.output_dir |
359 return 0 | 404 return 0 |
360 | 405 |
361 | 406 |
362 if __name__ == '__main__': | 407 if __name__ == '__main__': |
363 sys.exit(Main(sys.argv)) | 408 sys.exit(Main(sys.argv)) |
364 | 409 |
OLD | NEW |