| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2013 The Chromium Authors. All rights reserved. | |
| 4 # Use of this source code is governed by a BSD-style license that can be | |
| 5 # found in the LICENSE file. | |
| 6 | |
| 7 """Updates the Chrome reference builds. | |
| 8 | |
| 9 Before running this script, you should first verify that you are authenticated | |
| 10 for SVN. You can do this by running: | |
| 11 $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds | |
| 12 You may need to get your SVN password from https://chromium-access.appspot.com/. | |
| 13 | |
| 14 Usage: | |
| 15 $ cd /tmp | |
| 16 $ /path/to/update_reference_build.py | |
| 17 $ cd reference_builds/reference_builds | |
| 18 $ gcl change | |
| 19 $ gcl upload <change> | |
| 20 $ gcl commit <change> | |
| 21 """ | |
| 22 | |
| 23 import collections | |
| 24 import logging | |
| 25 import os | |
| 26 import shutil | |
| 27 import subprocess | |
| 28 import sys | |
| 29 import urllib2 | |
| 30 import zipfile | |
| 31 | |
| 32 | |
| 33 # Google storage location (no public web URL's), example: | |
| 34 # gs://chrome-unsigned/desktop-*/30.0.1595.0/precise32/chrome-precise32.zip | |
| 35 CHROME_GS_URL_FMT = ('gs://chrome-unsigned/desktop-*/%s/%s/%s') | |
| 36 | |
| 37 | |
| 38 def _ReportValueError(error_string): | |
| 39 # TODO(aiolos): alert sheriffs via email when an error is seen. | |
| 40 # This should be added when alerts are added when updating the build. | |
| 41 raise ValueError(error_string) | |
| 42 | |
| 43 | |
| 44 class BuildUpdater(object): | |
| 45 # Remove a platform name from this list to disable updating it. | |
| 46 _REF_BUILD_PLATFORMS = ['Mac64', 'Win', 'Linux', 'Linux_x64'] | |
| 47 | |
| 48 # Omaha is Chrome's autoupdate server. It reports the current versions used | |
| 49 # by each platform on each channel. | |
| 50 _OMAHA_PLATFORMS = ['mac', 'linux', 'win'] | |
| 51 | |
| 52 # All of the information we need to update each platform. | |
| 53 # omaha: name omaha uses for the platforms. | |
| 54 # zip_name: name of the zip file to be retrieved from cloud storage. | |
| 55 # gs_build: name of the Chrome build platform used in cloud storage. | |
| 56 # destination: Name of the folder to download the reference build to. | |
| 57 UpdateInfo = collections.namedtuple('UpdateInfo', | |
| 58 'omaha, gs_build, zip_name, destination') | |
| 59 _PLATFORM_MAP = {'Mac64': UpdateInfo(omaha='mac', | |
| 60 gs_build='mac64', | |
| 61 zip_name='chrome-mac.zip', | |
| 62 destination='chrome_mac'), | |
| 63 'Win': UpdateInfo(omaha='win', | |
| 64 gs_build='win', | |
| 65 zip_name='chrome-win.zip', | |
| 66 destination='chrome_win'), | |
| 67 'Linux': UpdateInfo(omaha='linux', | |
| 68 gs_build='precise32', | |
| 69 zip_name='chrome-precise32.zip', | |
| 70 destination='chrome_linux'), | |
| 71 'Linux_x64': UpdateInfo(omaha='linux', | |
| 72 gs_build='precise64', | |
| 73 zip_name='chrome-precise64.zip', | |
| 74 destination='chrome_linux64')} | |
| 75 | |
| 76 def __init__(self): | |
| 77 stable_versions = self._StableVersionsMap() | |
| 78 current_versions = self._CurrentRefBuildsMap() | |
| 79 self._platform_to_version_map = {} | |
| 80 for platform in stable_versions: | |
| 81 if (platform not in current_versions or | |
| 82 stable_versions[platform] != current_versions[platform]): | |
| 83 self._platform_to_version_map[platform] = stable_versions[platform] | |
| 84 | |
| 85 @classmethod | |
| 86 def _StableVersionsMap(cls): | |
| 87 omaha_versions_map = cls._OmahaVersionsMap() | |
| 88 versions_map = {} | |
| 89 for platform in cls._REF_BUILD_PLATFORMS: | |
| 90 omaha_platform = cls._PLATFORM_MAP[platform].omaha | |
| 91 if omaha_platform in omaha_versions_map: | |
| 92 versions_map[platform] = omaha_versions_map[omaha_platform] | |
| 93 return versions_map | |
| 94 | |
| 95 @classmethod | |
| 96 def _OmahaReport(cls): | |
| 97 url = 'https://omahaproxy.appspot.com/all?channel=stable' | |
| 98 lines = urllib2.urlopen(url).readlines() | |
| 99 return [l.split(',') for l in lines] | |
| 100 | |
| 101 @classmethod | |
| 102 def _OmahaVersionsMap(cls): | |
| 103 platforms = cls._OMAHA_PLATFORMS | |
| 104 rows = cls._OmahaReport() | |
| 105 if (len(rows) < 1 or | |
| 106 not rows[0][0:3] == ['os', 'channel', 'current_version']): | |
| 107 _ReportValueError('Omaha report is not in the expected form: %s.' | |
| 108 % rows) | |
| 109 versions_map = {} | |
| 110 for row in rows[1:]: | |
| 111 if row[1] != 'stable': | |
| 112 _ReportValueError('Omaha report contains a line with the channel %s' | |
| 113 % row[1]) | |
| 114 if row[0] in platforms: | |
| 115 versions_map[row[0]] = row[2] | |
| 116 | |
| 117 if not all(platform in versions_map for platform in platforms): | |
| 118 _ReportValueError('Omaha report did not contain all desired platforms') | |
| 119 return versions_map | |
| 120 | |
| 121 @classmethod | |
| 122 def _CurrentRefBuildsMap(cls): | |
| 123 # TODO(aiolos): Add logic for pulling the current reference build versions. | |
| 124 # Return an empty dictionary to force an update until we store the builds in | |
| 125 # in cloud storage. | |
| 126 return {} | |
| 127 | |
| 128 @staticmethod | |
| 129 def _GetCmdStatusAndOutput(args, cwd=None, shell=False): | |
| 130 """Executes a subprocess and returns its exit code and output. | |
| 131 | |
| 132 Args: | |
| 133 args: A string or a sequence of program arguments. | |
| 134 cwd: If not None, the subprocess's current directory will be changed to | |
| 135 |cwd| before it's executed. | |
| 136 shell: Whether to execute args as a shell command. | |
| 137 | |
| 138 Returns: | |
| 139 The tuple (exit code, output). | |
| 140 """ | |
| 141 logging.info(str(args) + ' ' + (cwd or '')) | |
| 142 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, | |
| 143 stderr=subprocess.PIPE, shell=shell) | |
| 144 stdout, stderr = p.communicate() | |
| 145 exit_code = p.returncode | |
| 146 if stderr: | |
| 147 logging.critical(stderr) | |
| 148 logging.info(stdout) | |
| 149 return (exit_code, stdout) | |
| 150 | |
| 151 def _GetBuildUrl(self, platform, version, filename): | |
| 152 """Returns the URL for fetching one file. | |
| 153 | |
| 154 Args: | |
| 155 platform: Platform name, must be a key in |self._PLATFORM_MAP|. | |
| 156 version: A Chrome version number, e.g. 30.0.1600.1. | |
| 157 filename: Name of the file to fetch. | |
| 158 | |
| 159 Returns: | |
| 160 The URL for fetching a file. This may be a GS or HTTP URL. | |
| 161 """ | |
| 162 return CHROME_GS_URL_FMT % ( | |
| 163 version, self._PLATFORM_MAP[platform].gs_build, filename) | |
| 164 | |
| 165 def _FindBuildVersion(self, platform, version, filename): | |
| 166 """Searches for a version where a filename can be found. | |
| 167 | |
| 168 Args: | |
| 169 platform: Platform name. | |
| 170 version: A Chrome version number, e.g. 30.0.1600.1. | |
| 171 filename: Filename to look for. | |
| 172 | |
| 173 Returns: | |
| 174 A version where the file could be found, or None. | |
| 175 """ | |
| 176 # TODO(shadi): Iterate over official versions to find a valid one. | |
| 177 return (version | |
| 178 if self._DoesBuildExist(platform, version, filename) else None) | |
| 179 | |
| 180 def _DoesBuildExist(self, platform, version, filename): | |
| 181 """Checks whether a file can be found for the given Chrome version. | |
| 182 | |
| 183 Args: | |
| 184 platform: Platform name. | |
| 185 version: Chrome version number, e.g. 30.0.1600.1. | |
| 186 filename: Filename to look for. | |
| 187 | |
| 188 Returns: | |
| 189 True if the file could be found, False otherwise. | |
| 190 """ | |
| 191 url = self._GetBuildUrl(platform, version, filename) | |
| 192 return self._DoesGSFileExist(url) | |
| 193 | |
| 194 def _DoesGSFileExist(self, gs_file_name): | |
| 195 """Returns True if the GS file can be found, False otherwise.""" | |
| 196 exit_code = BuildUpdater._GetCmdStatusAndOutput( | |
| 197 ['gsutil', 'ls', gs_file_name])[0] | |
| 198 return not exit_code | |
| 199 | |
| 200 def _GetPlatformFiles(self, platform): | |
| 201 """Returns the name of the zip file to fetch for |platform|.""" | |
| 202 return BuildUpdater._PLATFORM_MAP[platform].zip_name | |
| 203 | |
| 204 def _DownloadBuilds(self): | |
| 205 for platform in self._platform_to_version_map: | |
| 206 version = self._platform_to_version_map[platform] | |
| 207 filename = self._GetPlatformFiles(platform) | |
| 208 output = os.path.join('dl', platform, | |
| 209 '%s_%s_%s' % (platform, | |
| 210 version, | |
| 211 filename)) | |
| 212 if os.path.exists(output): | |
| 213 logging.info('%s alread exists, skipping download', output) | |
| 214 continue | |
| 215 build_version = self._FindBuildVersion(platform, version, filename) | |
| 216 if not build_version: | |
| 217 logging.critical('Failed to find %s build for r%s\n', platform, | |
| 218 version) | |
| 219 sys.exit(1) | |
| 220 dirname = os.path.dirname(output) | |
| 221 if dirname and not os.path.exists(dirname): | |
| 222 os.makedirs(dirname) | |
| 223 url = self._GetBuildUrl(platform, build_version, filename) | |
| 224 self._DownloadFile(url, output) | |
| 225 | |
| 226 def _DownloadFile(self, url, output): | |
| 227 logging.info('Downloading %s, saving to %s', url, output) | |
| 228 BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output]) | |
| 229 | |
| 230 def _FetchSvnRepos(self): | |
| 231 if not os.path.exists('reference_builds'): | |
| 232 os.makedirs('reference_builds') | |
| 233 BuildUpdater._GetCmdStatusAndOutput( | |
| 234 ['gclient', 'config', | |
| 235 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], | |
| 236 'reference_builds') | |
| 237 BuildUpdater._GetCmdStatusAndOutput( | |
| 238 ['gclient', 'sync'], 'reference_builds') | |
| 239 | |
| 240 def _UnzipFile(self, dl_file, dest_dir): | |
| 241 """Unzips a file if it is a zip file. | |
| 242 | |
| 243 Args: | |
| 244 dl_file: The downloaded file to unzip. | |
| 245 dest_dir: The destination directory to unzip to. | |
| 246 | |
| 247 Returns: | |
| 248 True if the file was unzipped. False if it wasn't a zip file. | |
| 249 """ | |
| 250 if not zipfile.is_zipfile(dl_file): | |
| 251 return False | |
| 252 logging.info('Opening %s', dl_file) | |
| 253 with zipfile.ZipFile(dl_file, 'r') as z: | |
| 254 for content in z.namelist(): | |
| 255 dest = os.path.join(dest_dir, content[content.find('/') + 1:]) | |
| 256 # Create dest parent dir if it does not exist. | |
| 257 if not os.path.isdir(os.path.dirname(dest)): | |
| 258 os.makedirs(os.path.dirname(dest)) | |
| 259 # If dest is just a dir listing, do nothing. | |
| 260 if not os.path.basename(dest): | |
| 261 continue | |
| 262 if not os.path.isdir(os.path.dirname(dest)): | |
| 263 os.makedirs(os.path.dirname(dest)) | |
| 264 with z.open(content) as unzipped_content: | |
| 265 logging.info('Extracting %s to %s (%s)', content, dest, dl_file) | |
| 266 with file(dest, 'wb') as dest_file: | |
| 267 dest_file.write(unzipped_content.read()) | |
| 268 permissions = z.getinfo(content).external_attr >> 16 | |
| 269 if permissions: | |
| 270 os.chmod(dest, permissions) | |
| 271 return True | |
| 272 | |
| 273 def _ClearDir(self, directory): | |
| 274 """Clears all files in |directory| except for hidden files and folders.""" | |
| 275 for root, dirs, files in os.walk(directory): | |
| 276 # Skip hidden files and folders (like .svn and .git). | |
| 277 files = [f for f in files if f[0] != '.'] | |
| 278 dirs[:] = [d for d in dirs if d[0] != '.'] | |
| 279 | |
| 280 for f in files: | |
| 281 os.remove(os.path.join(root, f)) | |
| 282 | |
| 283 def _ExtractBuilds(self): | |
| 284 for platform in self._platform_to_version_map: | |
| 285 if os.path.exists('tmp_unzip'): | |
| 286 os.path.unlink('tmp_unzip') | |
| 287 dest_dir = os.path.join( | |
| 288 'reference_builds', 'reference_builds', | |
| 289 BuildUpdater._PLATFORM_MAP[platform].destination) | |
| 290 self._ClearDir(dest_dir) | |
| 291 for root, _, dl_files in os.walk(os.path.join('dl', platform)): | |
| 292 for dl_file in dl_files: | |
| 293 dl_file = os.path.join(root, dl_file) | |
| 294 if not self._UnzipFile(dl_file, dest_dir): | |
| 295 logging.info('Copying %s to %s', dl_file, dest_dir) | |
| 296 shutil.copy(dl_file, dest_dir) | |
| 297 | |
| 298 def _SvnAddAndRemove(self): | |
| 299 svn_dir = os.path.join('reference_builds', 'reference_builds') | |
| 300 # List all changes without ignoring any files. | |
| 301 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'], | |
| 302 svn_dir)[1] | |
| 303 for line in stat.splitlines(): | |
| 304 action, filename = line.split(None, 1) | |
| 305 # Add new and ignored files. | |
| 306 if action == '?' or action == 'I': | |
| 307 BuildUpdater._GetCmdStatusAndOutput( | |
| 308 ['svn', 'add', filename], svn_dir) | |
| 309 elif action == '!': | |
| 310 BuildUpdater._GetCmdStatusAndOutput( | |
| 311 ['svn', 'delete', filename], svn_dir) | |
| 312 filepath = os.path.join(svn_dir, filename) | |
| 313 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): | |
| 314 BuildUpdater._GetCmdStatusAndOutput( | |
| 315 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) | |
| 316 | |
| 317 def DownloadAndUpdateBuilds(self): | |
| 318 self._DownloadBuilds() | |
| 319 self._FetchSvnRepos() | |
| 320 self._ExtractBuilds() | |
| 321 self._SvnAddAndRemove() | |
| 322 | |
| 323 | |
| 324 def main(): | |
| 325 logging.getLogger().setLevel(logging.DEBUG) | |
| 326 # TODO(aiolos): check that there are no options passed (argparse). | |
| 327 b = BuildUpdater() | |
| 328 b.DownloadAndUpdateBuilds() | |
| 329 logging.info('Successfully updated reference builds. Move to ' | |
| 330 'reference_builds/reference_builds and make a change with gcl.') | |
| 331 | |
| 332 if __name__ == '__main__': | |
| 333 main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1797303003:
Remove telemetry reference builds from chromium DEPS. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master