Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(268)

Side by Side Diff: build/get_syzygy_binaries.py

Issue 2101243005: Add a snapshot of flutter/engine/src/build to our sdk (Closed) Base URL: git@github.com:dart-lang/sdk.git@master
Patch Set: add README.dart Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « build/get_sdk_extras_packages.py ('k') | build/git-hooks/OWNERS » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """A utility script for downloading versioned Syzygy binaries."""
7
8 import hashlib
9 import errno
10 import json
11 import logging
12 import optparse
13 import os
14 import re
15 import shutil
16 import stat
17 import sys
18 import subprocess
19 import tempfile
20 import time
21 import zipfile
22
23
24 _LOGGER = logging.getLogger(os.path.basename(__file__))
25
26 # The relative path where official builds are archived in their GS bucket.
27 _SYZYGY_ARCHIVE_PATH = ('/builds/official/%(revision)s')
28
29 # A JSON file containing the state of the download directory. If this file and
30 # directory state do not agree, then the binaries will be downloaded and
31 # installed again.
32 _STATE = '.state'
33
34 # This matches an integer (an SVN revision number) or a SHA1 value (a GIT hash).
35 # The archive exclusively uses lowercase GIT hashes.
36 _REVISION_RE = re.compile('^(?:\d+|[a-f0-9]{40})$')
37
38 # This matches an MD5 hash.
39 _MD5_RE = re.compile('^[a-f0-9]{32}$')
40
41 # List of reources to be downloaded and installed. These are tuples with the
42 # following format:
43 # (basename, logging name, relative installation path, extraction filter)
44 _RESOURCES = [
45 ('benchmark.zip', 'benchmark', '', None),
46 ('binaries.zip', 'binaries', 'exe', None),
47 ('symbols.zip', 'symbols', 'exe',
48 lambda x: x.filename.endswith('.dll.pdb'))]
49
50
51 def _LoadState(output_dir):
52 """Loads the contents of the state file for a given |output_dir|, returning
53 None if it doesn't exist.
54 """
55 path = os.path.join(output_dir, _STATE)
56 if not os.path.exists(path):
57 _LOGGER.debug('No state file found.')
58 return None
59 with open(path, 'rb') as f:
60 _LOGGER.debug('Reading state file: %s', path)
61 try:
62 return json.load(f)
63 except ValueError:
64 _LOGGER.debug('Invalid state file.')
65 return None
66
67
68 def _SaveState(output_dir, state, dry_run=False):
69 """Saves the |state| dictionary to the given |output_dir| as a JSON file."""
70 path = os.path.join(output_dir, _STATE)
71 _LOGGER.debug('Writing state file: %s', path)
72 if dry_run:
73 return
74 with open(path, 'wb') as f:
75 f.write(json.dumps(state, sort_keys=True, indent=2))
76
77
78 def _Md5(path):
79 """Returns the MD5 hash of the file at |path|, which must exist."""
80 return hashlib.md5(open(path, 'rb').read()).hexdigest()
81
82
83 def _StateIsValid(state):
84 """Returns true if the given state structure is valid."""
85 if not isinstance(state, dict):
86 _LOGGER.debug('State must be a dict.')
87 return False
88 r = state.get('revision', None)
89 if not isinstance(r, basestring) or not _REVISION_RE.match(r):
90 _LOGGER.debug('State contains an invalid revision.')
91 return False
92 c = state.get('contents', None)
93 if not isinstance(c, dict):
94 _LOGGER.debug('State must contain a contents dict.')
95 return False
96 for (relpath, md5) in c.iteritems():
97 if not isinstance(relpath, basestring) or len(relpath) == 0:
98 _LOGGER.debug('State contents dict contains an invalid path.')
99 return False
100 if not isinstance(md5, basestring) or not _MD5_RE.match(md5):
101 _LOGGER.debug('State contents dict contains an invalid MD5 digest.')
102 return False
103 return True
104
105
106 def _BuildActualState(stored, revision, output_dir):
107 """Builds the actual state using the provided |stored| state as a template.
108 Only examines files listed in the stored state, causing the script to ignore
109 files that have been added to the directories locally. |stored| must be a
110 valid state dictionary.
111 """
112 contents = {}
113 state = { 'revision': revision, 'contents': contents }
114 for relpath, md5 in stored['contents'].iteritems():
115 abspath = os.path.abspath(os.path.join(output_dir, relpath))
116 if os.path.isfile(abspath):
117 m = _Md5(abspath)
118 contents[relpath] = m
119
120 return state
121
122
123 def _StatesAreConsistent(stored, actual):
124 """Validates whether two state dictionaries are consistent. Both must be valid
125 state dictionaries. Additional entries in |actual| are ignored.
126 """
127 if stored['revision'] != actual['revision']:
128 _LOGGER.debug('Mismatched revision number.')
129 return False
130 cont_stored = stored['contents']
131 cont_actual = actual['contents']
132 for relpath, md5 in cont_stored.iteritems():
133 if relpath not in cont_actual:
134 _LOGGER.debug('Missing content: %s', relpath)
135 return False
136 if md5 != cont_actual[relpath]:
137 _LOGGER.debug('Modified content: %s', relpath)
138 return False
139 return True
140
141
142 def _GetCurrentState(revision, output_dir):
143 """Loads the current state and checks to see if it is consistent. Returns
144 a tuple (state, bool). The returned state will always be valid, even if an
145 invalid state is present on disk.
146 """
147 stored = _LoadState(output_dir)
148 if not _StateIsValid(stored):
149 _LOGGER.debug('State is invalid.')
150 # Return a valid but empty state.
151 return ({'revision': '0', 'contents': {}}, False)
152 actual = _BuildActualState(stored, revision, output_dir)
153 # If the script has been modified consider the state invalid.
154 path = os.path.join(output_dir, _STATE)
155 if os.path.getmtime(__file__) > os.path.getmtime(path):
156 return (stored, False)
157 # Otherwise, explicitly validate the state.
158 if not _StatesAreConsistent(stored, actual):
159 return (stored, False)
160 return (stored, True)
161
162
163 def _DirIsEmpty(path):
164 """Returns true if the given directory is empty, false otherwise."""
165 for root, dirs, files in os.walk(path):
166 return not dirs and not files
167
168
169 def _RmTreeHandleReadOnly(func, path, exc):
170 """An error handling function for use with shutil.rmtree. This will
171 detect failures to remove read-only files, and will change their properties
172 prior to removing them. This is necessary on Windows as os.remove will return
173 an access error for read-only files, and git repos contain read-only
174 pack/index files.
175 """
176 excvalue = exc[1]
177 if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
178 _LOGGER.debug('Removing read-only path: %s', path)
179 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
180 func(path)
181 else:
182 raise
183
184
185 def _RmTree(path):
186 """A wrapper of shutil.rmtree that handles read-only files."""
187 shutil.rmtree(path, ignore_errors=False, onerror=_RmTreeHandleReadOnly)
188
189
190 def _CleanState(output_dir, state, dry_run=False):
191 """Cleans up files/directories in |output_dir| that are referenced by
192 the given |state|. Raises an error if there are local changes. Returns a
193 dictionary of files that were deleted.
194 """
195 _LOGGER.debug('Deleting files from previous installation.')
196 deleted = {}
197
198 # Generate a list of files to delete, relative to |output_dir|.
199 contents = state['contents']
200 files = sorted(contents.keys())
201
202 # Try to delete the files. Keep track of directories to delete as well.
203 dirs = {}
204 for relpath in files:
205 fullpath = os.path.join(output_dir, relpath)
206 fulldir = os.path.dirname(fullpath)
207 dirs[fulldir] = True
208 if os.path.exists(fullpath):
209 # If somehow the file has become a directory complain about it.
210 if os.path.isdir(fullpath):
211 raise Exception('Directory exists where file expected: %s' % fullpath)
212
213 # Double check that the file doesn't have local changes. If it does
214 # then refuse to delete it.
215 if relpath in contents:
216 stored_md5 = contents[relpath]
217 actual_md5 = _Md5(fullpath)
218 if actual_md5 != stored_md5:
219 raise Exception('File has local changes: %s' % fullpath)
220
221 # The file is unchanged so it can safely be deleted.
222 _LOGGER.debug('Deleting file "%s".', fullpath)
223 deleted[relpath] = True
224 if not dry_run:
225 os.unlink(fullpath)
226
227 # Sort directories from longest name to shortest. This lets us remove empty
228 # directories from the most nested paths first.
229 dirs = sorted(dirs.keys(), key=lambda x: len(x), reverse=True)
230 for p in dirs:
231 if os.path.exists(p) and _DirIsEmpty(p):
232 _LOGGER.debug('Deleting empty directory "%s".', p)
233 if not dry_run:
234 _RmTree(p)
235
236 return deleted
237
238
239 def _FindGsUtil():
240 """Looks for depot_tools and returns the absolute path to gsutil.py."""
241 for path in os.environ['PATH'].split(os.pathsep):
242 path = os.path.abspath(path)
243 git_cl = os.path.join(path, 'git_cl.py')
244 gs_util = os.path.join(path, 'gsutil.py')
245 if os.path.exists(git_cl) and os.path.exists(gs_util):
246 return gs_util
247 return None
248
249
250 def _GsUtil(*cmd):
251 """Runs the given command in gsutil with exponential backoff and retries."""
252 gs_util = _FindGsUtil()
253 cmd = [sys.executable, gs_util] + list(cmd)
254
255 retries = 3
256 timeout = 4 # Seconds.
257 while True:
258 _LOGGER.debug('Running %s', cmd)
259 prog = subprocess.Popen(cmd, shell=False)
260 prog.communicate()
261
262 # Stop retrying on success.
263 if prog.returncode == 0:
264 return
265
266 # Raise a permanent failure if retries have been exhausted.
267 if retries == 0:
268 raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode))
269
270 _LOGGER.debug('Sleeping %d seconds and trying again.', timeout)
271 time.sleep(timeout)
272 retries -= 1
273 timeout *= 2
274
275
276 def _Download(resource):
277 """Downloads the given GS resource to a temporary file, returning its path."""
278 tmp = tempfile.mkstemp(suffix='syzygy_archive')
279 os.close(tmp[0])
280 url = 'gs://syzygy-archive' + resource
281 _GsUtil('cp', url, tmp[1])
282 return tmp[1]
283
284
285 def _InstallBinaries(options, deleted={}):
286 """Installs Syzygy binaries. This assumes that the output directory has
287 already been cleaned, as it will refuse to overwrite existing files."""
288 contents = {}
289 state = { 'revision': options.revision, 'contents': contents }
290 archive_path = _SYZYGY_ARCHIVE_PATH % { 'revision': options.revision }
291 if options.resources:
292 resources = [(resource, resource, '', None)
293 for resource in options.resources]
294 else:
295 resources = _RESOURCES
296 for (base, name, subdir, filt) in resources:
297 # Create the output directory if it doesn't exist.
298 fulldir = os.path.join(options.output_dir, subdir)
299 if os.path.isfile(fulldir):
300 raise Exception('File exists where a directory needs to be created: %s' %
301 fulldir)
302 if not os.path.exists(fulldir):
303 _LOGGER.debug('Creating directory: %s', fulldir)
304 if not options.dry_run:
305 os.makedirs(fulldir)
306
307 # Download and read the archive.
308 resource = archive_path + '/' + base
309 _LOGGER.debug('Retrieving %s archive at "%s".', name, resource)
310 path = _Download(resource)
311
312 _LOGGER.debug('Unzipping %s archive.', name)
313 with open(path, 'rb') as data:
314 archive = zipfile.ZipFile(data)
315 for entry in archive.infolist():
316 if not filt or filt(entry):
317 fullpath = os.path.normpath(os.path.join(fulldir, entry.filename))
318 relpath = os.path.relpath(fullpath, options.output_dir)
319 if os.path.exists(fullpath):
320 # If in a dry-run take into account the fact that the file *would*
321 # have been deleted.
322 if options.dry_run and relpath in deleted:
323 pass
324 else:
325 raise Exception('Path already exists: %s' % fullpath)
326
327 # Extract the file and update the state dictionary.
328 _LOGGER.debug('Extracting "%s".', fullpath)
329 if not options.dry_run:
330 archive.extract(entry.filename, fulldir)
331 md5 = _Md5(fullpath)
332 contents[relpath] = md5
333 if sys.platform == 'cygwin':
334 os.chmod(fullpath, os.stat(fullpath).st_mode | stat.S_IXUSR)
335
336 _LOGGER.debug('Removing temporary file "%s".', path)
337 os.remove(path)
338
339 return state
340
341
342 def _ParseCommandLine():
343 """Parses the command-line and returns an options structure."""
344 option_parser = optparse.OptionParser()
345 option_parser.add_option('--dry-run', action='store_true', default=False,
346 help='If true then will simply list actions that would be performed.')
347 option_parser.add_option('--force', action='store_true', default=False,
348 help='Force an installation even if the binaries are up to date.')
349 option_parser.add_option('--no-cleanup', action='store_true', default=False,
350 help='Allow installation on non-Windows platforms, and skip the forced '
351 'cleanup step.')
352 option_parser.add_option('--output-dir', type='string',
353 help='The path where the binaries will be replaced. Existing binaries '
354 'will only be overwritten if not up to date.')
355 option_parser.add_option('--overwrite', action='store_true', default=False,
356 help='If specified then the installation will happily delete and rewrite '
357 'the entire output directory, blasting any local changes.')
358 option_parser.add_option('--revision', type='string',
359 help='The SVN revision or GIT hash associated with the required version.')
360 option_parser.add_option('--revision-file', type='string',
361 help='A text file containing an SVN revision or GIT hash.')
362 option_parser.add_option('--resource', type='string', action='append',
363 dest='resources', help='A resource to be downloaded.')
364 option_parser.add_option('--verbose', dest='log_level', action='store_const',
365 default=logging.INFO, const=logging.DEBUG,
366 help='Enables verbose logging.')
367 option_parser.add_option('--quiet', dest='log_level', action='store_const',
368 default=logging.INFO, const=logging.ERROR,
369 help='Disables all output except for errors.')
370 options, args = option_parser.parse_args()
371 if args:
372 option_parser.error('Unexpected arguments: %s' % args)
373 if not options.output_dir:
374 option_parser.error('Must specify --output-dir.')
375 if not options.revision and not options.revision_file:
376 option_parser.error('Must specify one of --revision or --revision-file.')
377 if options.revision and options.revision_file:
378 option_parser.error('Must not specify both --revision and --revision-file.')
379
380 # Configure logging.
381 logging.basicConfig(level=options.log_level)
382
383 # If a revision file has been specified then read it.
384 if options.revision_file:
385 options.revision = open(options.revision_file, 'rb').read().strip()
386 _LOGGER.debug('Parsed revision "%s" from file "%s".',
387 options.revision, options.revision_file)
388
389 # Ensure that the specified SVN revision or GIT hash is valid.
390 if not _REVISION_RE.match(options.revision):
391 option_parser.error('Must specify a valid SVN or GIT revision.')
392
393 # This just makes output prettier to read.
394 options.output_dir = os.path.normpath(options.output_dir)
395
396 return options
397
398
399 def _RemoveOrphanedFiles(options):
400 """This is run on non-Windows systems to remove orphaned files that may have
401 been downloaded by a previous version of this script.
402 """
403 # Reconfigure logging to output info messages. This will allow inspection of
404 # cleanup status on non-Windows buildbots.
405 _LOGGER.setLevel(logging.INFO)
406
407 output_dir = os.path.abspath(options.output_dir)
408
409 # We only want to clean up the folder in 'src/third_party/syzygy', and we
410 # expect to be called with that as an output directory. This is an attempt to
411 # not start deleting random things if the script is run from an alternate
412 # location, or not called from the gclient hooks.
413 expected_syzygy_dir = os.path.abspath(os.path.join(
414 os.path.dirname(__file__), '..', 'third_party', 'syzygy'))
415 expected_output_dir = os.path.join(expected_syzygy_dir, 'binaries')
416 if expected_output_dir != output_dir:
417 _LOGGER.info('Unexpected output directory, skipping cleanup.')
418 return
419
420 if not os.path.isdir(expected_syzygy_dir):
421 _LOGGER.info('Output directory does not exist, skipping cleanup.')
422 return
423
424 def OnError(function, path, excinfo):
425 """Logs error encountered by shutil.rmtree."""
426 _LOGGER.error('Error when running %s(%s)', function, path, exc_info=excinfo)
427
428 _LOGGER.info('Removing orphaned files from %s', expected_syzygy_dir)
429 if not options.dry_run:
430 shutil.rmtree(expected_syzygy_dir, True, OnError)
431
432
433 def main():
434 options = _ParseCommandLine()
435
436 if options.dry_run:
437 _LOGGER.debug('Performing a dry-run.')
438
439 # We only care about Windows platforms, as the Syzygy binaries aren't used
440 # elsewhere. However, there was a short period of time where this script
441 # wasn't gated on OS types, and those OSes downloaded and installed binaries.
442 # This will cleanup orphaned files on those operating systems.
443 if sys.platform not in ('win32', 'cygwin'):
444 if options.no_cleanup:
445 _LOGGER.debug('Skipping usual cleanup for non-Windows platforms.')
446 else:
447 return _RemoveOrphanedFiles(options)
448
449 # Load the current installation state, and validate it against the
450 # requested installation.
451 state, is_consistent = _GetCurrentState(options.revision, options.output_dir)
452
453 # Decide whether or not an install is necessary.
454 if options.force:
455 _LOGGER.debug('Forcing reinstall of binaries.')
456 elif is_consistent:
457 # Avoid doing any work if the contents of the directory are consistent.
458 _LOGGER.debug('State unchanged, no reinstall necessary.')
459 return
460
461 # Under normal logging this is the only only message that will be reported.
462 _LOGGER.info('Installing revision %s Syzygy binaries.',
463 options.revision[0:12])
464
465 # Clean up the old state to begin with.
466 deleted = []
467 if options.overwrite:
468 if os.path.exists(options.output_dir):
469 # If overwrite was specified then take a heavy-handed approach.
470 _LOGGER.debug('Deleting entire installation directory.')
471 if not options.dry_run:
472 _RmTree(options.output_dir)
473 else:
474 # Otherwise only delete things that the previous installation put in place,
475 # and take care to preserve any local changes.
476 deleted = _CleanState(options.output_dir, state, options.dry_run)
477
478 # Install the new binaries. In a dry-run this will actually download the
479 # archives, but it won't write anything to disk.
480 state = _InstallBinaries(options, deleted)
481
482 # Build and save the state for the directory.
483 _SaveState(options.output_dir, state, options.dry_run)
484
485
486 if __name__ == '__main__':
487 main()
OLDNEW
« no previous file with comments | « build/get_sdk_extras_packages.py ('k') | build/git-hooks/OWNERS » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698