OLD | NEW |
(Empty) | |
| 1 # Copyright 2014 Google Inc. All Rights Reserved. |
| 2 # |
| 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 # you may not use this file except in compliance with the License. |
| 5 # You may obtain a copy of the License at |
| 6 # |
| 7 # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 # |
| 9 # Unless required by applicable law or agreed to in writing, software |
| 10 # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 # See the License for the specific language governing permissions and |
| 13 # limitations under the License. |
| 14 # |
| 15 # This file was originally copied from syzygy project available at |
| 16 # https://github.com/google/syzygy. |
| 17 """A utility script for checking out subdirectories of many GIT repositories |
| 18 to specified locations, like is possible with SVN and gclient. This uses a |
| 19 combination of GIT, sparse-checkout, shallow-clone and filesystem junctions. |
| 20 |
| 21 For each dependency in a 'gitdeps' file this script will checkout one |
| 22 subdirectory of one repository into a specified location. The input is as |
| 23 follows: |
| 24 |
| 25 - The user specifies a local destination for the checkout. |
| 26 - The user specifies a source repository. |
| 27 - The user specifies a list of subdirectories of the repository to get. |
| 28 - The user specifies a revision. |
| 29 |
| 30 The checkout works as follows: |
| 31 |
| 32 - An empty git checkout is initialized in the cache directory. This will be |
| 33 in a subfolder with an essentially random name. |
| 34 - The specified repository is added as a remote to that repo. |
| 35 - A sparse-checkout directive is added to select only the desired |
| 36 subdirectories. |
| 37 - The repository is cloned using a depth of 1 (no history, only the actual |
| 38 contents of the desired revision). |
| 39 - The destination directories are created as junctions pointing to the |
| 40 desired subdirectory of the checkout in the cache directory. |
| 41 |
| 42 The script maintains its state in the root of the cache directory, allowing it |
| 43 to reuse checkout directories when possible. |
| 44 """ |
| 45 |
| 46 import ast |
| 47 import glob |
| 48 import hashlib |
| 49 import logging |
| 50 import optparse |
| 51 import os |
| 52 import random |
| 53 import re |
| 54 import subprocess |
| 55 import threading |
| 56 |
| 57 |
| 58 _LOGGER = logging.getLogger(os.path.basename(__file__)) |
| 59 |
| 60 |
| 61 # Matches a SHA1 hash used as a git revision. |
| 62 _GIT_SHA1_RE = re.compile('^[A-Fa-f0-9]{40}$') |
| 63 |
| 64 |
| 65 def _ParseCommandLine(): |
| 66 """Parses the command-line and returns an options structure.""" |
| 67 option_parser = optparse.OptionParser() |
| 68 option_parser.add_option('--cache-dir', type='string', |
| 69 default='.gitdeps-cache', |
| 70 help='The directory to be used for storing cache files. Defaults to ' |
| 71 '.gitdeps-cache in the current working directory.') |
| 72 option_parser.add_option('--output-dir', type='string', default='.', |
| 73 help='The directory to be used as the root of all output. Defaults to ' |
| 74 'the current working directory.') |
| 75 option_parser.add_option('--dry-run', action='store_true', default=False, |
| 76 help='If true then will simply list actions that would be performed.') |
| 77 option_parser.add_option('--force', action='store_true', default=False, |
| 78 help='If true then will force the checkout to be completely rebuilt.') |
| 79 option_parser.add_option('--verbose', dest='log_level', action='store_const', |
| 80 default=logging.INFO, const=logging.DEBUG, |
| 81 help='Enables verbose logging.') |
| 82 option_parser.add_option('--quiet', dest='log_level', action='store_const', |
| 83 default=logging.INFO, const=logging.ERROR, |
| 84 help='Disables all output except for errors.') |
| 85 |
| 86 options, args = option_parser.parse_args() |
| 87 |
| 88 # Configure logging. |
| 89 logging.basicConfig(level=options.log_level) |
| 90 |
| 91 # Set default values. |
| 92 if not args: |
| 93 # Default to checking for a file in the current working directory. |
| 94 _LOGGER.info('Defaulting to using GITDEPS in current working directory.') |
| 95 args = ['GITDEPS'] |
| 96 |
| 97 # Validate arguments and options. |
| 98 if not os.path.isdir(options.output_dir): |
| 99 option_parser.error('Output directory does not exist: %s' % |
| 100 options.output_dir) |
| 101 for path in args: |
| 102 if not os.path.exists(path): |
| 103 option_parser.error('Missing dependency file: %s' % path) |
| 104 |
| 105 # Normalize local paths for prettier output. |
| 106 options.cache_dir = os.path.normpath(os.path.abspath(options.cache_dir)) |
| 107 options.output_dir = os.path.normpath(os.path.abspath(options.output_dir)) |
| 108 |
| 109 return options, args |
| 110 |
| 111 |
| 112 class RepoOptions(object): |
| 113 """Light object used for shuttling around information about a dependency.""" |
| 114 |
| 115 def __init__(self): |
| 116 self.repository = None |
| 117 self.revision = None |
| 118 self.output_dir = None |
| 119 self.remote_dirs = [] |
| 120 self.deps_file = None |
| 121 self.checkout_dir = None |
| 122 self.recurse = False |
| 123 |
| 124 def __str__(self): |
| 125 """Stringifies this object for debugging.""" |
| 126 return ('RepoOptions(repository=%s, revision=%s, output_dir=%s, ' |
| 127 'remote_dirs=%s, deps_file=%s, checkout_dir=%s, recurse=%s)') % ( |
| 128 self.repository.__repr__(), |
| 129 self.revision.__repr__(), |
| 130 self.output_dir.__repr__(), |
| 131 self.remote_dirs.__repr__(), |
| 132 self.deps_file.__repr__(), |
| 133 self.checkout_dir.__repr__(), |
| 134 self.recurse.__repr__()) |
| 135 |
| 136 |
| 137 def _ParseRepoOptions(cache_dir, root_output_dir, deps_file_path, key, value): |
| 138 """Given the |root_output_dir| specified on the command line, a |key| and |
| 139 |value| pair from a GITDEPS file, and the path of the deps file, generates |
| 140 a corresponding RepoOptions object. The |key| is the output path of the |
| 141 checkout relative to |root_output_dir|, and |value| consists of a |
| 142 (repository URL, remote directory, revision hash) tuple. This can raise an |
| 143 Exception on failure. |
| 144 """ |
| 145 bad = False |
| 146 if ((type(value) != list and type(value) != tuple) or len(value) < 3 or |
| 147 len(value) > 4 or (type(value[1]) != list and type(value[1]) != tuple)): |
| 148 bad = True |
| 149 if len(value) == 4 and type(value[3]) != dict: |
| 150 bad = True |
| 151 if bad: |
| 152 _LOGGER.error('Invalid dependency tuple: %s', value) |
| 153 raise Exception() |
| 154 |
| 155 # Always use lowercase SHA1 hashes for consistency. |
| 156 refspec = value[2] |
| 157 if _GIT_SHA1_RE.match(refspec): |
| 158 refspec = refspec.lower() |
| 159 |
| 160 repo_options = RepoOptions() |
| 161 repo_options.output_dir = os.path.normpath(os.path.abspath(os.path.join( |
| 162 root_output_dir, key))) |
| 163 repo_options.repository = value[0] |
| 164 repo_options.remote_dirs = value[1] |
| 165 repo_options.revision = refspec |
| 166 repo_options.deps_file = deps_file_path |
| 167 |
| 168 # Parse additional options. |
| 169 if len(value) > 3: |
| 170 repo_options.recurse = value[3].get('recurse', False) == True |
| 171 |
| 172 # Create a unique name for the checkout in the cache directory. Make the |
| 173 # output directory relative to the cache directory so that they can be |
| 174 # moved around together. |
| 175 output_dir_rel = os.path.relpath(repo_options.output_dir, |
| 176 root_output_dir).lower() |
| 177 if output_dir_rel.startswith('..'): |
| 178 raise Exception('Invalid output directory: %s' % key) |
| 179 n = hashlib.md5(output_dir_rel).hexdigest() |
| 180 repo_options.checkout_dir = os.path.abspath(os.path.join(cache_dir, n, 'src')) |
| 181 |
| 182 return repo_options |
| 183 |
| 184 |
| 185 def _EnsureDirectoryExists(path, comment_name, dry_run): |
| 186 """Ensures that the given |path| exists. Only actually creates the directory |
| 187 if |dry_run| is False. |comment_name| is used during logging of this |
| 188 operation. |
| 189 """ |
| 190 if not comment_name: |
| 191 comment_name += ' ' |
| 192 else: |
| 193 comment_name = '' |
| 194 if not os.path.exists(path): |
| 195 _LOGGER.debug('Creating %sdirectory: %s', comment_name, path) |
| 196 if not dry_run: |
| 197 os.makedirs(path) |
| 198 |
| 199 |
| 200 def _GetCasedFilename(filename): |
| 201 """Returns the full case-sensitive filename for the given |filename|. If the |
| 202 path does not exist, returns the original |filename| as is. |
| 203 """ |
| 204 pattern = '%s[%s]' % (filename[:-1], filename[-1]) |
| 205 filenames = glob.glob(pattern) |
| 206 if not filenames: |
| 207 return filename |
| 208 return filenames[0] |
| 209 |
| 210 |
| 211 def _Shell(*cmd, **kw): |
| 212 """Runs |cmd|, returns the results from Popen(cmd).communicate(). Additional |
| 213 keyword arguments are passed on to subprocess.Popen. If |stdout| and |stderr| |
| 214 are not specified, they default to subprocess.PIPE. If |dry_run| is not |
| 215 specified it defaults to True. The command is only actually run if |dry_run| |
| 216 is False. This can raise a RuntimeError on failure. |
| 217 """ |
| 218 if 'cwd' in kw: |
| 219 _LOGGER.debug('Executing %s in "%s".', cmd, kw['cwd']) |
| 220 else: |
| 221 _LOGGER.debug('Executing %s.', cmd) |
| 222 if kw.get('dry_run', True): |
| 223 return ('', '') |
| 224 kw.pop('dry_run', None) |
| 225 dump_on_error = kw.pop('dump_on_error', False) |
| 226 |
| 227 kw['shell'] = True |
| 228 kw.setdefault('stdout', subprocess.PIPE) |
| 229 kw.setdefault('stderr', subprocess.PIPE) |
| 230 prog = subprocess.Popen(cmd, **kw) |
| 231 |
| 232 stdout, stderr = prog.communicate() |
| 233 if prog.returncode != 0: |
| 234 if dump_on_error: |
| 235 print stdout |
| 236 print stderr |
| 237 raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode)) |
| 238 return (stdout, stderr) |
| 239 |
| 240 |
| 241 def _IsGitCheckoutRoot(path): |
| 242 """Return true if the given |path| is the root of a git checkout.""" |
| 243 return os.path.exists(os.path.join(path, '.git')) |
| 244 |
| 245 |
| 246 # Matches a GIT config file section header, and grabs the name of the section |
| 247 # in the first group. Used by _GetGitOrigin. |
| 248 _GIT_CONFIG_SECTION_RE = re.compile(r'^\s*\[(.*?)\]\s*$') |
| 249 # Matches the URL line from a 'remote' section of a GIT config. Used by |
| 250 # _GetGitOrigin. |
| 251 _GIT_CONFIG_REMOTE_URL_RE = re.compile(r'^\s*url\s*=\s*(.*?)\s*$') |
| 252 |
| 253 |
| 254 def _GetGitOrigin(path): |
| 255 """Returns the URL of the 'origin' remote for the git repo in |path|. Returns |
| 256 None if the 'origin' remote doesn't exist. Raises an IOError if |path| doesn't |
| 257 exist or is not a git repo. |
| 258 """ |
| 259 section = None |
| 260 for line in open(os.path.join(path, '.git', 'config'), 'rb'): |
| 261 m = _GIT_CONFIG_SECTION_RE.match(line) |
| 262 if m: |
| 263 section = m.group(1) |
| 264 continue |
| 265 |
| 266 # We only care about the 'origin' configuration. |
| 267 if section != 'remote "origin"': |
| 268 continue |
| 269 |
| 270 m = _GIT_CONFIG_REMOTE_URL_RE.match(line) |
| 271 if m: |
| 272 return m.group(1).strip() |
| 273 |
| 274 return None |
| 275 |
| 276 |
| 277 def _GetGitHead(path): |
| 278 """Returns the hash of the head of the git repo in |path|. Raises an IOError |
| 279 if |path| doesn't exist or is not a git repo. |
| 280 """ |
| 281 return open(os.path.join(path, '.git', 'HEAD'), 'rb').read().strip() |
| 282 |
| 283 |
| 284 def _NormalizeGitPath(path): |
| 285 """Given a |path| in a GIT repository (relative to its root), normalizes it so |
| 286 it will match only that exact path in a sparse checkout. |
| 287 """ |
| 288 path = path.strip() |
| 289 if not path.startswith('/'): |
| 290 path = '/' + path |
| 291 if not path.endswith('/'): |
| 292 path += '/' |
| 293 return path |
| 294 |
| 295 |
| 296 def _RenameCheckout(path, dry_run): |
| 297 """Renames the checkout in |path| so that it can be subsequently deleted. |
| 298 Only actually does the work if |dry_run| is False. Returns the path of the |
| 299 renamed checkout directory. Raises an Exception on failure. |
| 300 """ |
| 301 |
| 302 def _RenameCheckoutImpl(path, dry_run): |
| 303 if dry_run: |
| 304 return path + '-old-dryrun' |
| 305 attempts = 0 |
| 306 while attempts < 10: |
| 307 newpath = '%s-old-%04d' % (path, random.randint(0, 999)) |
| 308 try: |
| 309 os.rename(path, newpath) |
| 310 return newpath |
| 311 except WindowsError: |
| 312 attempts += 1 |
| 313 raise Exception('Unable to rename checkout directory: %s' % path) |
| 314 |
| 315 newpath = _RenameCheckoutImpl(path, dry_run) |
| 316 _LOGGER.debug('Renamed checkout directory: %s', newpath) |
| 317 return newpath |
| 318 |
| 319 |
| 320 def _DeleteCheckout(path, dry_run): |
| 321 """Deletes the checkout in |path|. Only actually deletes the checkout if |
| 322 |dry_run| is False. |
| 323 """ |
| 324 _LOGGER.info('Deleting checkout directory: %s', path) |
| 325 if dry_run: |
| 326 return |
| 327 _Shell('rmdir', '/S', '/Q', path, dry_run=False) |
| 328 |
| 329 |
| 330 def _GenerateSparseCheckoutPathAndContents(repo): |
| 331 """Generates the path to the sparse checkout file, and the desired |
| 332 contents. Returns a tuple of (path, contents). |repo| is a RepoOptions object. |
| 333 """ |
| 334 sparse_file = os.path.join(repo.checkout_dir, '.git', 'info', |
| 335 'sparse-checkout') |
| 336 if not repo.remote_dirs: |
| 337 contents = '*\n' |
| 338 else: |
| 339 contents = ''.join(_NormalizeGitPath(dir) + '\n' |
| 340 for dir in repo.remote_dirs) |
| 341 return (sparse_file, contents) |
| 342 |
| 343 |
| 344 def _HasValidSparseCheckoutConfig(repo): |
| 345 """Determines if the GIT repo in |path| has a valid sparse-checkout |
| 346 configuration as configured by the RepoOptions |repo|. Returns True or False. |
| 347 """ |
| 348 (sparse_file, contents) = _GenerateSparseCheckoutPathAndContents(repo) |
| 349 try: |
| 350 if open(sparse_file, 'rb').read() == contents: |
| 351 return True |
| 352 return False |
| 353 except IOError: |
| 354 return False |
| 355 |
| 356 |
| 357 def _CreateCheckout(path, repo, dry_run): |
| 358 """Creates a checkout in the provided |path|. The |path| must not already |
| 359 exist. Uses the repository configuration from the provided |repo| RepoOptions |
| 360 object. Only actually creates the checkout if |dry_run| is false. |
| 361 """ |
| 362 # We expect the directory not to exist, as this is a fresh checkout we are |
| 363 # creating. |
| 364 if not dry_run: |
| 365 if os.path.exists(path): |
| 366 raise Exception('Checkout directory already exists: %s' % path) |
| 367 |
| 368 _LOGGER.info('Creating checkout directory: %s', path) |
| 369 if not dry_run: |
| 370 os.makedirs(path) |
| 371 |
| 372 _LOGGER.debug('Initializing the checkout.') |
| 373 _Shell('git', 'init', cwd=path, dry_run=dry_run) |
| 374 _Shell('git', 'remote', 'add', 'origin', repo.repository, cwd=path, |
| 375 dry_run=dry_run) |
| 376 _Shell('git', 'config', 'core.sparsecheckout', 'true', cwd=path, |
| 377 dry_run=dry_run) |
| 378 if not dry_run: |
| 379 _LOGGER.debug('Creating sparse checkout configuration file for ' |
| 380 'directory: %s', repo.remote_dirs) |
| 381 if not dry_run: |
| 382 (path, contents) = _GenerateSparseCheckoutPathAndContents(repo) |
| 383 with open(path, 'wb') as io: |
| 384 io.write(contents) |
| 385 |
| 386 |
| 387 def _UpdateCheckout(path, repo, dry_run): |
| 388 """Updates a GIT checkout in |path| by pulling down a specific revision |
| 389 from it, as configured by RepoOptions |repo|. Only actually runs if |
| 390 |dry_run| is False. |
| 391 """ |
| 392 try: |
| 393 # Try a checkout first. If this fails then we'll actually need to fetch |
| 394 # the revision. |
| 395 _LOGGER.info('Trying to checkout revision %s.', repo.revision) |
| 396 _Shell('git', 'checkout', repo.revision, cwd=path, |
| 397 dry_run=dry_run) |
| 398 return |
| 399 except RuntimeError: |
| 400 pass |
| 401 |
| 402 # Fetch the revision and then check it out. Let output go to screen rather |
| 403 # than be buffered. |
| 404 _LOGGER.info('Fetching and checking out revision %s.', repo.revision) |
| 405 _Shell('git', 'fetch', '--depth=1', 'origin', repo.revision, |
| 406 cwd=path, dry_run=dry_run, stdout=None, stderr=None) |
| 407 _Shell('git', 'checkout', repo.revision, cwd=path, |
| 408 dry_run=dry_run, stdout=None, stderr=None) |
| 409 |
| 410 |
| 411 # Used by _GetJunctionInfo to extract information about junctions. |
| 412 _DIR_JUNCTION_RE = re.compile(r'^.*<JUNCTION>\s+(.+)\s+\[(.+)\]$') |
| 413 |
| 414 |
| 415 # TODO(chrisha): This is ugly, and there has to be a better way! |
| 416 def _GetJunctionInfo(junction): |
| 417 """Returns the target of a junction, if it exists, None otherwise.""" |
| 418 dirname = os.path.dirname(junction) |
| 419 basename = os.path.basename(junction) |
| 420 try: |
| 421 stdout, dummy_stderr = _Shell('dir', '/AL', '/N', dirname, dry_run=False) |
| 422 except RuntimeError: |
| 423 return |
| 424 |
| 425 lines = stdout.splitlines(False) |
| 426 for line in stdout.splitlines(False): |
| 427 m = _DIR_JUNCTION_RE.match(line) |
| 428 if not m: |
| 429 continue |
| 430 if m.group(1).lower() == basename.lower(): |
| 431 return m.group(2) |
| 432 |
| 433 return None |
| 434 |
| 435 |
| 436 def _EnsureJunction(cache_dir, target_dir, options, repo): |
| 437 """Ensures that the appropriate junction exists from the configured output |
| 438 directory to the specified sub-directory of the GIT checkout. |
| 439 """ |
| 440 # Ensure that the target directory was created. |
| 441 target_cache_dir = _GetCasedFilename(os.path.normpath( |
| 442 os.path.join(cache_dir, target_dir))) |
| 443 if not options.dry_run and not os.path.isdir(target_cache_dir): |
| 444 raise Exception('Checkout does not contain the desired remote folder.') |
| 445 |
| 446 # Ensure the parent directory exists before checking if the junction needs to |
| 447 # be created. |
| 448 output_dir = os.path.normpath(os.path.join(repo.output_dir, target_dir)) |
| 449 _EnsureDirectoryExists( |
| 450 os.path.dirname(output_dir), 'junction', options.dry_run) |
| 451 |
| 452 # Determine if the link needs to be created. |
| 453 create_link = True |
| 454 if os.path.exists(output_dir): |
| 455 dest = _GetJunctionInfo(output_dir) |
| 456 |
| 457 # If the junction is valid nothing needs to be done. If it points to the |
| 458 # wrong place or isn't a junction then delete it and let it be remade. |
| 459 if dest == target_cache_dir: |
| 460 _LOGGER.debug('Junction is up to date.') |
| 461 create_link = False |
| 462 else: |
| 463 if dest: |
| 464 _LOGGER.info('Erasing existing junction: %s', output_dir) |
| 465 else: |
| 466 _LOGGER.info('Deleting existing directory: %s', output_dir) |
| 467 _Shell('rmdir', '/S', '/Q', output_dir, dry_run=options.dry_run) |
| 468 |
| 469 if create_link: |
| 470 _LOGGER.info('Creating output junction: %s', output_dir) |
| 471 _Shell('mklink', '/J', output_dir, target_cache_dir, |
| 472 dry_run=options.dry_run) |
| 473 |
| 474 |
| 475 def _InstallRepository(options, repo): |
| 476 """Installs a repository as configured by the options. Assumes that the |
| 477 specified cache directory already exists. |
| 478 |
| 479 Returns True if the checkout was modified, False otherwise. |
| 480 """ |
| 481 |
| 482 _LOGGER.debug('Processing directories "%s" from repository "%s".', |
| 483 repo.remote_dirs, repo.repository) |
| 484 |
| 485 # Ensure the output directory's *parent* exists. |
| 486 output_dirname = os.path.dirname(repo.output_dir) |
| 487 output_basename = os.path.basename(repo.output_dir) |
| 488 _EnsureDirectoryExists(output_dirname, 'output', options.dry_run) |
| 489 |
| 490 # Get the properly cased names for the output directories. |
| 491 output_dirname = _GetCasedFilename(output_dirname) |
| 492 repo.output_dir = os.path.join(output_dirname, output_basename) |
| 493 |
| 494 # These are the 3 basic steps that need to occur. Depending on the state of |
| 495 # the checkout we may not need to perform all of them. We assume initially |
| 496 # that everything needs to be done, unless proven otherwise. |
| 497 create_checkout = True |
| 498 update_checkout = True |
| 499 |
| 500 # If the cache directory exists then lookup the repo and the revision and see |
| 501 # what needs to be updated. |
| 502 threads = [] |
| 503 if os.path.exists(repo.checkout_dir): |
| 504 keep_cache_dir = False |
| 505 |
| 506 # Only run these checks if we're not in 'force' mode. Otherwise, we |
| 507 # deliberately turf the cache directory and start from scratch. |
| 508 if not options.force and _IsGitCheckoutRoot(repo.checkout_dir): |
| 509 # Get the repo origin. |
| 510 repo_url = _GetGitOrigin(repo.checkout_dir) |
| 511 if (repo_url == repo.repository and |
| 512 _HasValidSparseCheckoutConfig(repo)): |
| 513 _LOGGER.debug('Checkout is for correct repository and subdirectory.') |
| 514 keep_cache_dir = True |
| 515 create_checkout = False |
| 516 |
| 517 # Get the checked out revision. |
| 518 revhash = _GetGitHead(repo.checkout_dir) |
| 519 if revhash == repo.revision: |
| 520 _LOGGER.debug('Checkout is already up to date.') |
| 521 update_checkout = False |
| 522 |
| 523 if not keep_cache_dir: |
| 524 # The old checkout directory is renamed and erased in a separate thread |
| 525 # so that the new checkout can start immediately. |
| 526 _LOGGER.info('Erasing stale checkout directory: %s', repo.checkout_dir) |
| 527 |
| 528 # Any existing junctions to this repo must be removed otherwise the |
| 529 # rename may fail. |
| 530 for d in repo.remote_dirs: |
| 531 j = os.path.abspath(os.path.join(repo.output_dir, d)) |
| 532 _RemoveOrphanedJunction(options, j) |
| 533 |
| 534 newpath = _RenameCheckout(repo.checkout_dir, options.dry_run) |
| 535 thread = threading.Thread(target=_DeleteCheckout, |
| 536 args=(newpath, options.dry_run)) |
| 537 threads.append(thread) |
| 538 thread.start() |
| 539 |
| 540 # Create and update the checkout as necessary. |
| 541 if create_checkout: |
| 542 _CreateCheckout(repo.checkout_dir, repo, options.dry_run) |
| 543 else: |
| 544 _LOGGER.debug('Reusing checkout directory: %s', repo.checkout_dir) |
| 545 if update_checkout: |
| 546 _UpdateCheckout(repo.checkout_dir, repo, options.dry_run) |
| 547 |
| 548 # Ensure the junctions exists. |
| 549 if repo.remote_dirs: |
| 550 for remote_dir in repo.remote_dirs: |
| 551 _EnsureJunction(repo.checkout_dir, remote_dir, options, repo) |
| 552 else: |
| 553 _EnsureJunction(repo.checkout_dir, '', options, repo) |
| 554 |
| 555 # Join any worker threads that are ongoing. |
| 556 for thread in threads: |
| 557 thread.join() |
| 558 |
| 559 # Return True if any modifications were made. |
| 560 return create_checkout or update_checkout |
| 561 |
| 562 |
| 563 def _WriteIfChanged(path, contents, dry_run): |
| 564 if os.path.exists(path): |
| 565 d = open(path, 'rb').read() |
| 566 if d == contents: |
| 567 _LOGGER.debug('Contents unchanged, not writing file: %s', path) |
| 568 return |
| 569 |
| 570 _LOGGER.info('Writing file: %s', path) |
| 571 if not dry_run: |
| 572 open(path, 'wb').write(contents) |
| 573 |
| 574 |
| 575 def _RecurseRepository(options, repo): |
| 576 """Recursively follows dependencies in the given repository.""" |
| 577 # Only run if there's an appropriate DEPS file. |
| 578 deps = os.path.isfile(os.path.join(repo.checkout_dir, 'DEPS')) |
| 579 gitdeps = os.path.isfile(os.path.join(repo.checkout_dir, '.DEPS.git')) |
| 580 if not deps and not gitdeps: |
| 581 _LOGGER.debug('No deps file found in repository: %s', repo.repository) |
| 582 return |
| 583 |
| 584 # Generate the .gclient solution file. |
| 585 cache_dir = os.path.dirname(os.path.abspath(repo.checkout_dir)) |
| 586 gclient_file = os.path.join(cache_dir, '.gclient') |
| 587 deps_file = 'DEPS' |
| 588 if gitdeps: |
| 589 deps_file = '.DEPS.git' |
| 590 solutions = [ |
| 591 { |
| 592 'name': 'src', |
| 593 'url': repo.repository, |
| 594 'managed': False, |
| 595 'custom_deps': [], |
| 596 'deps_file': deps_file, |
| 597 'safesync_url': '', |
| 598 } |
| 599 ] |
| 600 solutions = 'solutions=%s' % solutions.__repr__() |
| 601 _WriteIfChanged(gclient_file, solutions, options.dry_run) |
| 602 |
| 603 # Invoke 'gclient' on the sub-repository. |
| 604 _Shell('gclient', 'sync', cwd=repo.checkout_dir, dry_run=options.dry_run) |
| 605 |
| 606 |
| 607 def _FindGlobalVariableInAstTree(tree, name, functions=None): |
| 608 """Finds and evaluates to global assignment of the variables |name| in the |
| 609 AST |tree|. Will allow the evaluations of some functions as defined in |
| 610 |functions|. |
| 611 """ |
| 612 if functions is None: |
| 613 functions = {} |
| 614 |
| 615 class FunctionEvaluator(ast.NodeTransformer): |
| 616 """A tree transformer that evaluates permitted functions.""" |
| 617 |
| 618 def visit_BinOp(self, binop_node): |
| 619 """Is called for BinOp nodes. We only support string additions.""" |
| 620 if type(binop_node.op) != ast.Add: |
| 621 return binop_node |
| 622 left = ast.literal_eval(self.visit(binop_node.left)) |
| 623 right = ast.literal_eval(self.visit(binop_node.right)) |
| 624 value = left + right |
| 625 new_node = ast.Str(s=value) |
| 626 new_node = ast.copy_location(new_node, binop_node) |
| 627 return new_node |
| 628 |
| 629 def visit_Call(self, call_node): |
| 630 """Evaluates function calls that return a single string as output.""" |
| 631 func_name = call_node.func.id |
| 632 if func_name not in functions: |
| 633 return call_node |
| 634 func = functions[func_name] |
| 635 |
| 636 # Evaluate the arguments. We don't care about starargs, keywords or |
| 637 # kwargs. |
| 638 args = [ast.literal_eval(self.visit(arg)) for arg in |
| 639 call_node.args] |
| 640 |
| 641 # Now evaluate the function. |
| 642 value = func(*args) |
| 643 new_node = ast.Str(s=value) |
| 644 new_node = ast.copy_location(new_node, call_node) |
| 645 return new_node |
| 646 |
| 647 # Look for assignment nodes. |
| 648 for node in tree.body: |
| 649 if type(node) != ast.Assign: |
| 650 continue |
| 651 # Look for assignment in the 'store' context, to a variable with |
| 652 # the given name. |
| 653 for target in node.targets: |
| 654 if type(target) != ast.Name: |
| 655 continue |
| 656 if type(target.ctx) != ast.Store: |
| 657 continue |
| 658 if target.id == name: |
| 659 value = FunctionEvaluator().visit(node.value) |
| 660 value = ast.fix_missing_locations(value) |
| 661 value = ast.literal_eval(value) |
| 662 return value |
| 663 |
| 664 |
| 665 def _ParseDepsFile(path): |
| 666 """Parsed a DEPS-like file at the given |path|.""" |
| 667 # Utility function for performing variable expansions. |
| 668 vars_dict = {} |
| 669 def _Var(s): |
| 670 return vars_dict[s] |
| 671 |
| 672 contents = open(path, 'rb').read() |
| 673 tree = ast.parse(contents, path) |
| 674 vars_dict = _FindGlobalVariableInAstTree(tree, 'vars') |
| 675 deps_dict = _FindGlobalVariableInAstTree( |
| 676 tree, 'deps', functions={'Var': _Var}) |
| 677 return deps_dict |
| 678 |
| 679 |
| 680 def _RemoveFile(options, path): |
| 681 """Removes the provided file. If it doesn't exist, raises an Exception.""" |
| 682 _LOGGER.debug('Removing file: %s', path) |
| 683 if not os.path.isfile(path): |
| 684 raise Exception('Path does not exist: %s' % path) |
| 685 |
| 686 if not options.dry_run: |
| 687 os.remove(path) |
| 688 |
| 689 |
| 690 def _RemoveOrphanedJunction(options, junction): |
| 691 """Removes an orphaned junction at the path |junction|. If the path doesn't |
| 692 exist or is not a junction, raises an Exception. |
| 693 """ |
| 694 _LOGGER.debug('Removing orphaned junction: %s', junction) |
| 695 absdir = os.path.join(options.output_dir, junction) |
| 696 if not os.path.exists(absdir): |
| 697 _LOGGER.debug('Junction path does not exist, ignoring.') |
| 698 return |
| 699 if not _GetJunctionInfo(absdir): |
| 700 _LOGGER.error('Path is not a junction: %s', absdir) |
| 701 raise Exception() |
| 702 _Shell('rmdir', '/S', '/Q', absdir, dry_run=options.dry_run) |
| 703 |
| 704 reldir = os.path.dirname(junction) |
| 705 while reldir: |
| 706 absdir = os.path.join(options.output_dir, reldir) |
| 707 if os.listdir(absdir): |
| 708 return |
| 709 _LOGGER.debug('Removing empty parent directory of junction: %s', absdir) |
| 710 _Shell('rmdir', '/S', '/Q', absdir, dry_run=options.dry_run) |
| 711 reldir = os.path.dirname(reldir) |
| 712 |
| 713 |
| 714 def _GetCacheDirEntryVersion(path): |
| 715 """Returns the version of the cache directory entry, -1 if invalid.""" |
| 716 |
| 717 git = os.path.join(path, '.git') |
| 718 src = os.path.join(path, 'src') |
| 719 gclient = os.path.join(path, '.gclient') |
| 720 |
| 721 # Version 0 contains a '.git' directory and no '.gclient' entry. |
| 722 if os.path.isdir(git): |
| 723 if os.path.exists(gclient): |
| 724 return -1 |
| 725 return 0 |
| 726 |
| 727 # Version 1 contains a 'src' directory and no '.git' entry. |
| 728 if os.path.isdir(src): |
| 729 if os.path.exists(git): |
| 730 return -1 |
| 731 return 1 |
| 732 |
| 733 |
| 734 def _GetCacheDirEntries(cache_dir): |
| 735 """Returns the list of entries in the given |cache_dir|.""" |
| 736 entries = [] |
| 737 for path in os.listdir(cache_dir): |
| 738 if not re.match('^[a-z0-9]{32}$', path): |
| 739 continue |
| 740 entries.append(path) |
| 741 return entries |
| 742 |
| 743 |
| 744 def _GetCacheDirVersion(cache_dir): |
| 745 """Returns the version of the cache directory.""" |
| 746 # If it doesn't exist then it's clearly the latest version. |
| 747 if not os.path.exists(cache_dir): |
| 748 return 1 |
| 749 |
| 750 cache_version = None |
| 751 for path in _GetCacheDirEntries(cache_dir): |
| 752 repo = os.path.join(cache_dir, path) |
| 753 if not os.path.isdir(repo): |
| 754 return -1 |
| 755 |
| 756 entry_version = _GetCacheDirEntryVersion(repo) |
| 757 if entry_version == -1: |
| 758 return -1 |
| 759 |
| 760 if cache_version == None: |
| 761 cache_version = entry_version |
| 762 else: |
| 763 if cache_version != entry_version: |
| 764 return -1 |
| 765 |
| 766 # If there are no entries in the cache it may as well be the latest version. |
| 767 if cache_version is None: |
| 768 return 1 |
| 769 |
| 770 return cache_version |
| 771 |
| 772 |
| 773 def _GetJunctionStatePath(options): |
| 774 """Returns the junction state file path.""" |
| 775 return os.path.join(options.cache_dir, '.gitdeps_junctions') |
| 776 |
| 777 |
| 778 def _ReadJunctions(options): |
| 779 """Reads the list of junctions as a dictionary.""" |
| 780 state_path = _GetJunctionStatePath(options) |
| 781 old_junctions = {} |
| 782 if os.path.exists(state_path): |
| 783 _LOGGER.debug('Loading list of existing junctions.') |
| 784 for j in open(state_path, 'rb'): |
| 785 old_junctions[j.strip()] = True |
| 786 |
| 787 return old_junctions |
| 788 |
| 789 |
| 790 def _Rename(src, dst, dry_run): |
| 791 _LOGGER.debug('Renaming "%s" to "%s".', src, dst) |
| 792 if not dry_run: |
| 793 os.rename(src, dst) |
| 794 |
| 795 |
| 796 def _UpgradeCacheDir(options): |
| 797 """Upgrades the cache directory format to the most modern layout. |
| 798 |
| 799 Returns true on success, false otherwise. |
| 800 """ |
| 801 cache_version = _GetCacheDirVersion(options.cache_dir) |
| 802 if cache_version == 1: |
| 803 _LOGGER.debug('No cache directory upgrade required.') |
| 804 return |
| 805 |
| 806 _LOGGER.debug('Upgrading cache directory from version 0 to 1.') |
| 807 |
| 808 _LOGGER.debug('Removing all junctions.') |
| 809 junctions = _ReadJunctions(options).keys() |
| 810 junctions = sorted(junctions, key=lambda j: len(j), reverse=True) |
| 811 for junction in junctions: |
| 812 _RemoveOrphanedJunction(options, junction) |
| 813 _RemoveFile(options, _GetJunctionStatePath(options)) |
| 814 |
| 815 for entry in _GetCacheDirEntries(options.cache_dir): |
| 816 _LOGGER.debug('Upgrading cache entry "%s".', entry) |
| 817 tmp_entry = os.path.abspath(os.path.join( |
| 818 options.cache_dir, |
| 819 'TMP%d-%04d' % (os.getpid(), random.randint(0, 999)))) |
| 820 abs_entry = os.path.abspath(os.path.join(options.cache_dir, entry)) |
| 821 src = os.path.join(abs_entry, 'src') |
| 822 _Rename(abs_entry, tmp_entry, options.dry_run) |
| 823 _EnsureDirectoryExists(abs_entry, 'cache entry', options.dry_run) |
| 824 _Rename(tmp_entry, src, options.dry_run) |
| 825 |
| 826 if options.dry_run: |
| 827 _LOGGER.debug('Cache needs upgrading, unable to further simulate dry-run.') |
| 828 raise Exception("") |
| 829 |
| 830 |
| 831 def main(): |
| 832 options, args = _ParseCommandLine() |
| 833 |
| 834 # Upgrade the cache directory if necessary. |
| 835 _UpgradeCacheDir(options) |
| 836 |
| 837 # Ensure the cache directory exists and get the full properly cased path to |
| 838 # it. |
| 839 _EnsureDirectoryExists(options.cache_dir, 'cache', options.dry_run) |
| 840 options.cache_dir = _GetCasedFilename(options.cache_dir) |
| 841 |
| 842 # Read junctions that have been written in previous runs. |
| 843 state_path = _GetJunctionStatePath(options) |
| 844 old_junctions = _ReadJunctions(options) |
| 845 |
| 846 # Parse each deps file in order, and extract the dependencies, looking for |
| 847 # conflicts in the output directories. |
| 848 output_dirs = {} |
| 849 all_deps = [] |
| 850 for deps_file in args: |
| 851 deps = _ParseDepsFile(deps_file) |
| 852 for key, value in deps.iteritems(): |
| 853 repo_options = _ParseRepoOptions( |
| 854 options.cache_dir, options.output_dir, deps_file, key, value) |
| 855 if repo_options.output_dir in output_dirs: |
| 856 other_repo_options = output_dirs[repo_options.output_dir] |
| 857 _LOGGER.error('Conflicting output directory: %s', |
| 858 repo_options.output_dir) |
| 859 _LOGGER.error('First specified in file: %s', |
| 860 other_repo_options.deps_file) |
| 861 _LOGGER.error('And then specified in file: %s', repo_options.deps_file) |
| 862 output_dirs[repo_options.output_dir] = repo_options |
| 863 all_deps.append(repo_options) |
| 864 output_dirs = {} |
| 865 |
| 866 # Handle each dependency, in order of shortest path names first. This ensures |
| 867 # that nested dependencies are handled properly. |
| 868 checkout_dirs = {} |
| 869 deps = sorted(all_deps, key=lambda x: len(x.deps_file)) |
| 870 junctions = [] |
| 871 for repo in all_deps: |
| 872 changes_made = _InstallRepository(options, repo) |
| 873 checkout_dirs[repo.checkout_dir] = changes_made |
| 874 |
| 875 new_junction_dirs = repo.remote_dirs if repo.remote_dirs else [''] |
| 876 for new_junction_dir in new_junction_dirs: |
| 877 junction = os.path.relpath( |
| 878 os.path.join(repo.output_dir, new_junction_dir), |
| 879 options.output_dir) |
| 880 old_junctions.pop(junction, None) |
| 881 # Write each junction as we create it. This allows for recovery from |
| 882 # partial runs. |
| 883 if not options.dry_run: |
| 884 open(state_path, 'ab').write(junction + '\n') |
| 885 junctions.append(junction) |
| 886 |
| 887 # Clean up orphaned junctions if there are any. |
| 888 if old_junctions: |
| 889 _LOGGER.debug('Removing orphaned junctions.') |
| 890 for j in old_junctions.iterkeys(): |
| 891 _RemoveOrphanedJunction(options, j) |
| 892 |
| 893 # Output the final list of junctions. |
| 894 _LOGGER.debug('Writing final list of junctions.') |
| 895 if not options.dry_run: |
| 896 with open(state_path, 'wb') as io: |
| 897 for j in sorted(junctions): |
| 898 io.write(j) |
| 899 io.write('\n') |
| 900 |
| 901 # Iterate all directories in the cache directory. Any that we didn't |
| 902 # specifically create or update should be cleaned up. Do this in parallel |
| 903 # so things are cleaned up as soon as possible. |
| 904 threads = [] |
| 905 for path in glob.glob(os.path.join(options.cache_dir, '*')): |
| 906 if os.path.join(path, 'src') not in checkout_dirs: |
| 907 _LOGGER.debug('Erasing orphaned checkout directory: %s', path) |
| 908 thread = threading.Thread(target=_DeleteCheckout, |
| 909 args=(path, options.dry_run)) |
| 910 threads.append(thread) |
| 911 thread.start() |
| 912 for thread in threads: |
| 913 thread.join() |
| 914 |
| 915 # Recursively process other dependencies. |
| 916 for repo in all_deps: |
| 917 if not repo.recurse: |
| 918 continue |
| 919 if not checkout_dirs[repo.checkout_dir] and not options.force: |
| 920 continue |
| 921 _RecurseRepository(options, repo) |
| 922 |
| 923 return |
| 924 |
| 925 |
| 926 if __name__ == '__main__': |
| 927 main() |
OLD | NEW |