Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Convert SVN based DEPS into .DEPS.git for use with NewGit.""" | 6 """Convert SVN based DEPS into .DEPS.git for use with NewGit.""" |
| 7 | 7 |
| 8 import collections | 8 import collections |
| 9 import json | 9 import json |
| 10 import optparse | 10 import optparse |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 65 else: | 65 else: |
| 66 refspec = 'refs/remotes/branch-heads/' + svn_branch_name | 66 refspec = 'refs/remotes/branch-heads/' + svn_branch_name |
| 67 else: | 67 else: |
| 68 if mirror: | 68 if mirror: |
| 69 refspec = 'refs/heads/master' | 69 refspec = 'refs/heads/master' |
| 70 else: | 70 else: |
| 71 refspec = 'refs/remotes/origin/master' | 71 refspec = 'refs/remotes/origin/master' |
| 72 | 72 |
| 73 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url) | 73 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url) |
| 74 | 74 |
| 75 def ConvertDepsToGit(deps, options, deps_vars, svn_deps_vars): | 75 |
| 76 def ConvertDepsToGit(deps, options, deps_vars): | |
| 76 """Convert a 'deps' section in a DEPS file from SVN to Git.""" | 77 """Convert a 'deps' section in a DEPS file from SVN to Git.""" |
| 77 new_deps = {} | 78 new_deps = {} |
| 78 bad_git_urls = set([]) | 79 bad_git_urls = set([]) |
| 79 | 80 |
| 80 svn_to_git_objs = [svn_to_git_public] | 81 svn_to_git_objs = [svn_to_git_public] |
| 81 if options.extra_rules: | 82 if options.extra_rules: |
|
szager1
2014/03/07 23:04:53
This is where extra svn_to_git_*.py modules are pu
| |
| 82 rules_dir, rules_file = os.path.split(options.extra_rules) | 83 rules_dir, rules_file = os.path.split(options.extra_rules) |
| 83 rules_file_base = os.path.splitext(rules_file)[0] | 84 rules_file_base = os.path.splitext(rules_file)[0] |
| 84 sys.path.insert(0, rules_dir) | 85 sys.path.insert(0, rules_dir) |
| 85 svn_to_git_objs.insert(0, __import__(rules_file_base)) | 86 svn_to_git_objs.insert(0, __import__(rules_file_base)) |
| 86 | 87 |
| 87 deps_overrides = {} | |
| 88 # Allow extra_rules file to override rules in public file. | |
| 89 for svn_to_git_obj in reversed(svn_to_git_objs): | |
| 90 deps_overrides.update(getattr(svn_to_git_obj, 'DEPS_OVERRIDES', {})) | |
| 91 | |
| 92 # Populate our deps list. | 88 # Populate our deps list. |
| 93 deps_to_process = {} | 89 deps_to_process = {} |
| 94 for dep, dep_url in deps.iteritems(): | 90 for dep, dep_url in deps.iteritems(): |
| 95 if not dep_url: # dep is 'None' and emitted to exclude the dep | 91 if not dep_url: # dep is 'None' and emitted to exclude the dep |
| 96 new_deps[dep] = None | 92 new_deps[dep] = None |
| 97 continue | 93 continue |
| 98 | 94 |
| 99 # Get the URL and the revision/hash for this dependency. | 95 # Get the URL and the revision/hash for this dependency. |
| 100 dep_url, dep_rev = SplitScmUrl(deps[dep]) | 96 dep_url, dep_rev = SplitScmUrl(deps[dep]) |
| 101 | 97 |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 147 except Queue.Empty: | 143 except Queue.Empty: |
| 148 sec_since += 1 | 144 sec_since += 1 |
| 149 line = ('Main> Heartbeat ping. We are still alive!! ' | 145 line = ('Main> Heartbeat ping. We are still alive!! ' |
| 150 'Seconds since last output: %d sec' % sec_since) | 146 'Seconds since last output: %d sec' % sec_since) |
| 151 if line is None: | 147 if line is None: |
| 152 num_threads -= 1 | 148 num_threads -= 1 |
| 153 else: | 149 else: |
| 154 print line | 150 print line |
| 155 pool.join() | 151 pool.join() |
| 156 | 152 |
| 157 | |
| 158 for dep, items in deps_to_process.iteritems(): | 153 for dep, items in deps_to_process.iteritems(): |
| 159 git_url, dep_url, path, git_host, dep_rev, svn_branch = items | 154 git_url, dep_url, path, git_host, dep_rev, svn_branch = items |
| 160 if options.verify: | 155 if options.verify: |
| 161 delay = 0.5 | 156 delay = 0.5 |
| 162 success = False | 157 success = False |
| 163 for try_index in range(1, 6): | 158 for try_index in range(1, 6): |
| 164 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index), | 159 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index), |
| 165 if git_tools.Ping(git_url, verbose=True): | 160 if git_tools.Ping(git_url, verbose=True): |
| 166 print >> sys.stderr, ' success' | 161 print >> sys.stderr, ' success' |
| 167 success = True | 162 success = True |
| 168 break | 163 break |
| 169 | 164 |
| 170 print >> sys.stderr, ' failure' | 165 print >> sys.stderr, ' failure' |
| 171 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay | 166 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay |
| 172 time.sleep(delay) | 167 time.sleep(delay) |
| 173 delay *= 2 | 168 delay *= 2 |
| 174 | 169 |
| 175 if not success: | 170 if not success: |
| 176 bad_git_urls.update([git_url]) | 171 bad_git_urls.update([git_url]) |
| 177 | 172 |
| 178 # Get the Git hash based off the SVN rev. | 173 # Get the Git hash based off the SVN rev. |
| 179 git_hash = '' | 174 git_hash = '' |
| 180 if dep_rev != 'HEAD': | 175 if dep_rev != 'HEAD': |
| 181 if dep in deps_overrides: | 176 # Pass-through the hash for Git repositories. Resolve the hash for |
| 182 # Transfer any required variables over from SVN DEPS. | 177 # subversion repositories. |
| 183 if not deps_overrides[dep] in svn_deps_vars: | 178 if dep_url.endswith('.git'): |
| 184 raise Exception('Missing DEPS variable: %s' % deps_overrides[dep]) | 179 git_hash = '@%s' % dep_rev |
| 185 deps_vars[deps_overrides[dep]] = ( | |
| 186 '@' + svn_deps_vars[deps_overrides[dep]].lstrip('@')) | |
| 187 # Tag this variable as needing a transform by Varify() later. | |
| 188 git_hash = '%s_%s' % (deps_utils.VARIFY_MARKER_TAG_PREFIX, | |
| 189 deps_overrides[dep]) | |
| 190 else: | 180 else: |
| 191 # Pass-through the hash for Git repositories. Resolve the hash for | 181 git_hash = '@%s' % SvnRevToGitHash( |
| 192 # subversion repositories. | 182 dep_rev, git_url, options.repos, options.workspace, path, |
| 193 if dep_url.endswith('.git'): | 183 git_host, svn_branch, options.cache_dir) |
| 194 git_hash = '@%s' % dep_rev | |
| 195 else: | |
| 196 git_hash = '@%s' % SvnRevToGitHash( | |
| 197 dep_rev, git_url, options.repos, options.workspace, path, | |
| 198 git_host, svn_branch, options.cache_dir) | |
| 199 | 184 |
| 200 # If this is webkit, we need to add the var for the hash. | 185 # If this is webkit, we need to add the var for the hash. |
| 201 if dep == 'src/third_party/WebKit' and dep_rev: | 186 if dep == 'src/third_party/WebKit' and dep_rev: |
| 202 deps_vars['webkit_rev'] = git_hash | 187 deps_vars['webkit_rev'] = git_hash |
| 203 git_hash = 'VAR_WEBKIT_REV' | 188 git_hash = 'VAR_WEBKIT_REV' |
| 204 | 189 |
| 205 # Add this Git dep to the new deps. | 190 # Add this Git dep to the new deps. |
| 206 new_deps[path] = '%s%s' % (git_url, git_hash) | 191 new_deps[path] = '%s%s' % (git_url, git_hash) |
| 207 | 192 |
| 208 return new_deps, bad_git_urls | 193 return new_deps, bad_git_urls |
| (...skipping 17 matching lines...) Expand all Loading... | |
| 226 help='top level of a gclient git cache diretory.') | 211 help='top level of a gclient git cache diretory.') |
| 227 parser.add_option('-s', '--shallow', action='store_true', | 212 parser.add_option('-s', '--shallow', action='store_true', |
| 228 help='Use shallow checkouts when populating cache dirs.') | 213 help='Use shallow checkouts when populating cache dirs.') |
| 229 parser.add_option('--verify', action='store_true', | 214 parser.add_option('--verify', action='store_true', |
| 230 help='ping each Git repo to make sure it exists') | 215 help='ping each Git repo to make sure it exists') |
| 231 parser.add_option('--json', | 216 parser.add_option('--json', |
| 232 help='path to a JSON file for machine-readable output') | 217 help='path to a JSON file for machine-readable output') |
| 233 options = parser.parse_args()[0] | 218 options = parser.parse_args()[0] |
| 234 | 219 |
| 235 # Get the content of the DEPS file. | 220 # Get the content of the DEPS file. |
| 236 deps_content = deps_utils.GetDepsContent(options.deps) | 221 deps, deps_os, include_rules, skip_child_includes, hooks = ( |
| 237 (deps, deps_os, include_rules, skip_child_includes, hooks, | 222 deps_utils.GetDepsContent(options.deps)) |
| 238 svn_deps_vars) = deps_content | |
| 239 | 223 |
| 240 if options.extra_rules and options.type: | 224 if options.extra_rules and options.type: |
| 241 parser.error('Can\'t specify type and extra-rules at the same time.') | 225 parser.error('Can\'t specify type and extra-rules at the same time.') |
| 242 elif options.type: | 226 elif options.type: |
| 243 options.extra_rules = os.path.join( | 227 options.extra_rules = os.path.join( |
| 244 os.path.abspath(os.path.dirname(__file__)), | 228 os.path.abspath(os.path.dirname(__file__)), |
| 245 'svn_to_git_%s.py' % options.type) | 229 'svn_to_git_%s.py' % options.type) |
| 246 if options.cache_dir and options.repos: | 230 if options.cache_dir and options.repos: |
| 247 parser.error('Can\'t specify both cache_dir and repos at the same time.') | 231 parser.error('Can\'t specify both cache_dir and repos at the same time.') |
| 248 if options.shallow and not options.cache_dir: | 232 if options.shallow and not options.cache_dir: |
| 249 parser.error('--shallow only supported with --cache_dir.') | 233 parser.error('--shallow only supported with --cache_dir.') |
| 250 | 234 |
| 251 if options.cache_dir: | 235 if options.cache_dir: |
| 252 options.cache_dir = os.path.abspath(options.cache_dir) | 236 options.cache_dir = os.path.abspath(options.cache_dir) |
| 253 | 237 |
| 254 if options.extra_rules and not os.path.exists(options.extra_rules): | 238 if options.extra_rules and not os.path.exists(options.extra_rules): |
| 255 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules) | 239 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules) |
| 256 | 240 |
| 257 # Create a var containing the Git and Webkit URL, this will make it easy for | 241 # Create a var containing the Git and Webkit URL, this will make it easy for |
| 258 # people to use a mirror instead. | 242 # people to use a mirror instead. |
| 259 git_url = 'https://chromium.googlesource.com' | 243 git_url = 'https://chromium.googlesource.com' |
| 260 deps_vars = { | 244 deps_vars = { |
| 261 'git_url': git_url, | 245 'git_url': git_url, |
| 262 'webkit_url': git_url + '/chromium/blink.git', | 246 'webkit_url': git_url + '/chromium/blink.git', |
| 263 } | 247 } |
| 264 | 248 |
| 265 # Convert the DEPS file to Git. | 249 # Convert the DEPS file to Git. |
| 266 deps, baddeps = ConvertDepsToGit(deps, options, deps_vars, svn_deps_vars) | 250 deps, baddeps = ConvertDepsToGit(deps, options, deps_vars) |
|
szager1
2014/03/07 22:43:53
There might be other uses of svn_deps_vars besides
DaleCurtis
2014/03/07 22:50:51
None exist at this time AFAICT so it seems better
szager1
2014/03/07 23:04:53
Yes, there are svn_to_git_*.py modules for interna
mmoss
2014/03/07 23:27:18
The only one I'm aware of doesn't have and DEPS_OV
DaleCurtis
2014/03/07 23:30:26
Yeah, I checked that one and didn't see any.
| |
| 267 for os_dep in deps_os: | 251 for os_dep in deps_os: |
| 268 deps_os[os_dep], os_bad_deps = ConvertDepsToGit( | 252 deps_os[os_dep], os_bad_deps = ConvertDepsToGit( |
| 269 deps_os[os_dep], options, deps_vars, svn_deps_vars) | 253 deps_os[os_dep], options, deps_vars) |
| 270 baddeps = baddeps.union(os_bad_deps) | 254 baddeps = baddeps.union(os_bad_deps) |
| 271 | 255 |
| 272 if options.json: | 256 if options.json: |
| 273 with open(options.json, 'w') as f: | 257 with open(options.json, 'w') as f: |
| 274 json.dump(list(baddeps), f, sort_keys=True, indent=2) | 258 json.dump(list(baddeps), f, sort_keys=True, indent=2) |
| 275 | 259 |
| 276 if baddeps: | 260 if baddeps: |
| 277 print >> sys.stderr, ('\nUnable to resolve the following repositories. ' | 261 print >> sys.stderr, ('\nUnable to resolve the following repositories. ' |
| 278 'Please make sure\nthat any svn URLs have a git mirror associated with ' | 262 'Please make sure\nthat any svn URLs have a git mirror associated with ' |
| 279 'them.\nTo see the exact error, run `git ls-remote [repository]` where' | 263 'them.\nTo see the exact error, run `git ls-remote [repository]` where' |
| (...skipping 10 matching lines...) Expand all Loading... | |
| 290 return 0 | 274 return 0 |
| 291 | 275 |
| 292 # Write the DEPS file to disk. | 276 # Write the DEPS file to disk. |
| 293 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules, | 277 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules, |
| 294 skip_child_includes, hooks) | 278 skip_child_includes, hooks) |
| 295 return 0 | 279 return 0 |
| 296 | 280 |
| 297 | 281 |
| 298 if '__main__' == __name__: | 282 if '__main__' == __name__: |
| 299 sys.exit(main()) | 283 sys.exit(main()) |
| OLD | NEW |