OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Convert SVN based DEPS into .DEPS.git for use with NewGit.""" | 6 """Convert SVN based DEPS into .DEPS.git for use with NewGit.""" |
7 | 7 |
8 import collections | 8 import collections |
9 import json | 9 import json |
10 import optparse | 10 import optparse |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
65 else: | 65 else: |
66 refspec = 'refs/remotes/branch-heads/' + svn_branch_name | 66 refspec = 'refs/remotes/branch-heads/' + svn_branch_name |
67 else: | 67 else: |
68 if mirror: | 68 if mirror: |
69 refspec = 'refs/heads/master' | 69 refspec = 'refs/heads/master' |
70 else: | 70 else: |
71 refspec = 'refs/remotes/origin/master' | 71 refspec = 'refs/remotes/origin/master' |
72 | 72 |
73 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url) | 73 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url) |
74 | 74 |
75 def ConvertDepsToGit(deps, options, deps_vars, svn_deps_vars): | 75 |
76 def ConvertDepsToGit(deps, options, deps_vars): | |
76 """Convert a 'deps' section in a DEPS file from SVN to Git.""" | 77 """Convert a 'deps' section in a DEPS file from SVN to Git.""" |
77 new_deps = {} | 78 new_deps = {} |
78 bad_git_urls = set([]) | 79 bad_git_urls = set([]) |
79 | 80 |
80 svn_to_git_objs = [svn_to_git_public] | 81 svn_to_git_objs = [svn_to_git_public] |
81 if options.extra_rules: | 82 if options.extra_rules: |
szager1
2014/03/07 23:04:53
This is where extra svn_to_git_*.py modules are pu
| |
82 rules_dir, rules_file = os.path.split(options.extra_rules) | 83 rules_dir, rules_file = os.path.split(options.extra_rules) |
83 rules_file_base = os.path.splitext(rules_file)[0] | 84 rules_file_base = os.path.splitext(rules_file)[0] |
84 sys.path.insert(0, rules_dir) | 85 sys.path.insert(0, rules_dir) |
85 svn_to_git_objs.insert(0, __import__(rules_file_base)) | 86 svn_to_git_objs.insert(0, __import__(rules_file_base)) |
86 | 87 |
87 deps_overrides = {} | |
88 # Allow extra_rules file to override rules in public file. | |
89 for svn_to_git_obj in reversed(svn_to_git_objs): | |
90 deps_overrides.update(getattr(svn_to_git_obj, 'DEPS_OVERRIDES', {})) | |
91 | |
92 # Populate our deps list. | 88 # Populate our deps list. |
93 deps_to_process = {} | 89 deps_to_process = {} |
94 for dep, dep_url in deps.iteritems(): | 90 for dep, dep_url in deps.iteritems(): |
95 if not dep_url: # dep is 'None' and emitted to exclude the dep | 91 if not dep_url: # dep is 'None' and emitted to exclude the dep |
96 new_deps[dep] = None | 92 new_deps[dep] = None |
97 continue | 93 continue |
98 | 94 |
99 # Get the URL and the revision/hash for this dependency. | 95 # Get the URL and the revision/hash for this dependency. |
100 dep_url, dep_rev = SplitScmUrl(deps[dep]) | 96 dep_url, dep_rev = SplitScmUrl(deps[dep]) |
101 | 97 |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
147 except Queue.Empty: | 143 except Queue.Empty: |
148 sec_since += 1 | 144 sec_since += 1 |
149 line = ('Main> Heartbeat ping. We are still alive!! ' | 145 line = ('Main> Heartbeat ping. We are still alive!! ' |
150 'Seconds since last output: %d sec' % sec_since) | 146 'Seconds since last output: %d sec' % sec_since) |
151 if line is None: | 147 if line is None: |
152 num_threads -= 1 | 148 num_threads -= 1 |
153 else: | 149 else: |
154 print line | 150 print line |
155 pool.join() | 151 pool.join() |
156 | 152 |
157 | |
158 for dep, items in deps_to_process.iteritems(): | 153 for dep, items in deps_to_process.iteritems(): |
159 git_url, dep_url, path, git_host, dep_rev, svn_branch = items | 154 git_url, dep_url, path, git_host, dep_rev, svn_branch = items |
160 if options.verify: | 155 if options.verify: |
161 delay = 0.5 | 156 delay = 0.5 |
162 success = False | 157 success = False |
163 for try_index in range(1, 6): | 158 for try_index in range(1, 6): |
164 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index), | 159 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index), |
165 if git_tools.Ping(git_url, verbose=True): | 160 if git_tools.Ping(git_url, verbose=True): |
166 print >> sys.stderr, ' success' | 161 print >> sys.stderr, ' success' |
167 success = True | 162 success = True |
168 break | 163 break |
169 | 164 |
170 print >> sys.stderr, ' failure' | 165 print >> sys.stderr, ' failure' |
171 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay | 166 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay |
172 time.sleep(delay) | 167 time.sleep(delay) |
173 delay *= 2 | 168 delay *= 2 |
174 | 169 |
175 if not success: | 170 if not success: |
176 bad_git_urls.update([git_url]) | 171 bad_git_urls.update([git_url]) |
177 | 172 |
178 # Get the Git hash based off the SVN rev. | 173 # Get the Git hash based off the SVN rev. |
179 git_hash = '' | 174 git_hash = '' |
180 if dep_rev != 'HEAD': | 175 if dep_rev != 'HEAD': |
181 if dep in deps_overrides: | 176 # Pass-through the hash for Git repositories. Resolve the hash for |
182 # Transfer any required variables over from SVN DEPS. | 177 # subversion repositories. |
183 if not deps_overrides[dep] in svn_deps_vars: | 178 if dep_url.endswith('.git'): |
184 raise Exception('Missing DEPS variable: %s' % deps_overrides[dep]) | 179 git_hash = '@%s' % dep_rev |
185 deps_vars[deps_overrides[dep]] = ( | |
186 '@' + svn_deps_vars[deps_overrides[dep]].lstrip('@')) | |
187 # Tag this variable as needing a transform by Varify() later. | |
188 git_hash = '%s_%s' % (deps_utils.VARIFY_MARKER_TAG_PREFIX, | |
189 deps_overrides[dep]) | |
190 else: | 180 else: |
191 # Pass-through the hash for Git repositories. Resolve the hash for | 181 git_hash = '@%s' % SvnRevToGitHash( |
192 # subversion repositories. | 182 dep_rev, git_url, options.repos, options.workspace, path, |
193 if dep_url.endswith('.git'): | 183 git_host, svn_branch, options.cache_dir) |
194 git_hash = '@%s' % dep_rev | |
195 else: | |
196 git_hash = '@%s' % SvnRevToGitHash( | |
197 dep_rev, git_url, options.repos, options.workspace, path, | |
198 git_host, svn_branch, options.cache_dir) | |
199 | 184 |
200 # If this is webkit, we need to add the var for the hash. | 185 # If this is webkit, we need to add the var for the hash. |
201 if dep == 'src/third_party/WebKit' and dep_rev: | 186 if dep == 'src/third_party/WebKit' and dep_rev: |
202 deps_vars['webkit_rev'] = git_hash | 187 deps_vars['webkit_rev'] = git_hash |
203 git_hash = 'VAR_WEBKIT_REV' | 188 git_hash = 'VAR_WEBKIT_REV' |
204 | 189 |
205 # Add this Git dep to the new deps. | 190 # Add this Git dep to the new deps. |
206 new_deps[path] = '%s%s' % (git_url, git_hash) | 191 new_deps[path] = '%s%s' % (git_url, git_hash) |
207 | 192 |
208 return new_deps, bad_git_urls | 193 return new_deps, bad_git_urls |
(...skipping 17 matching lines...) Expand all Loading... | |
226 help='top level of a gclient git cache diretory.') | 211 help='top level of a gclient git cache diretory.') |
227 parser.add_option('-s', '--shallow', action='store_true', | 212 parser.add_option('-s', '--shallow', action='store_true', |
228 help='Use shallow checkouts when populating cache dirs.') | 213 help='Use shallow checkouts when populating cache dirs.') |
229 parser.add_option('--verify', action='store_true', | 214 parser.add_option('--verify', action='store_true', |
230 help='ping each Git repo to make sure it exists') | 215 help='ping each Git repo to make sure it exists') |
231 parser.add_option('--json', | 216 parser.add_option('--json', |
232 help='path to a JSON file for machine-readable output') | 217 help='path to a JSON file for machine-readable output') |
233 options = parser.parse_args()[0] | 218 options = parser.parse_args()[0] |
234 | 219 |
235 # Get the content of the DEPS file. | 220 # Get the content of the DEPS file. |
236 deps_content = deps_utils.GetDepsContent(options.deps) | 221 deps, deps_os, include_rules, skip_child_includes, hooks = ( |
237 (deps, deps_os, include_rules, skip_child_includes, hooks, | 222 deps_utils.GetDepsContent(options.deps)) |
238 svn_deps_vars) = deps_content | |
239 | 223 |
240 if options.extra_rules and options.type: | 224 if options.extra_rules and options.type: |
241 parser.error('Can\'t specify type and extra-rules at the same time.') | 225 parser.error('Can\'t specify type and extra-rules at the same time.') |
242 elif options.type: | 226 elif options.type: |
243 options.extra_rules = os.path.join( | 227 options.extra_rules = os.path.join( |
244 os.path.abspath(os.path.dirname(__file__)), | 228 os.path.abspath(os.path.dirname(__file__)), |
245 'svn_to_git_%s.py' % options.type) | 229 'svn_to_git_%s.py' % options.type) |
246 if options.cache_dir and options.repos: | 230 if options.cache_dir and options.repos: |
247 parser.error('Can\'t specify both cache_dir and repos at the same time.') | 231 parser.error('Can\'t specify both cache_dir and repos at the same time.') |
248 if options.shallow and not options.cache_dir: | 232 if options.shallow and not options.cache_dir: |
249 parser.error('--shallow only supported with --cache_dir.') | 233 parser.error('--shallow only supported with --cache_dir.') |
250 | 234 |
251 if options.cache_dir: | 235 if options.cache_dir: |
252 options.cache_dir = os.path.abspath(options.cache_dir) | 236 options.cache_dir = os.path.abspath(options.cache_dir) |
253 | 237 |
254 if options.extra_rules and not os.path.exists(options.extra_rules): | 238 if options.extra_rules and not os.path.exists(options.extra_rules): |
255 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules) | 239 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules) |
256 | 240 |
257 # Create a var containing the Git and Webkit URL, this will make it easy for | 241 # Create a var containing the Git and Webkit URL, this will make it easy for |
258 # people to use a mirror instead. | 242 # people to use a mirror instead. |
259 git_url = 'https://chromium.googlesource.com' | 243 git_url = 'https://chromium.googlesource.com' |
260 deps_vars = { | 244 deps_vars = { |
261 'git_url': git_url, | 245 'git_url': git_url, |
262 'webkit_url': git_url + '/chromium/blink.git', | 246 'webkit_url': git_url + '/chromium/blink.git', |
263 } | 247 } |
264 | 248 |
265 # Convert the DEPS file to Git. | 249 # Convert the DEPS file to Git. |
266 deps, baddeps = ConvertDepsToGit(deps, options, deps_vars, svn_deps_vars) | 250 deps, baddeps = ConvertDepsToGit(deps, options, deps_vars) |
szager1
2014/03/07 22:43:53
There might be other uses of svn_deps_vars besides
DaleCurtis
2014/03/07 22:50:51
None exist at this time AFAICT so it seems better
szager1
2014/03/07 23:04:53
Yes, there are svn_to_git_*.py modules for interna
mmoss
2014/03/07 23:27:18
The only one I'm aware of doesn't have and DEPS_OV
DaleCurtis
2014/03/07 23:30:26
Yeah, I checked that one and didn't see any.
| |
267 for os_dep in deps_os: | 251 for os_dep in deps_os: |
268 deps_os[os_dep], os_bad_deps = ConvertDepsToGit( | 252 deps_os[os_dep], os_bad_deps = ConvertDepsToGit( |
269 deps_os[os_dep], options, deps_vars, svn_deps_vars) | 253 deps_os[os_dep], options, deps_vars) |
270 baddeps = baddeps.union(os_bad_deps) | 254 baddeps = baddeps.union(os_bad_deps) |
271 | 255 |
272 if options.json: | 256 if options.json: |
273 with open(options.json, 'w') as f: | 257 with open(options.json, 'w') as f: |
274 json.dump(list(baddeps), f, sort_keys=True, indent=2) | 258 json.dump(list(baddeps), f, sort_keys=True, indent=2) |
275 | 259 |
276 if baddeps: | 260 if baddeps: |
277 print >> sys.stderr, ('\nUnable to resolve the following repositories. ' | 261 print >> sys.stderr, ('\nUnable to resolve the following repositories. ' |
278 'Please make sure\nthat any svn URLs have a git mirror associated with ' | 262 'Please make sure\nthat any svn URLs have a git mirror associated with ' |
279 'them.\nTo see the exact error, run `git ls-remote [repository]` where' | 263 'them.\nTo see the exact error, run `git ls-remote [repository]` where' |
(...skipping 10 matching lines...) Expand all Loading... | |
290 return 0 | 274 return 0 |
291 | 275 |
292 # Write the DEPS file to disk. | 276 # Write the DEPS file to disk. |
293 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules, | 277 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules, |
294 skip_child_includes, hooks) | 278 skip_child_includes, hooks) |
295 return 0 | 279 return 0 |
296 | 280 |
297 | 281 |
298 if '__main__' == __name__: | 282 if '__main__' == __name__: |
299 sys.exit(main()) | 283 sys.exit(main()) |
OLD | NEW |