OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Convert SVN based DEPS into .DEPS.git for use with NewGit.""" | 6 """Convert SVN based DEPS into .DEPS.git for use with NewGit.""" |
7 | 7 |
8 import collections | 8 import collections |
9 import json | 9 import json |
10 import optparse | 10 import optparse |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
104 if (git_url.startswith('https://chromium.googlesource.com/external/pefile') | 104 if (git_url.startswith('https://chromium.googlesource.com/external/pefile') |
105 and int(svn_rev) == 63): | 105 and int(svn_rev) == 63): |
106 return '1ceaa279daa62b71e3431e58f68be6a96dd1519a' | 106 return '1ceaa279daa62b71e3431e58f68be6a96dd1519a' |
107 | 107 |
108 try: | 108 try: |
109 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url) | 109 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url) |
110 except Exception: | 110 except Exception: |
111 print >> sys.stderr, '%s <-> ERROR' % git_repo_path | 111 print >> sys.stderr, '%s <-> ERROR' % git_repo_path |
112 raise | 112 raise |
113 | 113 |
114 def ConvertDepsToGit(deps, options, deps_vars, svn_deps_vars, svn_to_git_objs, | 114 def ConvertDepsToGit(deps, options, deps_vars, svn_to_git_objs): |
115 deps_overrides): | |
116 """Convert a 'deps' section in a DEPS file from SVN to Git.""" | 115 """Convert a 'deps' section in a DEPS file from SVN to Git.""" |
117 new_deps = {} | 116 new_deps = {} |
118 bad_git_urls = set([]) | 117 bad_git_urls = set([]) |
119 bad_dep_urls = [] | 118 bad_dep_urls = [] |
120 bad_override = [] | 119 bad_override = [] |
121 bad_git_hash = [] | 120 bad_git_hash = [] |
122 | 121 |
123 # Populate our deps list. | 122 # Populate our deps list. |
124 deps_to_process = {} | 123 deps_to_process = {} |
125 for dep, dep_url in deps.iteritems(): | 124 for dep, dep_url in deps.iteritems(): |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
181 except Queue.Empty: | 180 except Queue.Empty: |
182 sec_since += 1 | 181 sec_since += 1 |
183 line = ('Main> Heartbeat ping. We are still alive!! ' | 182 line = ('Main> Heartbeat ping. We are still alive!! ' |
184 'Seconds since last output: %d sec' % sec_since) | 183 'Seconds since last output: %d sec' % sec_since) |
185 if line is None: | 184 if line is None: |
186 num_threads -= 1 | 185 num_threads -= 1 |
187 else: | 186 else: |
188 print line | 187 print line |
189 pool.join() | 188 pool.join() |
190 | 189 |
191 | |
192 for dep, items in deps_to_process.iteritems(): | 190 for dep, items in deps_to_process.iteritems(): |
193 git_url, dep_url, path, git_host, dep_rev, svn_branch = items | 191 git_url, dep_url, path, git_host, dep_rev, svn_branch = items |
194 if options.verify: | 192 if options.verify: |
195 delay = 0.5 | 193 delay = 0.5 |
196 success = False | 194 success = False |
197 for try_index in range(1, 6): | 195 for try_index in range(1, 6): |
198 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index), | 196 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index), |
199 if git_tools.Ping(git_url, verbose=True): | 197 if git_tools.Ping(git_url, verbose=True): |
200 print >> sys.stderr, ' success' | 198 print >> sys.stderr, ' success' |
201 success = True | 199 success = True |
202 break | 200 break |
203 | 201 |
204 print >> sys.stderr, ' failure' | 202 print >> sys.stderr, ' failure' |
205 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay | 203 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay |
206 time.sleep(delay) | 204 time.sleep(delay) |
207 delay *= 2 | 205 delay *= 2 |
208 | 206 |
209 if not success: | 207 if not success: |
210 bad_git_urls.update([git_url]) | 208 bad_git_urls.update([git_url]) |
211 | 209 |
212 # Get the Git hash based off the SVN rev. | 210 # Get the Git hash based off the SVN rev. |
213 git_hash = '' | 211 git_hash = '' |
214 if dep_rev != 'HEAD': | 212 if dep_rev != 'HEAD': |
215 if dep in deps_overrides and deps_overrides[dep]: | 213 # Pass-through the hash for Git repositories. Resolve the hash for |
216 # Transfer any required variables over from SVN DEPS. | 214 # subversion repositories. |
217 if not deps_overrides[dep] in svn_deps_vars: | 215 if dep_url.endswith('.git'): |
| 216 git_hash = '@%s' % dep_rev |
| 217 else: |
| 218 try: |
| 219 git_hash = '@%s' % SvnRevToGitHash( |
| 220 dep_rev, git_url, options.repos, options.workspace, path, |
| 221 git_host, svn_branch, options.cache_dir) |
| 222 except Exception as e: |
218 if options.no_fail_fast: | 223 if options.no_fail_fast: |
219 bad_override.append(deps_overrides[dep]) | 224 bad_git_hash.append(e) |
220 continue | 225 continue |
221 raise Exception('Missing DEPS variable: %s' % deps_overrides[dep]) | 226 raise |
222 deps_vars[deps_overrides[dep]] = ( | |
223 '@' + svn_deps_vars[deps_overrides[dep]].lstrip('@')) | |
224 # Tag this variable as needing a transform by Varify() later. | |
225 git_hash = '%s_%s' % (deps_utils.VARIFY_MARKER_TAG_PREFIX, | |
226 deps_overrides[dep]) | |
227 else: | |
228 # Pass-through the hash for Git repositories. Resolve the hash for | |
229 # subversion repositories. | |
230 if dep_url.endswith('.git'): | |
231 git_hash = '@%s' % dep_rev | |
232 else: | |
233 try: | |
234 git_hash = '@%s' % SvnRevToGitHash( | |
235 dep_rev, git_url, options.repos, options.workspace, path, | |
236 git_host, svn_branch, options.cache_dir) | |
237 except Exception as e: | |
238 if options.no_fail_fast: | |
239 bad_git_hash.append(e) | |
240 continue | |
241 raise | |
242 | 227 |
243 # If this is webkit, we need to add the var for the hash. | 228 # If this is webkit, we need to add the var for the hash. |
244 if dep == 'src/third_party/WebKit' and dep_rev: | 229 if dep == 'src/third_party/WebKit' and dep_rev: |
245 deps_vars['webkit_rev'] = git_hash | 230 deps_vars['webkit_rev'] = git_hash |
246 git_hash = 'VAR_WEBKIT_REV' | 231 git_hash = 'VAR_WEBKIT_REV' |
247 | 232 |
248 # Hack to preserve the angle_revision variable in .DEPS.git. | 233 # Hack to preserve the angle_revision variable in .DEPS.git. |
249 # This will go away as soon as deps2git does. | 234 # This will go away as soon as deps2git does. |
250 if dep == 'src/third_party/angle' and git_hash: | 235 if dep == 'src/third_party/angle' and git_hash: |
251 # Cut the leading '@' so this variable has the same semantics in | 236 # Cut the leading '@' so this variable has the same semantics in |
(...skipping 28 matching lines...) Expand all Loading... |
280 parser.add_option('--no_fail_fast', action='store_true', | 265 parser.add_option('--no_fail_fast', action='store_true', |
281 help='Try to process the whole DEPS, rather than failing ' | 266 help='Try to process the whole DEPS, rather than failing ' |
282 'on the first bad entry.') | 267 'on the first bad entry.') |
283 parser.add_option('--verify', action='store_true', | 268 parser.add_option('--verify', action='store_true', |
284 help='ping each Git repo to make sure it exists') | 269 help='ping each Git repo to make sure it exists') |
285 parser.add_option('--json', | 270 parser.add_option('--json', |
286 help='path to a JSON file for machine-readable output') | 271 help='path to a JSON file for machine-readable output') |
287 options = parser.parse_args()[0] | 272 options = parser.parse_args()[0] |
288 | 273 |
289 # Get the content of the DEPS file. | 274 # Get the content of the DEPS file. |
290 deps_content = deps_utils.GetDepsContent(options.deps) | 275 deps, deps_os, include_rules, skip_child_includes, hooks = ( |
291 (deps, deps_os, include_rules, skip_child_includes, hooks, | 276 deps_utils.GetDepsContent(options.deps)) |
292 svn_deps_vars) = deps_content | |
293 | 277 |
294 if options.extra_rules and options.type: | 278 if options.extra_rules and options.type: |
295 parser.error('Can\'t specify type and extra-rules at the same time.') | 279 parser.error('Can\'t specify type and extra-rules at the same time.') |
296 elif options.type: | 280 elif options.type: |
297 options.extra_rules = os.path.join( | 281 options.extra_rules = os.path.join( |
298 os.path.abspath(os.path.dirname(__file__)), | 282 os.path.abspath(os.path.dirname(__file__)), |
299 'svn_to_git_%s.py' % options.type) | 283 'svn_to_git_%s.py' % options.type) |
300 if options.cache_dir and options.repos: | 284 if options.cache_dir and options.repos: |
301 parser.error('Can\'t specify both cache_dir and repos at the same time.') | 285 parser.error('Can\'t specify both cache_dir and repos at the same time.') |
302 if options.shallow and not options.cache_dir: | 286 if options.shallow and not options.cache_dir: |
303 parser.error('--shallow only supported with --cache_dir.') | 287 parser.error('--shallow only supported with --cache_dir.') |
304 | 288 |
305 if options.cache_dir: | 289 if options.cache_dir: |
306 options.cache_dir = os.path.abspath(options.cache_dir) | 290 options.cache_dir = os.path.abspath(options.cache_dir) |
307 | 291 |
308 if options.extra_rules and not os.path.exists(options.extra_rules): | 292 if options.extra_rules and not os.path.exists(options.extra_rules): |
309 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules) | 293 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules) |
310 | 294 |
311 # Create a var containing the Git and Webkit URL, this will make it easy for | 295 # Create a var containing the Git and Webkit URL, this will make it easy for |
312 # people to use a mirror instead. | 296 # people to use a mirror instead. |
313 git_url = 'https://chromium.googlesource.com' | 297 git_url = 'https://chromium.googlesource.com' |
314 deps_vars = { | 298 deps_vars = { |
315 'git_url': git_url, | 299 'git_url': git_url, |
316 'webkit_url': git_url + '/chromium/blink.git', | 300 'webkit_url': git_url + '/chromium/blink.git', |
317 } | 301 } |
318 | 302 |
319 # Find and load svn_to_git_* modules that handle the URL mapping. | 303 # Find and load svn_to_git_* modules that handle the URL mapping. |
320 svn_to_git_objs = [svn_to_git_public] | 304 svn_to_git_objs = [svn_to_git_public] |
321 deps_overrides = getattr(svn_to_git_public, 'DEPS_OVERRIDES', {}).copy() | |
322 if options.extra_rules: | 305 if options.extra_rules: |
323 rules_dir, rules_file = os.path.split(options.extra_rules) | 306 rules_dir, rules_file = os.path.split(options.extra_rules) |
324 rules_file_base = os.path.splitext(rules_file)[0] | 307 rules_file_base = os.path.splitext(rules_file)[0] |
325 sys.path.insert(0, rules_dir) | 308 sys.path.insert(0, rules_dir) |
326 svn_to_git_mod = __import__(rules_file_base) | 309 svn_to_git_mod = __import__(rules_file_base) |
327 svn_to_git_objs.insert(0, svn_to_git_mod) | 310 svn_to_git_objs.insert(0, svn_to_git_mod) |
328 # Allow extra_rules file to override rules in svn_to_git_public. | |
329 deps_overrides.update(getattr(svn_to_git_mod, 'DEPS_OVERRIDES', {})) | |
330 | 311 |
331 # If a workspace parameter is given, and a .gclient file is present, limit | 312 # If a workspace parameter is given, and a .gclient file is present, limit |
332 # DEPS conversion to only the repositories that are actually used in this | 313 # DEPS conversion to only the repositories that are actually used in this |
333 # checkout. Also, if a cache dir is specified in .gclient, honor it. | 314 # checkout. Also, if a cache dir is specified in .gclient, honor it. |
334 if options.workspace and os.path.exists( | 315 if options.workspace and os.path.exists( |
335 os.path.join(options.workspace, '.gclient')): | 316 os.path.join(options.workspace, '.gclient')): |
336 gclient_file = os.path.join(options.workspace, '.gclient') | 317 gclient_file = os.path.join(options.workspace, '.gclient') |
337 gclient_dict = {} | 318 gclient_dict = {} |
338 try: | 319 try: |
339 execfile(gclient_file, {}, gclient_dict) | 320 execfile(gclient_file, {}, gclient_dict) |
340 except IOError: | 321 except IOError: |
341 print >> sys.stderr, 'Could not open %s' % gclient_file | 322 print >> sys.stderr, 'Could not open %s' % gclient_file |
342 raise | 323 raise |
343 except SyntaxError: | 324 except SyntaxError: |
344 print >> sys.stderr, 'Could not parse %s' % gclient_file | 325 print >> sys.stderr, 'Could not parse %s' % gclient_file |
345 raise | 326 raise |
346 target_os = gclient_dict.get('target_os', []) | 327 target_os = gclient_dict.get('target_os', []) |
347 if not target_os or not gclient_dict.get('target_os_only'): | 328 if not target_os or not gclient_dict.get('target_os_only'): |
348 target_os.append(DEPS_OS_CHOICES.get(sys.platform, 'unix')) | 329 target_os.append(DEPS_OS_CHOICES.get(sys.platform, 'unix')) |
349 if 'all' not in target_os: | 330 if 'all' not in target_os: |
350 deps_os = dict([(k, v) for k, v in deps_os.iteritems() if k in target_os]) | 331 deps_os = dict([(k, v) for k, v in deps_os.iteritems() if k in target_os]) |
351 if not options.cache_dir and 'cache_dir' in gclient_dict: | 332 if not options.cache_dir and 'cache_dir' in gclient_dict: |
352 options.cache_dir = os.path.abspath(gclient_dict['cache_dir']) | 333 options.cache_dir = os.path.abspath(gclient_dict['cache_dir']) |
353 | 334 |
354 # Do general pre-processing of the DEPS data. | 335 # Do general pre-processing of the DEPS data. |
355 for svn_git_converter in svn_to_git_objs: | 336 for svn_git_converter in svn_to_git_objs: |
356 if hasattr(svn_git_converter, 'CleanDeps'): | 337 if hasattr(svn_git_converter, 'CleanDeps'): |
357 svn_git_converter.CleanDeps(deps, deps_os, include_rules, | 338 svn_git_converter.CleanDeps(deps, deps_os, include_rules, |
358 skip_child_includes, hooks, svn_deps_vars) | 339 skip_child_includes, hooks) |
359 | 340 |
360 # Convert the DEPS file to Git. | 341 # Convert the DEPS file to Git. |
361 deps, baddeps, badmaps, badvars, badhashes = ConvertDepsToGit( | 342 deps, baddeps, badmaps, badvars, badhashes = ConvertDepsToGit( |
362 deps, options, deps_vars, svn_deps_vars, svn_to_git_objs, deps_overrides) | 343 deps, options, deps_vars, svn_to_git_objs) |
363 for os_dep in deps_os: | 344 for os_dep in deps_os: |
364 result = ConvertDepsToGit(deps_os[os_dep], options, deps_vars, | 345 result = ConvertDepsToGit(deps_os[os_dep], options, deps_vars, |
365 svn_deps_vars, svn_to_git_objs, deps_overrides) | 346 svn_to_git_objs) |
366 deps_os[os_dep] = result[0] | 347 deps_os[os_dep] = result[0] |
367 baddeps = baddeps.union(result[1]) | 348 baddeps = baddeps.union(result[1]) |
368 badmaps.extend(result[2]) | 349 badmaps.extend(result[2]) |
369 badvars.extend(result[3]) | 350 badvars.extend(result[3]) |
370 badhashes.extend(result[4]) | 351 badhashes.extend(result[4]) |
371 | 352 |
372 if options.json: | 353 if options.json: |
373 with open(options.json, 'w') as f: | 354 with open(options.json, 'w') as f: |
374 json.dump(list(baddeps), f, sort_keys=True, indent=2) | 355 json.dump(list(baddeps), f, sort_keys=True, indent=2) |
375 | 356 |
(...skipping 28 matching lines...) Expand all Loading... |
404 return 0 | 385 return 0 |
405 | 386 |
406 # Write the DEPS file to disk. | 387 # Write the DEPS file to disk. |
407 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules, | 388 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules, |
408 skip_child_includes, hooks) | 389 skip_child_includes, hooks) |
409 return 0 | 390 return 0 |
410 | 391 |
411 | 392 |
412 if '__main__' == __name__: | 393 if '__main__' == __name__: |
413 sys.exit(main()) | 394 sys.exit(main()) |
OLD | NEW |