Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(275)

Side by Side Diff: deps2git.py

Issue 190853003: Remove special case svn to git translations for FFmpeg. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/deps2git/
Patch Set: Rebase. Created 6 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | deps_utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Convert SVN based DEPS into .DEPS.git for use with NewGit.""" 6 """Convert SVN based DEPS into .DEPS.git for use with NewGit."""
7 7
8 import collections 8 import collections
9 import json 9 import json
10 import optparse 10 import optparse
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after
104 if (git_url.startswith('https://chromium.googlesource.com/external/pefile') 104 if (git_url.startswith('https://chromium.googlesource.com/external/pefile')
105 and int(svn_rev) == 63): 105 and int(svn_rev) == 63):
106 return '1ceaa279daa62b71e3431e58f68be6a96dd1519a' 106 return '1ceaa279daa62b71e3431e58f68be6a96dd1519a'
107 107
108 try: 108 try:
109 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url) 109 return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url)
110 except Exception: 110 except Exception:
111 print >> sys.stderr, '%s <-> ERROR' % git_repo_path 111 print >> sys.stderr, '%s <-> ERROR' % git_repo_path
112 raise 112 raise
113 113
114 def ConvertDepsToGit(deps, options, deps_vars, svn_deps_vars, svn_to_git_objs, 114 def ConvertDepsToGit(deps, options, deps_vars, svn_to_git_objs):
115 deps_overrides):
116 """Convert a 'deps' section in a DEPS file from SVN to Git.""" 115 """Convert a 'deps' section in a DEPS file from SVN to Git."""
117 new_deps = {} 116 new_deps = {}
118 bad_git_urls = set([]) 117 bad_git_urls = set([])
119 bad_dep_urls = [] 118 bad_dep_urls = []
120 bad_override = [] 119 bad_override = []
121 bad_git_hash = [] 120 bad_git_hash = []
122 121
123 # Populate our deps list. 122 # Populate our deps list.
124 deps_to_process = {} 123 deps_to_process = {}
125 for dep, dep_url in deps.iteritems(): 124 for dep, dep_url in deps.iteritems():
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after
181 except Queue.Empty: 180 except Queue.Empty:
182 sec_since += 1 181 sec_since += 1
183 line = ('Main> Heartbeat ping. We are still alive!! ' 182 line = ('Main> Heartbeat ping. We are still alive!! '
184 'Seconds since last output: %d sec' % sec_since) 183 'Seconds since last output: %d sec' % sec_since)
185 if line is None: 184 if line is None:
186 num_threads -= 1 185 num_threads -= 1
187 else: 186 else:
188 print line 187 print line
189 pool.join() 188 pool.join()
190 189
191
192 for dep, items in deps_to_process.iteritems(): 190 for dep, items in deps_to_process.iteritems():
193 git_url, dep_url, path, git_host, dep_rev, svn_branch = items 191 git_url, dep_url, path, git_host, dep_rev, svn_branch = items
194 if options.verify: 192 if options.verify:
195 delay = 0.5 193 delay = 0.5
196 success = False 194 success = False
197 for try_index in range(1, 6): 195 for try_index in range(1, 6):
198 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index), 196 print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index),
199 if git_tools.Ping(git_url, verbose=True): 197 if git_tools.Ping(git_url, verbose=True):
200 print >> sys.stderr, ' success' 198 print >> sys.stderr, ' success'
201 success = True 199 success = True
202 break 200 break
203 201
204 print >> sys.stderr, ' failure' 202 print >> sys.stderr, ' failure'
205 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay 203 print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay
206 time.sleep(delay) 204 time.sleep(delay)
207 delay *= 2 205 delay *= 2
208 206
209 if not success: 207 if not success:
210 bad_git_urls.update([git_url]) 208 bad_git_urls.update([git_url])
211 209
212 # Get the Git hash based off the SVN rev. 210 # Get the Git hash based off the SVN rev.
213 git_hash = '' 211 git_hash = ''
214 if dep_rev != 'HEAD': 212 if dep_rev != 'HEAD':
215 if dep in deps_overrides and deps_overrides[dep]: 213 # Pass-through the hash for Git repositories. Resolve the hash for
216 # Transfer any required variables over from SVN DEPS. 214 # subversion repositories.
217 if not deps_overrides[dep] in svn_deps_vars: 215 if dep_url.endswith('.git'):
216 git_hash = '@%s' % dep_rev
217 else:
218 try:
219 git_hash = '@%s' % SvnRevToGitHash(
220 dep_rev, git_url, options.repos, options.workspace, path,
221 git_host, svn_branch, options.cache_dir)
222 except Exception as e:
218 if options.no_fail_fast: 223 if options.no_fail_fast:
219 bad_override.append(deps_overrides[dep]) 224 bad_git_hash.append(e)
220 continue 225 continue
221 raise Exception('Missing DEPS variable: %s' % deps_overrides[dep]) 226 raise
222 deps_vars[deps_overrides[dep]] = (
223 '@' + svn_deps_vars[deps_overrides[dep]].lstrip('@'))
224 # Tag this variable as needing a transform by Varify() later.
225 git_hash = '%s_%s' % (deps_utils.VARIFY_MARKER_TAG_PREFIX,
226 deps_overrides[dep])
227 else:
228 # Pass-through the hash for Git repositories. Resolve the hash for
229 # subversion repositories.
230 if dep_url.endswith('.git'):
231 git_hash = '@%s' % dep_rev
232 else:
233 try:
234 git_hash = '@%s' % SvnRevToGitHash(
235 dep_rev, git_url, options.repos, options.workspace, path,
236 git_host, svn_branch, options.cache_dir)
237 except Exception as e:
238 if options.no_fail_fast:
239 bad_git_hash.append(e)
240 continue
241 raise
242 227
243 # If this is webkit, we need to add the var for the hash. 228 # If this is webkit, we need to add the var for the hash.
244 if dep == 'src/third_party/WebKit' and dep_rev: 229 if dep == 'src/third_party/WebKit' and dep_rev:
245 deps_vars['webkit_rev'] = git_hash 230 deps_vars['webkit_rev'] = git_hash
246 git_hash = 'VAR_WEBKIT_REV' 231 git_hash = 'VAR_WEBKIT_REV'
247 232
248 # Hack to preserve the angle_revision variable in .DEPS.git. 233 # Hack to preserve the angle_revision variable in .DEPS.git.
249 # This will go away as soon as deps2git does. 234 # This will go away as soon as deps2git does.
250 if dep == 'src/third_party/angle' and git_hash: 235 if dep == 'src/third_party/angle' and git_hash:
251 # Cut the leading '@' so this variable has the same semantics in 236 # Cut the leading '@' so this variable has the same semantics in
(...skipping 28 matching lines...) Expand all
280 parser.add_option('--no_fail_fast', action='store_true', 265 parser.add_option('--no_fail_fast', action='store_true',
281 help='Try to process the whole DEPS, rather than failing ' 266 help='Try to process the whole DEPS, rather than failing '
282 'on the first bad entry.') 267 'on the first bad entry.')
283 parser.add_option('--verify', action='store_true', 268 parser.add_option('--verify', action='store_true',
284 help='ping each Git repo to make sure it exists') 269 help='ping each Git repo to make sure it exists')
285 parser.add_option('--json', 270 parser.add_option('--json',
286 help='path to a JSON file for machine-readable output') 271 help='path to a JSON file for machine-readable output')
287 options = parser.parse_args()[0] 272 options = parser.parse_args()[0]
288 273
289 # Get the content of the DEPS file. 274 # Get the content of the DEPS file.
290 deps_content = deps_utils.GetDepsContent(options.deps) 275 deps, deps_os, include_rules, skip_child_includes, hooks = (
291 (deps, deps_os, include_rules, skip_child_includes, hooks, 276 deps_utils.GetDepsContent(options.deps))
292 svn_deps_vars) = deps_content
293 277
294 if options.extra_rules and options.type: 278 if options.extra_rules and options.type:
295 parser.error('Can\'t specify type and extra-rules at the same time.') 279 parser.error('Can\'t specify type and extra-rules at the same time.')
296 elif options.type: 280 elif options.type:
297 options.extra_rules = os.path.join( 281 options.extra_rules = os.path.join(
298 os.path.abspath(os.path.dirname(__file__)), 282 os.path.abspath(os.path.dirname(__file__)),
299 'svn_to_git_%s.py' % options.type) 283 'svn_to_git_%s.py' % options.type)
300 if options.cache_dir and options.repos: 284 if options.cache_dir and options.repos:
301 parser.error('Can\'t specify both cache_dir and repos at the same time.') 285 parser.error('Can\'t specify both cache_dir and repos at the same time.')
302 if options.shallow and not options.cache_dir: 286 if options.shallow and not options.cache_dir:
303 parser.error('--shallow only supported with --cache_dir.') 287 parser.error('--shallow only supported with --cache_dir.')
304 288
305 if options.cache_dir: 289 if options.cache_dir:
306 options.cache_dir = os.path.abspath(options.cache_dir) 290 options.cache_dir = os.path.abspath(options.cache_dir)
307 291
308 if options.extra_rules and not os.path.exists(options.extra_rules): 292 if options.extra_rules and not os.path.exists(options.extra_rules):
309 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules) 293 raise Exception('Can\'t locate rules file "%s".' % options.extra_rules)
310 294
311 # Create a var containing the Git and Webkit URL, this will make it easy for 295 # Create a var containing the Git and Webkit URL, this will make it easy for
312 # people to use a mirror instead. 296 # people to use a mirror instead.
313 git_url = 'https://chromium.googlesource.com' 297 git_url = 'https://chromium.googlesource.com'
314 deps_vars = { 298 deps_vars = {
315 'git_url': git_url, 299 'git_url': git_url,
316 'webkit_url': git_url + '/chromium/blink.git', 300 'webkit_url': git_url + '/chromium/blink.git',
317 } 301 }
318 302
319 # Find and load svn_to_git_* modules that handle the URL mapping. 303 # Find and load svn_to_git_* modules that handle the URL mapping.
320 svn_to_git_objs = [svn_to_git_public] 304 svn_to_git_objs = [svn_to_git_public]
321 deps_overrides = getattr(svn_to_git_public, 'DEPS_OVERRIDES', {}).copy()
322 if options.extra_rules: 305 if options.extra_rules:
323 rules_dir, rules_file = os.path.split(options.extra_rules) 306 rules_dir, rules_file = os.path.split(options.extra_rules)
324 rules_file_base = os.path.splitext(rules_file)[0] 307 rules_file_base = os.path.splitext(rules_file)[0]
325 sys.path.insert(0, rules_dir) 308 sys.path.insert(0, rules_dir)
326 svn_to_git_mod = __import__(rules_file_base) 309 svn_to_git_mod = __import__(rules_file_base)
327 svn_to_git_objs.insert(0, svn_to_git_mod) 310 svn_to_git_objs.insert(0, svn_to_git_mod)
328 # Allow extra_rules file to override rules in svn_to_git_public.
329 deps_overrides.update(getattr(svn_to_git_mod, 'DEPS_OVERRIDES', {}))
330 311
331 # If a workspace parameter is given, and a .gclient file is present, limit 312 # If a workspace parameter is given, and a .gclient file is present, limit
332 # DEPS conversion to only the repositories that are actually used in this 313 # DEPS conversion to only the repositories that are actually used in this
333 # checkout. Also, if a cache dir is specified in .gclient, honor it. 314 # checkout. Also, if a cache dir is specified in .gclient, honor it.
334 if options.workspace and os.path.exists( 315 if options.workspace and os.path.exists(
335 os.path.join(options.workspace, '.gclient')): 316 os.path.join(options.workspace, '.gclient')):
336 gclient_file = os.path.join(options.workspace, '.gclient') 317 gclient_file = os.path.join(options.workspace, '.gclient')
337 gclient_dict = {} 318 gclient_dict = {}
338 try: 319 try:
339 execfile(gclient_file, {}, gclient_dict) 320 execfile(gclient_file, {}, gclient_dict)
340 except IOError: 321 except IOError:
341 print >> sys.stderr, 'Could not open %s' % gclient_file 322 print >> sys.stderr, 'Could not open %s' % gclient_file
342 raise 323 raise
343 except SyntaxError: 324 except SyntaxError:
344 print >> sys.stderr, 'Could not parse %s' % gclient_file 325 print >> sys.stderr, 'Could not parse %s' % gclient_file
345 raise 326 raise
346 target_os = gclient_dict.get('target_os', []) 327 target_os = gclient_dict.get('target_os', [])
347 if not target_os or not gclient_dict.get('target_os_only'): 328 if not target_os or not gclient_dict.get('target_os_only'):
348 target_os.append(DEPS_OS_CHOICES.get(sys.platform, 'unix')) 329 target_os.append(DEPS_OS_CHOICES.get(sys.platform, 'unix'))
349 if 'all' not in target_os: 330 if 'all' not in target_os:
350 deps_os = dict([(k, v) for k, v in deps_os.iteritems() if k in target_os]) 331 deps_os = dict([(k, v) for k, v in deps_os.iteritems() if k in target_os])
351 if not options.cache_dir and 'cache_dir' in gclient_dict: 332 if not options.cache_dir and 'cache_dir' in gclient_dict:
352 options.cache_dir = os.path.abspath(gclient_dict['cache_dir']) 333 options.cache_dir = os.path.abspath(gclient_dict['cache_dir'])
353 334
354 # Do general pre-processing of the DEPS data. 335 # Do general pre-processing of the DEPS data.
355 for svn_git_converter in svn_to_git_objs: 336 for svn_git_converter in svn_to_git_objs:
356 if hasattr(svn_git_converter, 'CleanDeps'): 337 if hasattr(svn_git_converter, 'CleanDeps'):
357 svn_git_converter.CleanDeps(deps, deps_os, include_rules, 338 svn_git_converter.CleanDeps(deps, deps_os, include_rules,
358 skip_child_includes, hooks, svn_deps_vars) 339 skip_child_includes, hooks)
359 340
360 # Convert the DEPS file to Git. 341 # Convert the DEPS file to Git.
361 deps, baddeps, badmaps, badvars, badhashes = ConvertDepsToGit( 342 deps, baddeps, badmaps, badvars, badhashes = ConvertDepsToGit(
362 deps, options, deps_vars, svn_deps_vars, svn_to_git_objs, deps_overrides) 343 deps, options, deps_vars, svn_to_git_objs)
363 for os_dep in deps_os: 344 for os_dep in deps_os:
364 result = ConvertDepsToGit(deps_os[os_dep], options, deps_vars, 345 result = ConvertDepsToGit(deps_os[os_dep], options, deps_vars,
365 svn_deps_vars, svn_to_git_objs, deps_overrides) 346 svn_to_git_objs)
366 deps_os[os_dep] = result[0] 347 deps_os[os_dep] = result[0]
367 baddeps = baddeps.union(result[1]) 348 baddeps = baddeps.union(result[1])
368 badmaps.extend(result[2]) 349 badmaps.extend(result[2])
369 badvars.extend(result[3]) 350 badvars.extend(result[3])
370 badhashes.extend(result[4]) 351 badhashes.extend(result[4])
371 352
372 if options.json: 353 if options.json:
373 with open(options.json, 'w') as f: 354 with open(options.json, 'w') as f:
374 json.dump(list(baddeps), f, sort_keys=True, indent=2) 355 json.dump(list(baddeps), f, sort_keys=True, indent=2)
375 356
(...skipping 28 matching lines...) Expand all
404 return 0 385 return 0
405 386
406 # Write the DEPS file to disk. 387 # Write the DEPS file to disk.
407 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules, 388 deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules,
408 skip_child_includes, hooks) 389 skip_child_includes, hooks)
409 return 0 390 return 0
410 391
411 392
412 if '__main__' == __name__: 393 if '__main__' == __name__:
413 sys.exit(main()) 394 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | deps_utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698