OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """Generate fake repositories for testing.""" | |
7 | |
8 import atexit | |
9 import datetime | |
10 import errno | |
11 import logging | |
12 import os | |
13 import pprint | |
14 import re | |
15 import socket | |
16 import sys | |
17 import tempfile | |
18 import time | |
19 | |
20 # trial_dir must be first for non-system libraries. | |
21 from testing_support import trial_dir | |
22 import gclient_utils | |
23 import scm | |
24 import subprocess2 | |
25 | |
26 | |
27 def write(path, content): | |
28 f = open(path, 'wb') | |
29 f.write(content) | |
30 f.close() | |
31 | |
32 | |
33 join = os.path.join | |
34 | |
35 | |
36 def read_tree(tree_root): | |
37 """Returns a dict of all the files in a tree. Defaults to self.root_dir.""" | |
38 tree = {} | |
39 for root, dirs, files in os.walk(tree_root): | |
40 for d in filter(lambda x: x.startswith('.'), dirs): | |
41 dirs.remove(d) | |
42 for f in [join(root, f) for f in files if not f.startswith('.')]: | |
43 filepath = f[len(tree_root) + 1:].replace(os.sep, '/') | |
44 assert len(filepath), f | |
45 tree[filepath] = open(join(root, f), 'rU').read() | |
46 return tree | |
47 | |
48 | |
49 def dict_diff(dict1, dict2): | |
50 diff = {} | |
51 for k, v in dict1.iteritems(): | |
52 if k not in dict2: | |
53 diff[k] = v | |
54 elif v != dict2[k]: | |
55 diff[k] = (v, dict2[k]) | |
56 for k, v in dict2.iteritems(): | |
57 if k not in dict1: | |
58 diff[k] = v | |
59 return diff | |
60 | |
61 | |
62 def commit_svn(repo, usr, pwd): | |
63 """Commits the changes and returns the new revision number.""" | |
64 to_add = [] | |
65 to_remove = [] | |
66 for status, filepath in scm.SVN.CaptureStatus(None, repo): | |
67 if status[0] == '?': | |
68 to_add.append(filepath) | |
69 elif status[0] == '!': | |
70 to_remove.append(filepath) | |
71 if to_add: | |
72 subprocess2.check_output( | |
73 ['svn', 'add', '--no-auto-props', '-q'] + to_add, cwd=repo) | |
74 if to_remove: | |
75 subprocess2.check_output(['svn', 'remove', '-q'] + to_remove, cwd=repo) | |
76 | |
77 out = subprocess2.check_output( | |
78 ['svn', 'commit', repo, '-m', 'foo', '--non-interactive', | |
79 '--no-auth-cache', | |
80 '--username', usr, '--password', pwd], | |
81 cwd=repo) | |
82 match = re.search(r'(\d+)', out) | |
83 if not match: | |
84 raise Exception('Commit failed', out) | |
85 rev = match.group(1) | |
86 status = subprocess2.check_output(['svn', 'status'], cwd=repo) | |
87 assert len(status) == 0, status | |
88 logging.debug('At revision %s' % rev) | |
89 return rev | |
90 | |
91 | |
92 def commit_git(repo): | |
93 """Commits the changes and returns the new hash.""" | |
94 subprocess2.check_call(['git', 'add', '-A', '-f'], cwd=repo) | |
95 subprocess2.check_call(['git', 'commit', '-q', '--message', 'foo'], cwd=repo) | |
96 rev = subprocess2.check_output( | |
97 ['git', 'show-ref', '--head', 'HEAD'], cwd=repo).split(' ', 1)[0] | |
98 logging.debug('At revision %s' % rev) | |
99 return rev | |
100 | |
101 | |
102 def test_port(host, port): | |
103 s = socket.socket() | |
104 try: | |
105 return s.connect_ex((host, port)) == 0 | |
106 finally: | |
107 s.close() | |
108 | |
109 | |
110 def find_free_port(host, base_port): | |
111 """Finds a listening port free to listen to.""" | |
112 while base_port < (2<<16): | |
113 if not test_port(host, base_port): | |
114 return base_port | |
115 base_port += 1 | |
116 assert False, 'Having issues finding an available port' | |
117 | |
118 | |
119 def wait_for_port_to_bind(host, port, process): | |
120 sock = socket.socket() | |
121 | |
122 if sys.platform == 'darwin': | |
123 # On Mac SnowLeopard, if we attempt to connect to the socket | |
124 # immediately, it fails with EINVAL and never gets a chance to | |
125 # connect (putting us into a hard spin and then failing). | |
126 # Linux doesn't need this. | |
127 time.sleep(0.2) | |
128 | |
129 try: | |
130 start = datetime.datetime.utcnow() | |
131 maxdelay = datetime.timedelta(seconds=30) | |
132 while (datetime.datetime.utcnow() - start) < maxdelay: | |
133 try: | |
134 sock.connect((host, port)) | |
135 logging.debug('%d is now bound' % port) | |
136 return | |
137 except (socket.error, EnvironmentError): | |
138 pass | |
139 logging.debug('%d is still not bound' % port) | |
140 finally: | |
141 sock.close() | |
142 # The process failed to bind. Kill it and dump its ouput. | |
143 process.kill() | |
144 logging.error('%s' % process.communicate()[0]) | |
145 assert False, '%d is still not bound' % port | |
146 | |
147 | |
148 def wait_for_port_to_free(host, port): | |
149 start = datetime.datetime.utcnow() | |
150 maxdelay = datetime.timedelta(seconds=30) | |
151 while (datetime.datetime.utcnow() - start) < maxdelay: | |
152 try: | |
153 sock = socket.socket() | |
154 sock.connect((host, port)) | |
155 logging.debug('%d was bound, waiting to free' % port) | |
156 except (socket.error, EnvironmentError): | |
157 logging.debug('%d now free' % port) | |
158 return | |
159 finally: | |
160 sock.close() | |
161 assert False, '%d is still bound' % port | |
162 | |
163 | |
164 class FakeReposBase(object): | |
165 """Generate both svn and git repositories to test gclient functionality. | |
166 | |
167 Many DEPS functionalities need to be tested: Var, File, From, deps_os, hooks, | |
168 use_relative_paths. | |
169 | |
170 And types of dependencies: Relative urls, Full urls, both svn and git. | |
171 | |
172 populateSvn() and populateGit() need to be implemented by the subclass. | |
173 """ | |
174 # Hostname | |
175 NB_GIT_REPOS = 1 | |
176 USERS = [ | |
177 ('user1@example.com', 'foo'), | |
178 ('user2@example.com', 'bar'), | |
179 ] | |
180 | |
181 def __init__(self, host=None): | |
182 self.trial = trial_dir.TrialDir('repos') | |
183 self.host = host or '127.0.0.1' | |
184 # Format is [ None, tree, tree, ...] | |
185 # i.e. revisions are 1-based. | |
186 self.svn_revs = [None] | |
187 # Format is { repo: [ None, (hash, tree), (hash, tree), ... ], ... } | |
188 # so reference looks like self.git_hashes[repo][rev][0] for hash and | |
189 # self.git_hashes[repo][rev][1] for it's tree snapshot. | |
190 # For consistency with self.svn_revs, it is 1-based too. | |
191 self.git_hashes = {} | |
192 self.svnserve = None | |
193 self.gitdaemon = None | |
194 self.git_pid_file = None | |
195 self.git_root = None | |
196 self.svn_checkout = None | |
197 self.svn_repo = None | |
198 self.git_dirty = False | |
199 self.svn_dirty = False | |
200 self.svn_port = None | |
201 self.git_port = None | |
202 self.svn_base = None | |
203 self.git_base = None | |
204 | |
205 @property | |
206 def root_dir(self): | |
207 return self.trial.root_dir | |
208 | |
209 def set_up(self): | |
210 """All late initialization comes here.""" | |
211 self.cleanup_dirt() | |
212 if not self.root_dir: | |
213 try: | |
214 # self.root_dir is not set before this call. | |
215 self.trial.set_up() | |
216 self.git_root = join(self.root_dir, 'git') | |
217 self.svn_checkout = join(self.root_dir, 'svn_checkout') | |
218 self.svn_repo = join(self.root_dir, 'svn') | |
219 finally: | |
220 # Registers cleanup. | |
221 atexit.register(self.tear_down) | |
222 | |
223 def cleanup_dirt(self): | |
224 """For each dirty repository, destroy it.""" | |
225 if self.svn_dirty: | |
226 if not self.tear_down_svn(): | |
227 logging.error('Using both leaking checkout and svn dirty checkout') | |
228 if self.git_dirty: | |
229 if not self.tear_down_git(): | |
230 logging.error('Using both leaking checkout and git dirty checkout') | |
231 | |
232 def tear_down(self): | |
233 """Kills the servers and delete the directories.""" | |
234 self.tear_down_svn() | |
235 self.tear_down_git() | |
236 # This deletes the directories. | |
237 self.trial.tear_down() | |
238 self.trial = None | |
239 | |
240 def tear_down_svn(self): | |
241 if self.svnserve: | |
242 logging.debug('Killing svnserve pid %s' % self.svnserve.pid) | |
243 try: | |
244 self.svnserve.kill() | |
245 except OSError as e: | |
246 if e.errno != errno.ESRCH: # no such process | |
247 raise | |
248 wait_for_port_to_free(self.host, self.svn_port) | |
249 self.svnserve = None | |
250 self.svn_port = None | |
251 self.svn_base = None | |
252 if not self.trial.SHOULD_LEAK: | |
253 logging.debug('Removing %s' % self.svn_repo) | |
254 gclient_utils.rmtree(self.svn_repo) | |
255 logging.debug('Removing %s' % self.svn_checkout) | |
256 gclient_utils.rmtree(self.svn_checkout) | |
257 else: | |
258 return False | |
259 return True | |
260 | |
261 def tear_down_git(self): | |
262 if self.gitdaemon: | |
263 logging.debug('Killing git-daemon pid %s' % self.gitdaemon.pid) | |
264 self.gitdaemon.kill() | |
265 self.gitdaemon = None | |
266 if self.git_pid_file: | |
267 pid = int(self.git_pid_file.read()) | |
268 self.git_pid_file.close() | |
269 logging.debug('Killing git daemon pid %s' % pid) | |
270 try: | |
271 subprocess2.kill_pid(pid) | |
272 except OSError as e: | |
273 if e.errno != errno.ESRCH: # no such process | |
274 raise | |
275 self.git_pid_file = None | |
276 wait_for_port_to_free(self.host, self.git_port) | |
277 self.git_port = None | |
278 self.git_base = None | |
279 if not self.trial.SHOULD_LEAK: | |
280 logging.debug('Removing %s' % self.git_root) | |
281 gclient_utils.rmtree(self.git_root) | |
282 else: | |
283 return False | |
284 return True | |
285 | |
286 @staticmethod | |
287 def _genTree(root, tree_dict): | |
288 """For a dictionary of file contents, generate a filesystem.""" | |
289 if not os.path.isdir(root): | |
290 os.makedirs(root) | |
291 for (k, v) in tree_dict.iteritems(): | |
292 k_os = k.replace('/', os.sep) | |
293 k_arr = k_os.split(os.sep) | |
294 if len(k_arr) > 1: | |
295 p = os.sep.join([root] + k_arr[:-1]) | |
296 if not os.path.isdir(p): | |
297 os.makedirs(p) | |
298 if v is None: | |
299 os.remove(join(root, k)) | |
300 else: | |
301 write(join(root, k), v) | |
302 | |
303 def set_up_svn(self): | |
304 """Creates subversion repositories and start the servers.""" | |
305 self.set_up() | |
306 if self.svnserve: | |
307 return True | |
308 try: | |
309 subprocess2.check_call(['svnadmin', 'create', self.svn_repo]) | |
310 except (OSError, subprocess2.CalledProcessError): | |
311 return False | |
312 write(join(self.svn_repo, 'conf', 'svnserve.conf'), | |
313 '[general]\n' | |
314 'anon-access = read\n' | |
315 'auth-access = write\n' | |
316 'password-db = passwd\n') | |
317 text = '[users]\n' | |
318 text += ''.join('%s = %s\n' % (usr, pwd) for usr, pwd in self.USERS) | |
319 write(join(self.svn_repo, 'conf', 'passwd'), text) | |
320 | |
321 # Necessary to be able to change revision properties | |
322 revprop_hook_filename = join(self.svn_repo, 'hooks', 'pre-revprop-change') | |
323 if sys.platform == 'win32': | |
324 # TODO(kustermann): Test on Windows one day. | |
325 write("%s.bat" % revprop_hook_filename, "") | |
326 else: | |
327 write(revprop_hook_filename, | |
328 '#!/bin/sh\n' | |
329 'exit 0\n') | |
330 os.chmod(revprop_hook_filename, 0755) | |
331 | |
332 # Mac 10.6 ships with a buggy subversion build and we need this line | |
333 # to work around the bug. | |
334 write(join(self.svn_repo, 'db', 'fsfs.conf'), | |
335 '[rep-sharing]\n' | |
336 'enable-rep-sharing = false\n') | |
337 | |
338 # Start the daemon. | |
339 self.svn_port = find_free_port(self.host, 10000) | |
340 logging.debug('Using port %d' % self.svn_port) | |
341 cmd = ['svnserve', '-d', '--foreground', '-r', self.root_dir, | |
342 '--listen-port=%d' % self.svn_port] | |
343 if self.host == '127.0.0.1': | |
344 cmd.append('--listen-host=' + self.host) | |
345 self.check_port_is_free(self.svn_port) | |
346 self.svnserve = subprocess2.Popen( | |
347 cmd, | |
348 cwd=self.svn_repo, | |
349 stdout=subprocess2.PIPE, | |
350 stderr=subprocess2.PIPE) | |
351 wait_for_port_to_bind(self.host, self.svn_port, self.svnserve) | |
352 self.svn_base = 'svn://%s:%d/svn/' % (self.host, self.svn_port) | |
353 self.populateSvn() | |
354 self.svn_dirty = False | |
355 return True | |
356 | |
357 def set_up_git(self): | |
358 """Creates git repositories and start the servers.""" | |
359 self.set_up() | |
360 if self.gitdaemon: | |
361 return True | |
362 assert self.git_pid_file == None | |
363 try: | |
364 subprocess2.check_output(['git', '--version']) | |
365 except (OSError, subprocess2.CalledProcessError): | |
366 return False | |
367 for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]: | |
368 subprocess2.check_call(['git', 'init', '-q', join(self.git_root, repo)]) | |
369 self.git_hashes[repo] = [None] | |
370 self.git_port = find_free_port(self.host, 20000) | |
371 self.git_base = 'git://%s:%d/git/' % (self.host, self.git_port) | |
372 # Start the daemon. | |
373 self.git_pid_file = tempfile.NamedTemporaryFile() | |
374 cmd = ['git', 'daemon', | |
375 '--export-all', | |
376 '--reuseaddr', | |
377 '--base-path=' + self.root_dir, | |
378 '--pid-file=' + self.git_pid_file.name, | |
379 '--port=%d' % self.git_port] | |
380 if self.host == '127.0.0.1': | |
381 cmd.append('--listen=' + self.host) | |
382 self.check_port_is_free(self.git_port) | |
383 self.gitdaemon = subprocess2.Popen( | |
384 cmd, | |
385 cwd=self.root_dir, | |
386 stdout=subprocess2.PIPE, | |
387 stderr=subprocess2.PIPE) | |
388 wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon) | |
389 self.populateGit() | |
390 self.git_dirty = False | |
391 return True | |
392 | |
393 def _commit_svn(self, tree): | |
394 self._genTree(self.svn_checkout, tree) | |
395 commit_svn(self.svn_checkout, self.USERS[0][0], self.USERS[0][1]) | |
396 if self.svn_revs and self.svn_revs[-1]: | |
397 new_tree = self.svn_revs[-1].copy() | |
398 new_tree.update(tree) | |
399 else: | |
400 new_tree = tree.copy() | |
401 self.svn_revs.append(new_tree) | |
402 | |
403 def _set_svn_commit_date(self, revision, date): | |
404 subprocess2.check_output( | |
405 ['svn', 'propset', 'svn:date', '--revprop', '-r', revision, date, | |
406 self.svn_base, | |
407 '--username', self.USERS[0][0], | |
408 '--password', self.USERS[0][1], | |
409 '--non-interactive']) | |
410 | |
411 def _commit_git(self, repo, tree): | |
412 repo_root = join(self.git_root, repo) | |
413 self._genTree(repo_root, tree) | |
414 commit_hash = commit_git(repo_root) | |
415 if self.git_hashes[repo][-1]: | |
416 new_tree = self.git_hashes[repo][-1][1].copy() | |
417 new_tree.update(tree) | |
418 else: | |
419 new_tree = tree.copy() | |
420 self.git_hashes[repo].append((commit_hash, new_tree)) | |
421 | |
422 def check_port_is_free(self, port): | |
423 sock = socket.socket() | |
424 try: | |
425 sock.connect((self.host, port)) | |
426 # It worked, throw. | |
427 assert False, '%d shouldn\'t be bound' % port | |
428 except (socket.error, EnvironmentError): | |
429 pass | |
430 finally: | |
431 sock.close() | |
432 | |
433 def populateSvn(self): | |
434 raise NotImplementedError() | |
435 | |
436 def populateGit(self): | |
437 raise NotImplementedError() | |
438 | |
439 | |
440 class FakeRepos(FakeReposBase): | |
441 """Implements populateSvn() and populateGit().""" | |
442 NB_GIT_REPOS = 5 | |
443 | |
444 def populateSvn(self): | |
445 """Creates a few revisions of changes including DEPS files.""" | |
446 # Repos | |
447 subprocess2.check_call( | |
448 ['svn', 'checkout', self.svn_base, self.svn_checkout, | |
449 '-q', '--non-interactive', '--no-auth-cache', | |
450 '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) | |
451 assert os.path.isdir(join(self.svn_checkout, '.svn')) | |
452 def file_system(rev, DEPS, DEPS_ALT=None): | |
453 fs = { | |
454 'origin': 'svn@%(rev)d\n', | |
455 'trunk/origin': 'svn/trunk@%(rev)d\n', | |
456 'trunk/src/origin': 'svn/trunk/src@%(rev)d\n', | |
457 'trunk/src/third_party/origin': 'svn/trunk/src/third_party@%(rev)d\n', | |
458 'trunk/other/origin': 'src/trunk/other@%(rev)d\n', | |
459 'trunk/third_party/origin': 'svn/trunk/third_party@%(rev)d\n', | |
460 'trunk/third_party/foo/origin': 'svn/trunk/third_party/foo@%(rev)d\n', | |
461 'trunk/third_party/prout/origin': 'svn/trunk/third_party/foo@%(rev)d\n', | |
462 } | |
463 for k in fs.iterkeys(): | |
464 fs[k] = fs[k] % { 'rev': rev } | |
465 fs['trunk/src/DEPS'] = DEPS | |
466 if DEPS_ALT: | |
467 fs['trunk/src/DEPS.alt'] = DEPS_ALT | |
468 return fs | |
469 | |
470 # Testing: | |
471 # - dependency disapear | |
472 # - dependency renamed | |
473 # - versioned and unversioned reference | |
474 # - relative and full reference | |
475 # - deps_os | |
476 # - var | |
477 # - hooks | |
478 # - From | |
479 # - File | |
480 # TODO(maruel): | |
481 # - $matching_files | |
482 # - use_relative_paths | |
483 DEPS = """ | |
484 vars = { | |
485 'DummyVariable': 'third_party', | |
486 } | |
487 deps = { | |
488 'src/other': '%(svn_base)strunk/other@1', | |
489 'src/third_party/fpp': '/trunk/' + Var('DummyVariable') + '/foo', | |
490 } | |
491 deps_os = { | |
492 'mac': { | |
493 'src/third_party/prout': '/trunk/third_party/prout', | |
494 }, | |
495 }""" % { 'svn_base': self.svn_base } | |
496 | |
497 DEPS_ALT = """ | |
498 deps = { | |
499 'src/other2': '%(svn_base)strunk/other@2' | |
500 } | |
501 """ % { 'svn_base': self.svn_base } | |
502 | |
503 fs = file_system(1, DEPS, DEPS_ALT) | |
504 self._commit_svn(fs) | |
505 | |
506 fs = file_system(2, """ | |
507 deps = { | |
508 'src/other': '%(svn_base)strunk/other', | |
509 # Load another DEPS and load a dependency from it. That's an example of | |
510 # WebKit's chromium checkout flow. Verify it works out of order. | |
511 'src/third_party/foo': From('src/file/other', 'foo/bar'), | |
512 'src/file/other': File('%(svn_base)strunk/other/DEPS'), | |
513 } | |
514 # I think this is wrong to have the hooks run from the base of the gclient | |
515 # checkout. It's maybe a bit too late to change that behavior. | |
516 hooks = [ | |
517 { | |
518 'pattern': '.', | |
519 'action': ['python', '-c', | |
520 'open(\\'src/svn_hooked1\\', \\'w\\').write(\\'svn_hooked1\\')'], | |
521 }, | |
522 { | |
523 # Should not be run. | |
524 'pattern': 'nonexistent', | |
525 'action': ['python', '-c', | |
526 'open(\\'src/svn_hooked2\\', \\'w\\').write(\\'svn_hooked2\\')'], | |
527 }, | |
528 ] | |
529 """ % { 'svn_base': self.svn_base }) | |
530 fs['trunk/other/DEPS'] = """ | |
531 deps = { | |
532 'foo/bar': '/trunk/third_party/foo@1', | |
533 # Only the requested deps should be processed. | |
534 'invalid': '/does_not_exist', | |
535 } | |
536 """ | |
537 # WebKit abuses this. | |
538 fs['trunk/webkit/.gclient'] = """ | |
539 solutions = [ | |
540 { | |
541 'name': './', | |
542 'url': None, | |
543 }, | |
544 ] | |
545 """ | |
546 fs['trunk/webkit/DEPS'] = """ | |
547 deps = { | |
548 'foo/bar': '%(svn_base)strunk/third_party/foo@1' | |
549 } | |
550 | |
551 hooks = [ | |
552 { | |
553 'pattern': '.*', | |
554 'action': ['echo', 'foo'], | |
555 }, | |
556 ] | |
557 """ % { 'svn_base': self.svn_base } | |
558 self._commit_svn(fs) | |
559 | |
560 def populateGit(self): | |
561 # Testing: | |
562 # - dependency disappear | |
563 # - dependency renamed | |
564 # - versioned and unversioned reference | |
565 # - relative and full reference | |
566 # - deps_os | |
567 # - var | |
568 # - hooks | |
569 # - From | |
570 # TODO(maruel): | |
571 # - File: File is hard to test here because it's SVN-only. It's | |
572 # implementation should probably be replaced to use urllib instead. | |
573 # - $matching_files | |
574 # - use_relative_paths | |
575 self._commit_git('repo_3', { | |
576 'origin': 'git/repo_3@1\n', | |
577 }) | |
578 | |
579 self._commit_git('repo_3', { | |
580 'origin': 'git/repo_3@2\n', | |
581 }) | |
582 | |
583 self._commit_git('repo_1', { | |
584 'DEPS': """ | |
585 vars = { | |
586 'DummyVariable': 'repo', | |
587 } | |
588 deps = { | |
589 'src/repo2': '%(git_base)srepo_2', | |
590 'src/repo2/repo3': '/' + Var('DummyVariable') + '_3@%(hash3)s', | |
591 } | |
592 deps_os = { | |
593 'mac': { | |
594 'src/repo4': '/repo_4', | |
595 }, | |
596 }""" % { | |
597 'git_base': self.git_base, | |
598 # See self.__init__() for the format. Grab's the hash of the first | |
599 # commit in repo_2. Only keep the first 7 character because of: | |
600 # TODO(maruel): http://crosbug.com/3591 We need to strip the hash.. | |
601 # duh. | |
602 'hash3': self.git_hashes['repo_3'][1][0][:7] | |
603 }, | |
604 'origin': 'git/repo_1@1\n', | |
605 }) | |
606 | |
607 self._commit_git('repo_2', { | |
608 'origin': 'git/repo_2@1\n', | |
609 'DEPS': """ | |
610 deps = { | |
611 'foo/bar': '/repo_3', | |
612 } | |
613 """, | |
614 }) | |
615 | |
616 self._commit_git('repo_2', { | |
617 'origin': 'git/repo_2@2\n', | |
618 }) | |
619 | |
620 self._commit_git('repo_4', { | |
621 'origin': 'git/repo_4@1\n', | |
622 }) | |
623 | |
624 self._commit_git('repo_4', { | |
625 'origin': 'git/repo_4@2\n', | |
626 }) | |
627 | |
628 self._commit_git('repo_1', { | |
629 'DEPS': """ | |
630 deps = { | |
631 'src/repo2': '%(git_base)srepo_2@%(hash)s', | |
632 #'src/repo2/repo_renamed': '/repo_3', | |
633 'src/repo2/repo_renamed': From('src/repo2', 'foo/bar'), | |
634 } | |
635 # I think this is wrong to have the hooks run from the base of the gclient | |
636 # checkout. It's maybe a bit too late to change that behavior. | |
637 hooks = [ | |
638 { | |
639 'pattern': '.', | |
640 'action': ['python', '-c', | |
641 'open(\\'src/git_hooked1\\', \\'w\\').write(\\'git_hooked1\\')'], | |
642 }, | |
643 { | |
644 # Should not be run. | |
645 'pattern': 'nonexistent', | |
646 'action': ['python', '-c', | |
647 'open(\\'src/git_hooked2\\', \\'w\\').write(\\'git_hooked2\\')'], | |
648 }, | |
649 ] | |
650 """ % { | |
651 'git_base': self.git_base, | |
652 # See self.__init__() for the format. Grab's the hash of the first | |
653 # commit in repo_2. Only keep the first 7 character because of: | |
654 # TODO(maruel): http://crosbug.com/3591 We need to strip the hash.. duh. | |
655 'hash': self.git_hashes['repo_2'][1][0][:7] | |
656 }, | |
657 'origin': 'git/repo_1@2\n', | |
658 }) | |
659 | |
660 self._commit_git('repo_5', {'origin': 'git/repo_5@1\n'}) | |
661 self._commit_git('repo_5', { | |
662 'DEPS': """ | |
663 deps = { | |
664 'src/repo1': '%(git_base)srepo_1@%(hash1)s', | |
665 'src/repo2': '%(git_base)srepo_2@%(hash2)s', | |
666 } | |
667 | |
668 # Hooks to run after a project is processed but before its dependencies are | |
669 # processed. | |
670 pre_deps_hooks = [ | |
671 { | |
672 'action': ['python', '-c', | |
673 'print "pre-deps hook"; open(\\'src/git_pre_deps_hooked\\', \\'w\
\').write(\\'git_pre_deps_hooked\\')'], | |
674 } | |
675 ] | |
676 """ % { | |
677 'git_base': self.git_base, | |
678 'hash1': self.git_hashes['repo_1'][2][0][:7], | |
679 'hash2': self.git_hashes['repo_2'][1][0][:7], | |
680 }, | |
681 'origin': 'git/repo_5@2\n', | |
682 }) | |
683 self._commit_git('repo_5', { | |
684 'DEPS': """ | |
685 deps = { | |
686 'src/repo1': '%(git_base)srepo_1@%(hash1)s', | |
687 'src/repo2': '%(git_base)srepo_2@%(hash2)s', | |
688 } | |
689 | |
690 # Hooks to run after a project is processed but before its dependencies are | |
691 # processed. | |
692 pre_deps_hooks = [ | |
693 { | |
694 'action': ['python', '-c', | |
695 'print "pre-deps hook"; open(\\'src/git_pre_deps_hooked\\', \\'w\
\').write(\\'git_pre_deps_hooked\\')'], | |
696 }, | |
697 { | |
698 'action': ['python', '-c', 'import sys; sys.exit(1)'], | |
699 } | |
700 ] | |
701 """ % { | |
702 'git_base': self.git_base, | |
703 'hash1': self.git_hashes['repo_1'][2][0][:7], | |
704 'hash2': self.git_hashes['repo_2'][1][0][:7], | |
705 }, | |
706 'origin': 'git/repo_5@3\n', | |
707 }) | |
708 | |
709 | |
710 class FakeRepoTransitive(FakeReposBase): | |
711 """Implements populateSvn()""" | |
712 | |
713 def populateSvn(self): | |
714 """Creates a few revisions of changes including a DEPS file.""" | |
715 # Repos | |
716 subprocess2.check_call( | |
717 ['svn', 'checkout', self.svn_base, self.svn_checkout, | |
718 '-q', '--non-interactive', '--no-auth-cache', | |
719 '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) | |
720 assert os.path.isdir(join(self.svn_checkout, '.svn')) | |
721 | |
722 def file_system(rev): | |
723 DEPS = """deps = { | |
724 'src/different_repo': '%(svn_base)strunk/third_party', | |
725 'src/different_repo_fixed': '%(svn_base)strunk/third_party@1', | |
726 'src/same_repo': '/trunk/third_party', | |
727 'src/same_repo_fixed': '/trunk/third_party@1', | |
728 }""" % { 'svn_base': self.svn_base } | |
729 return { | |
730 'trunk/src/DEPS': DEPS, | |
731 'trunk/src/origin': 'svn/trunk/src@%(rev)d' % { 'rev': rev }, | |
732 'trunk/third_party/origin': | |
733 'svn/trunk/third_party@%(rev)d' % { 'rev': rev }, | |
734 } | |
735 | |
736 # We make three commits. We use always the same DEPS contents but | |
737 # - 'trunk/src/origin' contains 'svn/trunk/src/origin@rX' | |
738 # - 'trunk/third_party/origin' contains 'svn/trunk/third_party/origin@rX' | |
739 # where 'X' is the revision number. | |
740 # So the 'origin' files will change in every commit. | |
741 self._commit_svn(file_system(1)) | |
742 self._commit_svn(file_system(2)) | |
743 self._commit_svn(file_system(3)) | |
744 # We rewrite the timestamps so we can test that '--transitive' will take the | |
745 # parent timestamp on different repositories and the parent revision | |
746 # otherwise. | |
747 self._set_svn_commit_date('1', '2011-10-01T03:00:00.000000Z') | |
748 self._set_svn_commit_date('2', '2011-10-09T03:00:00.000000Z') | |
749 self._set_svn_commit_date('3', '2011-10-02T03:00:00.000000Z') | |
750 | |
751 def populateGit(self): | |
752 pass | |
753 | |
754 | |
755 class FakeRepoSkiaDEPS(FakeReposBase): | |
756 """Simulates the Skia DEPS transition in Chrome.""" | |
757 | |
758 NB_GIT_REPOS = 5 | |
759 | |
760 DEPS_svn_pre = """deps = { | |
761 'src/third_party/skia/gyp': '%(svn_base)sskia/gyp', | |
762 'src/third_party/skia/include': '%(svn_base)sskia/include', | |
763 'src/third_party/skia/src': '%(svn_base)sskia/src', | |
764 }""" | |
765 | |
766 DEPS_git_pre = """deps = { | |
767 'src/third_party/skia/gyp': '%(git_base)srepo_3', | |
768 'src/third_party/skia/include': '%(git_base)srepo_4', | |
769 'src/third_party/skia/src': '%(git_base)srepo_5', | |
770 }""" | |
771 | |
772 DEPS_post = """deps = { | |
773 'src/third_party/skia': '%(git_base)srepo_1', | |
774 }""" | |
775 | |
776 def populateSvn(self): | |
777 """Create revisions which simulate the Skia DEPS transition in Chrome.""" | |
778 subprocess2.check_call( | |
779 ['svn', 'checkout', self.svn_base, self.svn_checkout, | |
780 '-q', '--non-interactive', '--no-auth-cache', | |
781 '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) | |
782 assert os.path.isdir(join(self.svn_checkout, '.svn')) | |
783 | |
784 # Skia repo. | |
785 self._commit_svn({ | |
786 'skia/skia_base_file': 'root-level file.', | |
787 'skia/gyp/gyp_file': 'file in the gyp directory', | |
788 'skia/include/include_file': 'file in the include directory', | |
789 'skia/src/src_file': 'file in the src directory', | |
790 }) | |
791 | |
792 # Chrome repo. | |
793 self._commit_svn({ | |
794 'trunk/src/DEPS': self.DEPS_svn_pre % {'svn_base': self.svn_base}, | |
795 'trunk/src/myfile': 'svn/trunk/src@1' | |
796 }) | |
797 self._commit_svn({ | |
798 'trunk/src/DEPS': self.DEPS_post % {'git_base': self.git_base}, | |
799 'trunk/src/myfile': 'svn/trunk/src@2' | |
800 }) | |
801 | |
802 def populateGit(self): | |
803 # Skia repo. | |
804 self._commit_git('repo_1', { | |
805 'skia_base_file': 'root-level file.', | |
806 'gyp/gyp_file': 'file in the gyp directory', | |
807 'include/include_file': 'file in the include directory', | |
808 'src/src_file': 'file in the src directory', | |
809 }) | |
810 self._commit_git('repo_3', { # skia/gyp | |
811 'gyp_file': 'file in the gyp directory', | |
812 }) | |
813 self._commit_git('repo_4', { # skia/include | |
814 'include_file': 'file in the include directory', | |
815 }) | |
816 self._commit_git('repo_5', { # skia/src | |
817 'src_file': 'file in the src directory', | |
818 }) | |
819 | |
820 # Chrome repo. | |
821 self._commit_git('repo_2', { | |
822 'DEPS': self.DEPS_git_pre % {'git_base': self.git_base}, | |
823 'myfile': 'svn/trunk/src@1' | |
824 }) | |
825 self._commit_git('repo_2', { | |
826 'DEPS': self.DEPS_post % {'git_base': self.git_base}, | |
827 'myfile': 'svn/trunk/src@2' | |
828 }) | |
829 | |
830 | |
831 class FakeRepoBlinkDEPS(FakeReposBase): | |
832 """Simulates the Blink DEPS transition in Chrome.""" | |
833 | |
834 NB_GIT_REPOS = 2 | |
835 DEPS_pre = 'deps = {"src/third_party/WebKit": "%(git_base)srepo_2",}' | |
836 DEPS_post = 'deps = {}' | |
837 | |
838 def populateGit(self): | |
839 # Blink repo. | |
840 self._commit_git('repo_2', { | |
841 'OWNERS': 'OWNERS-pre', | |
842 'Source/exists_always': '_ignored_', | |
843 'Source/exists_before_but_not_after': '_ignored_', | |
844 }) | |
845 | |
846 # Chrome repo. | |
847 self._commit_git('repo_1', { | |
848 'DEPS': self.DEPS_pre % {'git_base': self.git_base}, | |
849 'myfile': 'myfile@1', | |
850 '.gitignore': '/third_party/WebKit', | |
851 }) | |
852 self._commit_git('repo_1', { | |
853 'DEPS': self.DEPS_post % {'git_base': self.git_base}, | |
854 'myfile': 'myfile@2', | |
855 '.gitignore': '', | |
856 'third_party/WebKit/OWNERS': 'OWNERS-post', | |
857 'third_party/WebKit/Source/exists_always': '_ignored_', | |
858 'third_party/WebKit/Source/exists_after_but_not_before': '_ignored', | |
859 }) | |
860 | |
861 def populateSvn(self): | |
862 raise NotImplementedError() | |
863 | |
864 | |
865 class FakeReposTestBase(trial_dir.TestCase): | |
866 """This is vaguely inspired by twisted.""" | |
867 # Static FakeRepos instances. Lazy loaded. | |
868 CACHED_FAKE_REPOS = {} | |
869 # Override if necessary. | |
870 FAKE_REPOS_CLASS = FakeRepos | |
871 | |
872 def setUp(self): | |
873 super(FakeReposTestBase, self).setUp() | |
874 if not self.FAKE_REPOS_CLASS in self.CACHED_FAKE_REPOS: | |
875 self.CACHED_FAKE_REPOS[self.FAKE_REPOS_CLASS] = self.FAKE_REPOS_CLASS() | |
876 self.FAKE_REPOS = self.CACHED_FAKE_REPOS[self.FAKE_REPOS_CLASS] | |
877 # No need to call self.FAKE_REPOS.setUp(), it will be called by the child | |
878 # class. | |
879 # Do not define tearDown(), since super's version does the right thing and | |
880 # self.FAKE_REPOS is kept across tests. | |
881 | |
882 @property | |
883 def svn_base(self): | |
884 """Shortcut.""" | |
885 return self.FAKE_REPOS.svn_base | |
886 | |
887 @property | |
888 def git_base(self): | |
889 """Shortcut.""" | |
890 return self.FAKE_REPOS.git_base | |
891 | |
892 def checkString(self, expected, result, msg=None): | |
893 """Prints the diffs to ease debugging.""" | |
894 if expected != result: | |
895 # Strip the begining | |
896 while expected and result and expected[0] == result[0]: | |
897 expected = expected[1:] | |
898 result = result[1:] | |
899 # The exception trace makes it hard to read so dump it too. | |
900 if '\n' in result: | |
901 print result | |
902 self.assertEquals(expected, result, msg) | |
903 | |
904 def check(self, expected, results): | |
905 """Checks stdout, stderr, returncode.""" | |
906 self.checkString(expected[0], results[0]) | |
907 self.checkString(expected[1], results[1]) | |
908 self.assertEquals(expected[2], results[2]) | |
909 | |
910 def assertTree(self, tree, tree_root=None): | |
911 """Diff the checkout tree with a dict.""" | |
912 if not tree_root: | |
913 tree_root = self.root_dir | |
914 actual = read_tree(tree_root) | |
915 diff = dict_diff(tree, actual) | |
916 if diff: | |
917 logging.debug('Actual %s\n%s' % (tree_root, pprint.pformat(actual))) | |
918 logging.debug('Expected\n%s' % pprint.pformat(tree)) | |
919 logging.debug('Diff\n%s' % pprint.pformat(diff)) | |
920 self.assertEquals(diff, {}) | |
921 | |
922 def mangle_svn_tree(self, *args): | |
923 """Creates a 'virtual directory snapshot' to compare with the actual result | |
924 on disk.""" | |
925 result = {} | |
926 for item, new_root in args: | |
927 old_root, rev = item.split('@', 1) | |
928 tree = self.FAKE_REPOS.svn_revs[int(rev)] | |
929 for k, v in tree.iteritems(): | |
930 if not k.startswith(old_root): | |
931 continue | |
932 item = k[len(old_root) + 1:] | |
933 if item.startswith('.'): | |
934 continue | |
935 result[join(new_root, item).replace(os.sep, '/')] = v | |
936 return result | |
937 | |
938 def mangle_git_tree(self, *args): | |
939 """Creates a 'virtual directory snapshot' to compare with the actual result | |
940 on disk.""" | |
941 result = {} | |
942 for item, new_root in args: | |
943 repo, rev = item.split('@', 1) | |
944 tree = self.gittree(repo, rev) | |
945 for k, v in tree.iteritems(): | |
946 result[join(new_root, k)] = v | |
947 return result | |
948 | |
949 def githash(self, repo, rev): | |
950 """Sort-hand: Returns the hash for a git 'revision'.""" | |
951 return self.FAKE_REPOS.git_hashes[repo][int(rev)][0] | |
952 | |
953 def gittree(self, repo, rev): | |
954 """Sort-hand: returns the directory tree for a git 'revision'.""" | |
955 return self.FAKE_REPOS.git_hashes[repo][int(rev)][1] | |
956 | |
957 | |
958 def main(argv): | |
959 fake = FakeRepos() | |
960 print 'Using %s' % fake.root_dir | |
961 try: | |
962 fake.set_up_svn() | |
963 fake.set_up_git() | |
964 print('Fake setup, press enter to quit or Ctrl-C to keep the checkouts.') | |
965 sys.stdin.readline() | |
966 except KeyboardInterrupt: | |
967 trial_dir.TrialDir.SHOULD_LEAK.leak = True | |
968 return 0 | |
969 | |
970 | |
971 if __name__ == '__main__': | |
972 sys.exit(main(sys.argv)) | |
OLD | NEW |