OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Generate fake repositories for testing.""" | 6 """Generate fake repositories for testing.""" |
7 | 7 |
8 import atexit | 8 import atexit |
9 import datetime | 9 import datetime |
10 import errno | 10 import errno |
11 import logging | 11 import logging |
12 import os | 12 import os |
13 import pprint | 13 import pprint |
14 import re | 14 import re |
15 import socket | 15 import socket |
16 import subprocess | |
17 import sys | 16 import sys |
18 import tempfile | 17 import tempfile |
19 import time | 18 import time |
20 | 19 |
21 # trial_dir must be first for non-system libraries. | 20 # trial_dir must be first for non-system libraries. |
22 from tests import trial_dir | 21 from tests import trial_dir |
23 import gclient_utils | 22 import gclient_utils |
24 import scm | 23 import scm |
25 | 24 import subprocess2 |
26 ## Utility functions | |
27 | |
28 | |
29 def kill_pid(pid): | |
30 """Kills a process by its process id.""" | |
31 try: | |
32 # Unable to import 'module' | |
33 # pylint: disable=F0401 | |
34 import signal | |
35 return os.kill(pid, signal.SIGKILL) | |
36 except ImportError: | |
37 pass | |
38 | |
39 | |
40 def kill_win(process): | |
41 """Kills a process with its windows handle. | |
42 | |
43 Has no effect on other platforms. | |
44 """ | |
45 try: | |
46 # Unable to import 'module' | |
47 # pylint: disable=F0401 | |
48 import win32process | |
49 # Access to a protected member _handle of a client class | |
50 # pylint: disable=W0212 | |
51 return win32process.TerminateProcess(process._handle, -1) | |
52 except ImportError: | |
53 pass | |
54 | |
55 | |
56 def add_kill(): | |
57 """Adds kill() method to subprocess.Popen for python <2.6""" | |
58 if hasattr(subprocess.Popen, 'kill'): | |
59 return | |
60 | |
61 if sys.platform == 'win32': | |
62 subprocess.Popen.kill = kill_win | |
63 else: | |
64 def kill_nix(process): | |
65 return kill_pid(process.pid) | |
66 subprocess.Popen.kill = kill_nix | |
67 | 25 |
68 | 26 |
69 def write(path, content): | 27 def write(path, content): |
70 f = open(path, 'wb') | 28 f = open(path, 'wb') |
71 f.write(content) | 29 f.write(content) |
72 f.close() | 30 f.close() |
73 | 31 |
74 | 32 |
75 join = os.path.join | 33 join = os.path.join |
76 | 34 |
77 | 35 |
78 def check_call(*args, **kwargs): | |
79 logging.debug(args[0]) | |
80 subprocess.check_call(*args, **kwargs) | |
81 | |
82 | |
83 def Popen(*args, **kwargs): | |
84 kwargs.setdefault('stdout', subprocess.PIPE) | |
85 kwargs.setdefault('stderr', subprocess.STDOUT) | |
86 logging.debug(args[0]) | |
87 return subprocess.Popen(*args, **kwargs) | |
88 | |
89 | |
90 def read_tree(tree_root): | 36 def read_tree(tree_root): |
91 """Returns a dict of all the files in a tree. Defaults to self.root_dir.""" | 37 """Returns a dict of all the files in a tree. Defaults to self.root_dir.""" |
92 tree = {} | 38 tree = {} |
93 for root, dirs, files in os.walk(tree_root): | 39 for root, dirs, files in os.walk(tree_root): |
94 for d in filter(lambda x: x.startswith('.'), dirs): | 40 for d in filter(lambda x: x.startswith('.'), dirs): |
95 dirs.remove(d) | 41 dirs.remove(d) |
96 for f in [join(root, f) for f in files if not f.startswith('.')]: | 42 for f in [join(root, f) for f in files if not f.startswith('.')]: |
97 filepath = f[len(tree_root) + 1:].replace(os.sep, '/') | 43 filepath = f[len(tree_root) + 1:].replace(os.sep, '/') |
98 assert len(filepath), f | 44 assert len(filepath), f |
99 tree[filepath] = open(join(root, f), 'rU').read() | 45 tree[filepath] = open(join(root, f), 'rU').read() |
(...skipping 16 matching lines...) Expand all Loading... |
116 def commit_svn(repo, usr, pwd): | 62 def commit_svn(repo, usr, pwd): |
117 """Commits the changes and returns the new revision number.""" | 63 """Commits the changes and returns the new revision number.""" |
118 to_add = [] | 64 to_add = [] |
119 to_remove = [] | 65 to_remove = [] |
120 for status, filepath in scm.SVN.CaptureStatus(repo): | 66 for status, filepath in scm.SVN.CaptureStatus(repo): |
121 if status[0] == '?': | 67 if status[0] == '?': |
122 to_add.append(filepath) | 68 to_add.append(filepath) |
123 elif status[0] == '!': | 69 elif status[0] == '!': |
124 to_remove.append(filepath) | 70 to_remove.append(filepath) |
125 if to_add: | 71 if to_add: |
126 check_call(['svn', 'add', '--no-auto-props', '-q'] + to_add, cwd=repo) | 72 subprocess2.check_output( |
| 73 ['svn', 'add', '--no-auto-props', '-q'] + to_add, cwd=repo) |
127 if to_remove: | 74 if to_remove: |
128 check_call(['svn', 'remove', '-q'] + to_remove, cwd=repo) | 75 subprocess2.check_output(['svn', 'remove', '-q'] + to_remove, cwd=repo) |
129 proc = Popen( | 76 |
| 77 out = subprocess2.check_output( |
130 ['svn', 'commit', repo, '-m', 'foo', '--non-interactive', | 78 ['svn', 'commit', repo, '-m', 'foo', '--non-interactive', |
131 '--no-auth-cache', | 79 '--no-auth-cache', |
132 '--username', usr, '--password', pwd], | 80 '--username', usr, '--password', pwd], |
133 cwd=repo) | 81 cwd=repo) |
134 out, err = proc.communicate() | |
135 match = re.search(r'(\d+)', out) | 82 match = re.search(r'(\d+)', out) |
136 if not match: | 83 if not match: |
137 raise Exception('Commit failed', out, err, proc.returncode) | 84 raise Exception('Commit failed', out) |
138 rev = match.group(1) | 85 rev = match.group(1) |
139 st = Popen(['svn', 'status'], cwd=repo).communicate()[0] | 86 status = subprocess2.check_output(['svn', 'status'], cwd=repo) |
140 assert len(st) == 0, st | 87 assert len(status) == 0, status |
141 logging.debug('At revision %s' % rev) | 88 logging.debug('At revision %s' % rev) |
142 return rev | 89 return rev |
143 | 90 |
144 | 91 |
145 def commit_git(repo): | 92 def commit_git(repo): |
146 """Commits the changes and returns the new hash.""" | 93 """Commits the changes and returns the new hash.""" |
147 check_call(['git', 'add', '-A', '-f'], cwd=repo) | 94 subprocess2.check_call(['git', 'add', '-A', '-f'], cwd=repo) |
148 check_call(['git', 'commit', '-q', '--message', 'foo'], cwd=repo) | 95 subprocess2.check_call(['git', 'commit', '-q', '--message', 'foo'], cwd=repo) |
149 rev = Popen(['git', 'show-ref', '--head', 'HEAD'], | 96 rev = subprocess2.check_output( |
150 cwd=repo).communicate()[0].split(' ', 1)[0] | 97 ['git', 'show-ref', '--head', 'HEAD'], cwd=repo).split(' ', 1)[0] |
151 logging.debug('At revision %s' % rev) | 98 logging.debug('At revision %s' % rev) |
152 return rev | 99 return rev |
153 | 100 |
154 | 101 |
155 def test_port(host, port): | 102 def test_port(host, port): |
156 s = socket.socket() | 103 s = socket.socket() |
157 try: | 104 try: |
158 return s.connect_ex((host, port)) == 0 | 105 return s.connect_ex((host, port)) == 0 |
159 finally: | 106 finally: |
160 s.close() | 107 s.close() |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
264 | 211 |
265 @property | 212 @property |
266 def root_dir(self): | 213 def root_dir(self): |
267 return self.trial.root_dir | 214 return self.trial.root_dir |
268 | 215 |
269 def set_up(self): | 216 def set_up(self): |
270 """All late initialization comes here.""" | 217 """All late initialization comes here.""" |
271 self.cleanup_dirt() | 218 self.cleanup_dirt() |
272 if not self.root_dir: | 219 if not self.root_dir: |
273 try: | 220 try: |
274 add_kill() | |
275 # self.root_dir is not set before this call. | 221 # self.root_dir is not set before this call. |
276 self.trial.set_up() | 222 self.trial.set_up() |
277 self.git_root = join(self.root_dir, 'git') | 223 self.git_root = join(self.root_dir, 'git') |
278 self.svn_checkout = join(self.root_dir, 'svn_checkout') | 224 self.svn_checkout = join(self.root_dir, 'svn_checkout') |
279 self.svn_repo = join(self.root_dir, 'svn') | 225 self.svn_repo = join(self.root_dir, 'svn') |
280 finally: | 226 finally: |
281 # Registers cleanup. | 227 # Registers cleanup. |
282 atexit.register(self.tear_down) | 228 atexit.register(self.tear_down) |
283 | 229 |
284 def cleanup_dirt(self): | 230 def cleanup_dirt(self): |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
321 | 267 |
322 def tear_down_git(self): | 268 def tear_down_git(self): |
323 if self.gitdaemon: | 269 if self.gitdaemon: |
324 logging.debug('Killing git-daemon pid %s' % self.gitdaemon.pid) | 270 logging.debug('Killing git-daemon pid %s' % self.gitdaemon.pid) |
325 self.gitdaemon.kill() | 271 self.gitdaemon.kill() |
326 self.gitdaemon = None | 272 self.gitdaemon = None |
327 if self.git_pid_file: | 273 if self.git_pid_file: |
328 pid = int(self.git_pid_file.read()) | 274 pid = int(self.git_pid_file.read()) |
329 self.git_pid_file.close() | 275 self.git_pid_file.close() |
330 logging.debug('Killing git daemon pid %s' % pid) | 276 logging.debug('Killing git daemon pid %s' % pid) |
331 kill_pid(pid) | 277 subprocess2.kill_pid(pid) |
332 self.git_pid_file = None | 278 self.git_pid_file = None |
333 wait_for_port_to_free(self.host, self.git_port) | 279 wait_for_port_to_free(self.host, self.git_port) |
334 self.git_port = None | 280 self.git_port = None |
335 self.git_base = None | 281 self.git_base = None |
336 if not self.trial.SHOULD_LEAK: | 282 if not self.trial.SHOULD_LEAK: |
337 logging.debug('Removing %s' % self.git_root) | 283 logging.debug('Removing %s' % self.git_root) |
338 gclient_utils.rmtree(self.git_root) | 284 gclient_utils.rmtree(self.git_root) |
339 else: | 285 else: |
340 return False | 286 return False |
341 return True | 287 return True |
(...skipping 14 matching lines...) Expand all Loading... |
356 os.remove(join(root, k)) | 302 os.remove(join(root, k)) |
357 else: | 303 else: |
358 write(join(root, k), v) | 304 write(join(root, k), v) |
359 | 305 |
360 def set_up_svn(self): | 306 def set_up_svn(self): |
361 """Creates subversion repositories and start the servers.""" | 307 """Creates subversion repositories and start the servers.""" |
362 self.set_up() | 308 self.set_up() |
363 if self.svnserve: | 309 if self.svnserve: |
364 return True | 310 return True |
365 try: | 311 try: |
366 check_call(['svnadmin', 'create', self.svn_repo]) | 312 subprocess2.check_call(['svnadmin', 'create', self.svn_repo]) |
367 except OSError, e: | 313 except subprocess2.CalledProcessError, e: |
368 logging.debug('Failed with : %s' % e) | 314 logging.debug('Failed with : %s' % e) |
369 return False | 315 return False |
370 write(join(self.svn_repo, 'conf', 'svnserve.conf'), | 316 write(join(self.svn_repo, 'conf', 'svnserve.conf'), |
371 '[general]\n' | 317 '[general]\n' |
372 'anon-access = read\n' | 318 'anon-access = read\n' |
373 'auth-access = write\n' | 319 'auth-access = write\n' |
374 'password-db = passwd\n') | 320 'password-db = passwd\n') |
375 text = '[users]\n' | 321 text = '[users]\n' |
376 text += ''.join('%s = %s\n' % (usr, pwd) for usr, pwd in self.USERS) | 322 text += ''.join('%s = %s\n' % (usr, pwd) for usr, pwd in self.USERS) |
377 write(join(self.svn_repo, 'conf', 'passwd'), text) | 323 write(join(self.svn_repo, 'conf', 'passwd'), text) |
378 | 324 |
379 # Mac 10.6 ships with a buggy subversion build and we need this line | 325 # Mac 10.6 ships with a buggy subversion build and we need this line |
380 # to work around the bug. | 326 # to work around the bug. |
381 write(join(self.svn_repo, 'db', 'fsfs.conf'), | 327 write(join(self.svn_repo, 'db', 'fsfs.conf'), |
382 '[rep-sharing]\n' | 328 '[rep-sharing]\n' |
383 'enable-rep-sharing = false\n') | 329 'enable-rep-sharing = false\n') |
384 | 330 |
385 # Start the daemon. | 331 # Start the daemon. |
386 self.svn_port = find_free_port(self.host, 10000) | 332 self.svn_port = find_free_port(self.host, 10000) |
387 logging.debug('Using port %d' % self.svn_port) | 333 logging.debug('Using port %d' % self.svn_port) |
388 cmd = ['svnserve', '-d', '--foreground', '-r', self.root_dir, | 334 cmd = ['svnserve', '-d', '--foreground', '-r', self.root_dir, |
389 '--listen-port=%d' % self.svn_port] | 335 '--listen-port=%d' % self.svn_port] |
390 if self.host == '127.0.0.1': | 336 if self.host == '127.0.0.1': |
391 cmd.append('--listen-host=' + self.host) | 337 cmd.append('--listen-host=' + self.host) |
392 self.check_port_is_free(self.svn_port) | 338 self.check_port_is_free(self.svn_port) |
393 self.svnserve = Popen(cmd, cwd=self.svn_repo) | 339 self.svnserve = subprocess2.Popen( |
| 340 cmd, |
| 341 cwd=self.svn_repo, |
| 342 stdout=subprocess2.PIPE, |
| 343 stderr=subprocess2.PIPE) |
394 wait_for_port_to_bind(self.host, self.svn_port, self.svnserve) | 344 wait_for_port_to_bind(self.host, self.svn_port, self.svnserve) |
395 self.svn_base = 'svn://%s:%d/svn/' % (self.host, self.svn_port) | 345 self.svn_base = 'svn://%s:%d/svn/' % (self.host, self.svn_port) |
396 self.populateSvn() | 346 self.populateSvn() |
397 self.svn_dirty = False | 347 self.svn_dirty = False |
398 return True | 348 return True |
399 | 349 |
400 def set_up_git(self): | 350 def set_up_git(self): |
401 """Creates git repositories and start the servers.""" | 351 """Creates git repositories and start the servers.""" |
402 self.set_up() | 352 self.set_up() |
403 if self.gitdaemon: | 353 if self.gitdaemon: |
404 return True | 354 return True |
405 if sys.platform == 'win32': | 355 if sys.platform == 'win32': |
406 return False | 356 return False |
407 assert self.git_pid_file == None | 357 assert self.git_pid_file == None |
408 for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]: | 358 for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]: |
409 check_call(['git', 'init', '-q', join(self.git_root, repo)]) | 359 subprocess2.check_call(['git', 'init', '-q', join(self.git_root, repo)]) |
410 self.git_hashes[repo] = [None] | 360 self.git_hashes[repo] = [None] |
411 self.git_port = find_free_port(self.host, 20000) | 361 self.git_port = find_free_port(self.host, 20000) |
412 self.git_base = 'git://%s:%d/git/' % (self.host, self.git_port) | 362 self.git_base = 'git://%s:%d/git/' % (self.host, self.git_port) |
413 # Start the daemon. | 363 # Start the daemon. |
414 self.git_pid_file = tempfile.NamedTemporaryFile() | 364 self.git_pid_file = tempfile.NamedTemporaryFile() |
415 cmd = ['git', 'daemon', | 365 cmd = ['git', 'daemon', |
416 '--export-all', | 366 '--export-all', |
417 '--reuseaddr', | 367 '--reuseaddr', |
418 '--base-path=' + self.root_dir, | 368 '--base-path=' + self.root_dir, |
419 '--pid-file=' + self.git_pid_file.name, | 369 '--pid-file=' + self.git_pid_file.name, |
420 '--port=%d' % self.git_port] | 370 '--port=%d' % self.git_port] |
421 if self.host == '127.0.0.1': | 371 if self.host == '127.0.0.1': |
422 cmd.append('--listen=' + self.host) | 372 cmd.append('--listen=' + self.host) |
423 self.check_port_is_free(self.git_port) | 373 self.check_port_is_free(self.git_port) |
424 self.gitdaemon = Popen(cmd, cwd=self.root_dir) | 374 self.gitdaemon = subprocess2.Popen( |
| 375 cmd, |
| 376 cwd=self.root_dir, |
| 377 stdout=subprocess2.PIPE, |
| 378 stderr=subprocess2.PIPE) |
425 wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon) | 379 wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon) |
426 self.populateGit() | 380 self.populateGit() |
427 self.git_dirty = False | 381 self.git_dirty = False |
428 return True | 382 return True |
429 | 383 |
430 def _commit_svn(self, tree): | 384 def _commit_svn(self, tree): |
431 self._genTree(self.svn_checkout, tree) | 385 self._genTree(self.svn_checkout, tree) |
432 commit_svn(self.svn_checkout, self.USERS[0][0], self.USERS[0][1]) | 386 commit_svn(self.svn_checkout, self.USERS[0][0], self.USERS[0][1]) |
433 if self.svn_revs and self.svn_revs[-1]: | 387 if self.svn_revs and self.svn_revs[-1]: |
434 new_tree = self.svn_revs[-1].copy() | 388 new_tree = self.svn_revs[-1].copy() |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
466 raise NotImplementedError() | 420 raise NotImplementedError() |
467 | 421 |
468 | 422 |
469 class FakeRepos(FakeReposBase): | 423 class FakeRepos(FakeReposBase): |
470 """Implements populateSvn() and populateGit().""" | 424 """Implements populateSvn() and populateGit().""" |
471 NB_GIT_REPOS = 4 | 425 NB_GIT_REPOS = 4 |
472 | 426 |
473 def populateSvn(self): | 427 def populateSvn(self): |
474 """Creates a few revisions of changes including DEPS files.""" | 428 """Creates a few revisions of changes including DEPS files.""" |
475 # Repos | 429 # Repos |
476 check_call(['svn', 'checkout', self.svn_base, self.svn_checkout, | 430 subprocess2.check_call( |
477 '-q', '--non-interactive', '--no-auth-cache', | 431 ['svn', 'checkout', self.svn_base, self.svn_checkout, |
478 '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) | 432 '-q', '--non-interactive', '--no-auth-cache', |
| 433 '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) |
479 assert os.path.isdir(join(self.svn_checkout, '.svn')) | 434 assert os.path.isdir(join(self.svn_checkout, '.svn')) |
480 def file_system(rev, DEPS): | 435 def file_system(rev, DEPS): |
481 fs = { | 436 fs = { |
482 'origin': 'svn@%(rev)d\n', | 437 'origin': 'svn@%(rev)d\n', |
483 'trunk/origin': 'svn/trunk@%(rev)d\n', | 438 'trunk/origin': 'svn/trunk@%(rev)d\n', |
484 'trunk/src/origin': 'svn/trunk/src@%(rev)d\n', | 439 'trunk/src/origin': 'svn/trunk/src@%(rev)d\n', |
485 'trunk/src/third_party/origin': 'svn/trunk/src/third_party@%(rev)d\n', | 440 'trunk/src/third_party/origin': 'svn/trunk/src/third_party@%(rev)d\n', |
486 'trunk/other/origin': 'src/trunk/other@%(rev)d\n', | 441 'trunk/other/origin': 'src/trunk/other@%(rev)d\n', |
487 'trunk/third_party/origin': 'svn/trunk/third_party@%(rev)d\n', | 442 'trunk/third_party/origin': 'svn/trunk/third_party@%(rev)d\n', |
488 'trunk/third_party/foo/origin': 'svn/trunk/third_party/foo@%(rev)d\n', | 443 'trunk/third_party/foo/origin': 'svn/trunk/third_party/foo@%(rev)d\n', |
(...skipping 288 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
777 fake.set_up_git() | 732 fake.set_up_git() |
778 print('Fake setup, press enter to quit or Ctrl-C to keep the checkouts.') | 733 print('Fake setup, press enter to quit or Ctrl-C to keep the checkouts.') |
779 sys.stdin.readline() | 734 sys.stdin.readline() |
780 except KeyboardInterrupt: | 735 except KeyboardInterrupt: |
781 trial_dir.TrialDir.SHOULD_LEAK.leak = True | 736 trial_dir.TrialDir.SHOULD_LEAK.leak = True |
782 return 0 | 737 return 0 |
783 | 738 |
784 | 739 |
785 if __name__ == '__main__': | 740 if __name__ == '__main__': |
786 sys.exit(main(sys.argv)) | 741 sys.exit(main(sys.argv)) |
OLD | NEW |