OLD | NEW |
1 # coding=utf8 | 1 # coding=utf8 |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 """Manages a project checkout. | 5 """Manages a project checkout. |
6 | 6 |
7 Includes support for svn, git-svn and git. | 7 Includes support only for git. |
8 """ | 8 """ |
9 | 9 |
10 import fnmatch | 10 import fnmatch |
11 import logging | 11 import logging |
12 import os | 12 import os |
13 import re | 13 import re |
14 import shutil | 14 import shutil |
15 import subprocess | 15 import subprocess |
16 import sys | 16 import sys |
17 import tempfile | 17 import tempfile |
(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
154 def revisions(self, rev1, rev2): | 154 def revisions(self, rev1, rev2): |
155 """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]). | 155 """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]). |
156 | 156 |
157 If rev2 is None, it means 'HEAD'. | 157 If rev2 is None, it means 'HEAD'. |
158 | 158 |
159 Returns None if there is no link between the two. | 159 Returns None if there is no link between the two. |
160 """ | 160 """ |
161 raise NotImplementedError() | 161 raise NotImplementedError() |
162 | 162 |
163 | 163 |
164 class RawCheckout(CheckoutBase): | |
165 """Used to apply a patch locally without any intent to commit it. | |
166 | |
167 To be used by the try server. | |
168 """ | |
169 def prepare(self, revision): | |
170 """Stubbed out.""" | |
171 pass | |
172 | |
173 def apply_patch(self, patches, post_processors=None, verbose=False): | |
174 """Ignores svn properties.""" | |
175 post_processors = post_processors or self.post_processors or [] | |
176 for p in patches: | |
177 stdout = [] | |
178 try: | |
179 filepath = os.path.join(self.project_path, p.filename) | |
180 if p.is_delete: | |
181 os.remove(filepath) | |
182 assert(not os.path.exists(filepath)) | |
183 stdout.append('Deleted.') | |
184 else: | |
185 dirname = os.path.dirname(p.filename) | |
186 full_dir = os.path.join(self.project_path, dirname) | |
187 if dirname and not os.path.isdir(full_dir): | |
188 os.makedirs(full_dir) | |
189 stdout.append('Created missing directory %s.' % dirname) | |
190 | |
191 if p.is_binary: | |
192 content = p.get() | |
193 with open(filepath, 'wb') as f: | |
194 f.write(content) | |
195 stdout.append('Added binary file %d bytes.' % len(content)) | |
196 else: | |
197 if p.source_filename: | |
198 if not p.is_new: | |
199 raise PatchApplicationFailed( | |
200 p, | |
201 'File has a source filename specified but is not new') | |
202 # Copy the file first. | |
203 if os.path.isfile(filepath): | |
204 raise PatchApplicationFailed( | |
205 p, 'File exist but was about to be overwriten') | |
206 shutil.copy2( | |
207 os.path.join(self.project_path, p.source_filename), filepath) | |
208 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename)) | |
209 if p.diff_hunks: | |
210 cmd = ['patch', '-u', '--binary', '-p%s' % p.patchlevel] | |
211 if verbose: | |
212 cmd.append('--verbose') | |
213 env = os.environ.copy() | |
214 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch') | |
215 try: | |
216 stdout.append( | |
217 subprocess2.check_output( | |
218 cmd, | |
219 stdin=p.get(False), | |
220 stderr=subprocess2.STDOUT, | |
221 cwd=self.project_path, | |
222 timeout=GLOBAL_TIMEOUT, | |
223 env=env)) | |
224 finally: | |
225 shutil.rmtree(env['TMPDIR']) | |
226 elif p.is_new and not os.path.exists(filepath): | |
227 # There is only a header. Just create the file. | |
228 open(filepath, 'w').close() | |
229 stdout.append('Created an empty file.') | |
230 for post in post_processors: | |
231 post(self, p) | |
232 if verbose: | |
233 print p.filename | |
234 print align_stdout(stdout) | |
235 except OSError, e: | |
236 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e)) | |
237 except subprocess.CalledProcessError, e: | |
238 raise PatchApplicationFailed( | |
239 p, | |
240 'While running %s;\n%s%s' % ( | |
241 ' '.join(e.cmd), | |
242 align_stdout(stdout), | |
243 align_stdout([getattr(e, 'stdout', '')]))) | |
244 | |
245 def commit(self, commit_message, user): | |
246 """Stubbed out.""" | |
247 raise NotImplementedError('RawCheckout can\'t commit') | |
248 | |
249 def revisions(self, _rev1, _rev2): | |
250 return None | |
251 | |
252 | |
253 class SvnConfig(object): | |
254 """Parses a svn configuration file.""" | |
255 def __init__(self, svn_config_dir=None): | |
256 super(SvnConfig, self).__init__() | |
257 self.svn_config_dir = svn_config_dir | |
258 self.default = not bool(self.svn_config_dir) | |
259 if not self.svn_config_dir: | |
260 if sys.platform == 'win32': | |
261 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion') | |
262 else: | |
263 self.svn_config_dir = os.path.expanduser( | |
264 os.path.join('~', '.subversion')) | |
265 svn_config_file = os.path.join(self.svn_config_dir, 'config') | |
266 parser = configparser.SafeConfigParser() | |
267 if os.path.isfile(svn_config_file): | |
268 parser.read(svn_config_file) | |
269 else: | |
270 parser.add_section('auto-props') | |
271 self.auto_props = dict(parser.items('auto-props')) | |
272 | |
273 | |
274 class SvnMixIn(object): | |
275 """MixIn class to add svn commands common to both svn and git-svn clients.""" | |
276 # These members need to be set by the subclass. | |
277 commit_user = None | |
278 commit_pwd = None | |
279 svn_url = None | |
280 project_path = None | |
281 # Override at class level when necessary. If used, --non-interactive is | |
282 # implied. | |
283 svn_config = SvnConfig() | |
284 # Set to True when non-interactivity is necessary but a custom subversion | |
285 # configuration directory is not necessary. | |
286 non_interactive = False | |
287 | |
288 def _add_svn_flags(self, args, non_interactive, credentials=True): | |
289 args = ['svn'] + args | |
290 if not self.svn_config.default: | |
291 args.extend(['--config-dir', self.svn_config.svn_config_dir]) | |
292 if not self.svn_config.default or self.non_interactive or non_interactive: | |
293 args.append('--non-interactive') | |
294 if credentials: | |
295 if self.commit_user: | |
296 args.extend(['--username', self.commit_user]) | |
297 if self.commit_pwd: | |
298 args.extend(['--password', self.commit_pwd]) | |
299 return args | |
300 | |
301 def _check_call_svn(self, args, **kwargs): | |
302 """Runs svn and throws an exception if the command failed.""" | |
303 kwargs.setdefault('cwd', self.project_path) | |
304 kwargs.setdefault('stdout', self.VOID) | |
305 kwargs.setdefault('timeout', GLOBAL_TIMEOUT) | |
306 return subprocess2.check_call_out( | |
307 self._add_svn_flags(args, False), **kwargs) | |
308 | |
309 def _check_output_svn(self, args, credentials=True, **kwargs): | |
310 """Runs svn and throws an exception if the command failed. | |
311 | |
312 Returns the output. | |
313 """ | |
314 kwargs.setdefault('cwd', self.project_path) | |
315 return subprocess2.check_output( | |
316 self._add_svn_flags(args, True, credentials), | |
317 stderr=subprocess2.STDOUT, | |
318 timeout=GLOBAL_TIMEOUT, | |
319 **kwargs) | |
320 | |
321 @staticmethod | |
322 def _parse_svn_info(output, key): | |
323 """Returns value for key from svn info output. | |
324 | |
325 Case insensitive. | |
326 """ | |
327 values = {} | |
328 key = key.lower() | |
329 for line in output.splitlines(False): | |
330 if not line: | |
331 continue | |
332 k, v = line.split(':', 1) | |
333 k = k.strip().lower() | |
334 v = v.strip() | |
335 assert not k in values | |
336 values[k] = v | |
337 return values.get(key, None) | |
338 | |
339 | |
340 class SvnCheckout(CheckoutBase, SvnMixIn): | |
341 """Manages a subversion checkout.""" | |
342 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url, | |
343 post_processors=None): | |
344 CheckoutBase.__init__(self, root_dir, project_name, post_processors) | |
345 SvnMixIn.__init__(self) | |
346 self.commit_user = commit_user | |
347 self.commit_pwd = commit_pwd | |
348 self.svn_url = svn_url | |
349 assert bool(self.commit_user) >= bool(self.commit_pwd) | |
350 | |
351 def prepare(self, revision): | |
352 # Will checkout if the directory is not present. | |
353 assert self.svn_url | |
354 if not os.path.isdir(self.project_path): | |
355 logging.info('Checking out %s in %s' % | |
356 (self.project_name, self.project_path)) | |
357 return self._revert(revision) | |
358 | |
359 def apply_patch(self, patches, post_processors=None, verbose=False): | |
360 post_processors = post_processors or self.post_processors or [] | |
361 for p in patches: | |
362 stdout = [] | |
363 try: | |
364 filepath = os.path.join(self.project_path, p.filename) | |
365 # It is important to use credentials=False otherwise credentials could | |
366 # leak in the error message. Credentials are not necessary here for the | |
367 # following commands anyway. | |
368 if p.is_delete: | |
369 stdout.append(self._check_output_svn( | |
370 ['delete', p.filename, '--force'], credentials=False)) | |
371 assert(not os.path.exists(filepath)) | |
372 stdout.append('Deleted.') | |
373 else: | |
374 # svn add while creating directories otherwise svn add on the | |
375 # contained files will silently fail. | |
376 # First, find the root directory that exists. | |
377 dirname = os.path.dirname(p.filename) | |
378 dirs_to_create = [] | |
379 while (dirname and | |
380 not os.path.isdir(os.path.join(self.project_path, dirname))): | |
381 dirs_to_create.append(dirname) | |
382 dirname = os.path.dirname(dirname) | |
383 for dir_to_create in reversed(dirs_to_create): | |
384 os.mkdir(os.path.join(self.project_path, dir_to_create)) | |
385 stdout.append( | |
386 self._check_output_svn( | |
387 ['add', dir_to_create, '--force'], credentials=False)) | |
388 stdout.append('Created missing directory %s.' % dir_to_create) | |
389 | |
390 if p.is_binary: | |
391 content = p.get() | |
392 with open(filepath, 'wb') as f: | |
393 f.write(content) | |
394 stdout.append('Added binary file %d bytes.' % len(content)) | |
395 else: | |
396 if p.source_filename: | |
397 if not p.is_new: | |
398 raise PatchApplicationFailed( | |
399 p, | |
400 'File has a source filename specified but is not new') | |
401 # Copy the file first. | |
402 if os.path.isfile(filepath): | |
403 raise PatchApplicationFailed( | |
404 p, 'File exist but was about to be overwriten') | |
405 stdout.append( | |
406 self._check_output_svn( | |
407 ['copy', p.source_filename, p.filename])) | |
408 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename)) | |
409 if p.diff_hunks: | |
410 cmd = [ | |
411 'patch', | |
412 '-p%s' % p.patchlevel, | |
413 '--forward', | |
414 '--force', | |
415 '--no-backup-if-mismatch', | |
416 ] | |
417 env = os.environ.copy() | |
418 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch') | |
419 try: | |
420 stdout.append( | |
421 subprocess2.check_output( | |
422 cmd, | |
423 stdin=p.get(False), | |
424 cwd=self.project_path, | |
425 timeout=GLOBAL_TIMEOUT, | |
426 env=env)) | |
427 finally: | |
428 shutil.rmtree(env['TMPDIR']) | |
429 | |
430 elif p.is_new and not os.path.exists(filepath): | |
431 # There is only a header. Just create the file if it doesn't | |
432 # exist. | |
433 open(filepath, 'w').close() | |
434 stdout.append('Created an empty file.') | |
435 if p.is_new and not p.source_filename: | |
436 # Do not run it if p.source_filename is defined, since svn copy was | |
437 # using above. | |
438 stdout.append( | |
439 self._check_output_svn( | |
440 ['add', p.filename, '--force'], credentials=False)) | |
441 for name, value in p.svn_properties: | |
442 if value is None: | |
443 stdout.append( | |
444 self._check_output_svn( | |
445 ['propdel', '--quiet', name, p.filename], | |
446 credentials=False)) | |
447 stdout.append('Property %s deleted.' % name) | |
448 else: | |
449 stdout.append( | |
450 self._check_output_svn( | |
451 ['propset', name, value, p.filename], credentials=False)) | |
452 stdout.append('Property %s=%s' % (name, value)) | |
453 for prop, values in self.svn_config.auto_props.iteritems(): | |
454 if fnmatch.fnmatch(p.filename, prop): | |
455 for value in values.split(';'): | |
456 if '=' not in value: | |
457 params = [value, '.'] | |
458 else: | |
459 params = value.split('=', 1) | |
460 if params[1] == '*': | |
461 # Works around crbug.com/150960 on Windows. | |
462 params[1] = '.' | |
463 stdout.append( | |
464 self._check_output_svn( | |
465 ['propset'] + params + [p.filename], credentials=False)) | |
466 stdout.append('Property (auto) %s' % '='.join(params)) | |
467 for post in post_processors: | |
468 post(self, p) | |
469 if verbose: | |
470 print p.filename | |
471 print align_stdout(stdout) | |
472 except OSError, e: | |
473 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e)) | |
474 except subprocess.CalledProcessError, e: | |
475 raise PatchApplicationFailed( | |
476 p, | |
477 'While running %s;\n%s%s' % ( | |
478 ' '.join(e.cmd), | |
479 align_stdout(stdout), | |
480 align_stdout([getattr(e, 'stdout', '')]))) | |
481 | |
482 def commit(self, commit_message, user): | |
483 logging.info('Committing patch for %s' % user) | |
484 assert self.commit_user | |
485 assert isinstance(commit_message, unicode) | |
486 handle, commit_filename = tempfile.mkstemp(text=True) | |
487 try: | |
488 # Shouldn't assume default encoding is UTF-8. But really, if you are using | |
489 # anything else, you are living in another world. | |
490 os.write(handle, commit_message.encode('utf-8')) | |
491 os.close(handle) | |
492 # When committing, svn won't update the Revision metadata of the checkout, | |
493 # so if svn commit returns "Committed revision 3.", svn info will still | |
494 # return "Revision: 2". Since running svn update right after svn commit | |
495 # creates a race condition with other committers, this code _must_ parse | |
496 # the output of svn commit and use a regexp to grab the revision number. | |
497 # Note that "Committed revision N." is localized but subprocess2 forces | |
498 # LANGUAGE=en. | |
499 args = ['commit', '--file', commit_filename] | |
500 # realauthor is parsed by a server-side hook. | |
501 if user and user != self.commit_user: | |
502 args.extend(['--with-revprop', 'realauthor=%s' % user]) | |
503 out = self._check_output_svn(args) | |
504 finally: | |
505 os.remove(commit_filename) | |
506 lines = filter(None, out.splitlines()) | |
507 match = re.match(r'^Committed revision (\d+).$', lines[-1]) | |
508 if not match: | |
509 raise PatchApplicationFailed( | |
510 None, | |
511 'Couldn\'t make sense out of svn commit message:\n' + out) | |
512 return int(match.group(1)) | |
513 | |
514 def _revert(self, revision): | |
515 """Reverts local modifications or checks out if the directory is not | |
516 present. Use depot_tools's functionality to do this. | |
517 """ | |
518 flags = ['--ignore-externals'] | |
519 if revision: | |
520 flags.extend(['--revision', str(revision)]) | |
521 if os.path.isdir(self.project_path): | |
522 # This may remove any part (or all) of the checkout. | |
523 scm.SVN.Revert(self.project_path, no_ignore=True) | |
524 | |
525 if os.path.isdir(self.project_path): | |
526 # Revive files that were deleted in scm.SVN.Revert(). | |
527 self._check_call_svn(['update', '--force'] + flags, | |
528 timeout=FETCH_TIMEOUT) | |
529 else: | |
530 logging.info( | |
531 'Directory %s is not present, checking it out.' % self.project_path) | |
532 self._check_call_svn( | |
533 ['checkout', self.svn_url, self.project_path] + flags, cwd=None, | |
534 timeout=FETCH_TIMEOUT) | |
535 return self._get_revision() | |
536 | |
537 def _get_revision(self): | |
538 out = self._check_output_svn(['info', '.']) | |
539 revision = int(self._parse_svn_info(out, 'revision')) | |
540 if revision != self._last_seen_revision: | |
541 logging.info('Updated to revision %d' % revision) | |
542 self._last_seen_revision = revision | |
543 return revision | |
544 | |
545 def revisions(self, rev1, rev2): | |
546 """Returns the number of actual commits, not just the difference between | |
547 numbers. | |
548 """ | |
549 rev2 = rev2 or 'HEAD' | |
550 # Revision range is inclusive and ordering doesn't matter, they'll appear in | |
551 # the order specified. | |
552 try: | |
553 out = self._check_output_svn( | |
554 ['log', '-q', self.svn_url, '-r', '%s:%s' % (rev1, rev2)]) | |
555 except subprocess.CalledProcessError: | |
556 return None | |
557 # Ignore the '----' lines. | |
558 return len([l for l in out.splitlines() if l.startswith('r')]) - 1 | |
559 | |
560 | |
561 class GitCheckout(CheckoutBase): | 164 class GitCheckout(CheckoutBase): |
562 """Manages a git checkout.""" | 165 """Manages a git checkout.""" |
563 def __init__(self, root_dir, project_name, remote_branch, git_url, | 166 def __init__(self, root_dir, project_name, remote_branch, git_url, |
564 commit_user, post_processors=None): | 167 commit_user, post_processors=None): |
565 super(GitCheckout, self).__init__(root_dir, project_name, post_processors) | 168 super(GitCheckout, self).__init__(root_dir, project_name, post_processors) |
566 self.git_url = git_url | 169 self.git_url = git_url |
567 self.commit_user = commit_user | 170 self.commit_user = commit_user |
568 self.remote_branch = remote_branch | 171 self.remote_branch = remote_branch |
569 # The working branch where patches will be applied. It will track the | 172 # The working branch where patches will be applied. It will track the |
570 # remote branch. | 173 # remote branch. |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
632 '--quiet']) | 235 '--quiet']) |
633 | 236 |
634 def _get_head_commit_hash(self): | 237 def _get_head_commit_hash(self): |
635 """Gets the current revision (in unicode) from the local branch.""" | 238 """Gets the current revision (in unicode) from the local branch.""" |
636 return unicode(self._check_output_git(['rev-parse', 'HEAD']).strip()) | 239 return unicode(self._check_output_git(['rev-parse', 'HEAD']).strip()) |
637 | 240 |
638 def apply_patch(self, patches, post_processors=None, verbose=False): | 241 def apply_patch(self, patches, post_processors=None, verbose=False): |
639 """Applies a patch on 'working_branch' and switches to it. | 242 """Applies a patch on 'working_branch' and switches to it. |
640 | 243 |
641 The changes remain staged on the current branch. | 244 The changes remain staged on the current branch. |
642 | |
643 Ignores svn properties and raise an exception on unexpected ones. | |
644 """ | 245 """ |
645 post_processors = post_processors or self.post_processors or [] | 246 post_processors = post_processors or self.post_processors or [] |
646 # It this throws, the checkout is corrupted. Maybe worth deleting it and | 247 # It this throws, the checkout is corrupted. Maybe worth deleting it and |
647 # trying again? | 248 # trying again? |
648 if self.remote_branch: | 249 if self.remote_branch: |
649 self._check_call_git( | 250 self._check_call_git( |
650 ['checkout', '-b', self.working_branch, '-t', self.remote_branch, | 251 ['checkout', '-b', self.working_branch, '-t', self.remote_branch, |
651 '--quiet']) | 252 '--quiet']) |
652 | 253 |
653 for index, p in enumerate(patches): | 254 for index, p in enumerate(patches): |
(...skipping 25 matching lines...) Expand all Loading... |
679 if verbose: | 280 if verbose: |
680 cmd.append('--verbose') | 281 cmd.append('--verbose') |
681 stdout.append(self._check_output_git(cmd)) | 282 stdout.append(self._check_output_git(cmd)) |
682 else: | 283 else: |
683 # No need to do anything special with p.is_new or if not | 284 # No need to do anything special with p.is_new or if not |
684 # p.diff_hunks. git apply manages all that already. | 285 # p.diff_hunks. git apply manages all that already. |
685 cmd = ['apply', '--index', '-3', '-p%s' % p.patchlevel] | 286 cmd = ['apply', '--index', '-3', '-p%s' % p.patchlevel] |
686 if verbose: | 287 if verbose: |
687 cmd.append('--verbose') | 288 cmd.append('--verbose') |
688 stdout.append(self._check_output_git(cmd, stdin=p.get(True))) | 289 stdout.append(self._check_output_git(cmd, stdin=p.get(True))) |
689 for key, value in p.svn_properties: | |
690 # Ignore some known auto-props flags through .subversion/config, | |
691 # bails out on the other ones. | |
692 # TODO(maruel): Read ~/.subversion/config and detect the rules that | |
693 # applies here to figure out if the property will be correctly | |
694 # handled. | |
695 stdout.append('Property %s=%s' % (key, value)) | |
696 if not key in ( | |
697 'svn:eol-style', 'svn:executable', 'svn:mime-type'): | |
698 raise patch.UnsupportedPatchFormat( | |
699 p.filename, | |
700 'Cannot apply svn property %s to file %s.' % ( | |
701 key, p.filename)) | |
702 for post in post_processors: | 290 for post in post_processors: |
703 post(self, p) | 291 post(self, p) |
704 if verbose: | 292 if verbose: |
705 print p.filename | 293 print p.filename |
706 print align_stdout(stdout) | 294 print align_stdout(stdout) |
707 except OSError, e: | 295 except OSError, e: |
708 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e)) | 296 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e)) |
709 except subprocess.CalledProcessError, e: | 297 except subprocess.CalledProcessError, e: |
710 raise PatchApplicationFailed( | 298 raise PatchApplicationFailed( |
711 p, | 299 p, |
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
835 def revisions(self, rev1, rev2): | 423 def revisions(self, rev1, rev2): |
836 return self.checkout.revisions(rev1, rev2) | 424 return self.checkout.revisions(rev1, rev2) |
837 | 425 |
838 @property | 426 @property |
839 def project_name(self): | 427 def project_name(self): |
840 return self.checkout.project_name | 428 return self.checkout.project_name |
841 | 429 |
842 @property | 430 @property |
843 def project_path(self): | 431 def project_path(self): |
844 return self.checkout.project_path | 432 return self.checkout.project_path |
OLD | NEW |