Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(413)

Side by Side Diff: checkout.py

Issue 2398603003: Remove SVN support from checkout.py (Closed)
Patch Set: Updated patchset dependency Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « apply_issue.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding=utf8 1 # coding=utf8
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 """Manages a project checkout. 5 """Manages a project checkout.
6 6
7 Includes support for svn, git-svn and git. 7 Includes support only for git.
8 """ 8 """
9 9
10 import fnmatch 10 import fnmatch
11 import logging 11 import logging
12 import os 12 import os
13 import re 13 import re
14 import shutil 14 import shutil
15 import subprocess 15 import subprocess
16 import sys 16 import sys
17 import tempfile 17 import tempfile
(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after
154 def revisions(self, rev1, rev2): 154 def revisions(self, rev1, rev2):
155 """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]). 155 """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]).
156 156
157 If rev2 is None, it means 'HEAD'. 157 If rev2 is None, it means 'HEAD'.
158 158
159 Returns None if there is no link between the two. 159 Returns None if there is no link between the two.
160 """ 160 """
161 raise NotImplementedError() 161 raise NotImplementedError()
162 162
163 163
164 class RawCheckout(CheckoutBase):
165 """Used to apply a patch locally without any intent to commit it.
166
167 To be used by the try server.
168 """
169 def prepare(self, revision):
170 """Stubbed out."""
171 pass
172
173 def apply_patch(self, patches, post_processors=None, verbose=False):
174 """Ignores svn properties."""
175 post_processors = post_processors or self.post_processors or []
176 for p in patches:
177 stdout = []
178 try:
179 filepath = os.path.join(self.project_path, p.filename)
180 if p.is_delete:
181 os.remove(filepath)
182 assert(not os.path.exists(filepath))
183 stdout.append('Deleted.')
184 else:
185 dirname = os.path.dirname(p.filename)
186 full_dir = os.path.join(self.project_path, dirname)
187 if dirname and not os.path.isdir(full_dir):
188 os.makedirs(full_dir)
189 stdout.append('Created missing directory %s.' % dirname)
190
191 if p.is_binary:
192 content = p.get()
193 with open(filepath, 'wb') as f:
194 f.write(content)
195 stdout.append('Added binary file %d bytes.' % len(content))
196 else:
197 if p.source_filename:
198 if not p.is_new:
199 raise PatchApplicationFailed(
200 p,
201 'File has a source filename specified but is not new')
202 # Copy the file first.
203 if os.path.isfile(filepath):
204 raise PatchApplicationFailed(
205 p, 'File exist but was about to be overwriten')
206 shutil.copy2(
207 os.path.join(self.project_path, p.source_filename), filepath)
208 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
209 if p.diff_hunks:
210 cmd = ['patch', '-u', '--binary', '-p%s' % p.patchlevel]
211 if verbose:
212 cmd.append('--verbose')
213 env = os.environ.copy()
214 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
215 try:
216 stdout.append(
217 subprocess2.check_output(
218 cmd,
219 stdin=p.get(False),
220 stderr=subprocess2.STDOUT,
221 cwd=self.project_path,
222 timeout=GLOBAL_TIMEOUT,
223 env=env))
224 finally:
225 shutil.rmtree(env['TMPDIR'])
226 elif p.is_new and not os.path.exists(filepath):
227 # There is only a header. Just create the file.
228 open(filepath, 'w').close()
229 stdout.append('Created an empty file.')
230 for post in post_processors:
231 post(self, p)
232 if verbose:
233 print p.filename
234 print align_stdout(stdout)
235 except OSError, e:
236 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
237 except subprocess.CalledProcessError, e:
238 raise PatchApplicationFailed(
239 p,
240 'While running %s;\n%s%s' % (
241 ' '.join(e.cmd),
242 align_stdout(stdout),
243 align_stdout([getattr(e, 'stdout', '')])))
244
245 def commit(self, commit_message, user):
246 """Stubbed out."""
247 raise NotImplementedError('RawCheckout can\'t commit')
248
249 def revisions(self, _rev1, _rev2):
250 return None
251
252
253 class SvnConfig(object):
254 """Parses a svn configuration file."""
255 def __init__(self, svn_config_dir=None):
256 super(SvnConfig, self).__init__()
257 self.svn_config_dir = svn_config_dir
258 self.default = not bool(self.svn_config_dir)
259 if not self.svn_config_dir:
260 if sys.platform == 'win32':
261 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
262 else:
263 self.svn_config_dir = os.path.expanduser(
264 os.path.join('~', '.subversion'))
265 svn_config_file = os.path.join(self.svn_config_dir, 'config')
266 parser = configparser.SafeConfigParser()
267 if os.path.isfile(svn_config_file):
268 parser.read(svn_config_file)
269 else:
270 parser.add_section('auto-props')
271 self.auto_props = dict(parser.items('auto-props'))
272
273
274 class SvnMixIn(object):
275 """MixIn class to add svn commands common to both svn and git-svn clients."""
276 # These members need to be set by the subclass.
277 commit_user = None
278 commit_pwd = None
279 svn_url = None
280 project_path = None
281 # Override at class level when necessary. If used, --non-interactive is
282 # implied.
283 svn_config = SvnConfig()
284 # Set to True when non-interactivity is necessary but a custom subversion
285 # configuration directory is not necessary.
286 non_interactive = False
287
288 def _add_svn_flags(self, args, non_interactive, credentials=True):
289 args = ['svn'] + args
290 if not self.svn_config.default:
291 args.extend(['--config-dir', self.svn_config.svn_config_dir])
292 if not self.svn_config.default or self.non_interactive or non_interactive:
293 args.append('--non-interactive')
294 if credentials:
295 if self.commit_user:
296 args.extend(['--username', self.commit_user])
297 if self.commit_pwd:
298 args.extend(['--password', self.commit_pwd])
299 return args
300
301 def _check_call_svn(self, args, **kwargs):
302 """Runs svn and throws an exception if the command failed."""
303 kwargs.setdefault('cwd', self.project_path)
304 kwargs.setdefault('stdout', self.VOID)
305 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
306 return subprocess2.check_call_out(
307 self._add_svn_flags(args, False), **kwargs)
308
309 def _check_output_svn(self, args, credentials=True, **kwargs):
310 """Runs svn and throws an exception if the command failed.
311
312 Returns the output.
313 """
314 kwargs.setdefault('cwd', self.project_path)
315 return subprocess2.check_output(
316 self._add_svn_flags(args, True, credentials),
317 stderr=subprocess2.STDOUT,
318 timeout=GLOBAL_TIMEOUT,
319 **kwargs)
320
321 @staticmethod
322 def _parse_svn_info(output, key):
323 """Returns value for key from svn info output.
324
325 Case insensitive.
326 """
327 values = {}
328 key = key.lower()
329 for line in output.splitlines(False):
330 if not line:
331 continue
332 k, v = line.split(':', 1)
333 k = k.strip().lower()
334 v = v.strip()
335 assert not k in values
336 values[k] = v
337 return values.get(key, None)
338
339
340 class SvnCheckout(CheckoutBase, SvnMixIn):
341 """Manages a subversion checkout."""
342 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
343 post_processors=None):
344 CheckoutBase.__init__(self, root_dir, project_name, post_processors)
345 SvnMixIn.__init__(self)
346 self.commit_user = commit_user
347 self.commit_pwd = commit_pwd
348 self.svn_url = svn_url
349 assert bool(self.commit_user) >= bool(self.commit_pwd)
350
351 def prepare(self, revision):
352 # Will checkout if the directory is not present.
353 assert self.svn_url
354 if not os.path.isdir(self.project_path):
355 logging.info('Checking out %s in %s' %
356 (self.project_name, self.project_path))
357 return self._revert(revision)
358
359 def apply_patch(self, patches, post_processors=None, verbose=False):
360 post_processors = post_processors or self.post_processors or []
361 for p in patches:
362 stdout = []
363 try:
364 filepath = os.path.join(self.project_path, p.filename)
365 # It is important to use credentials=False otherwise credentials could
366 # leak in the error message. Credentials are not necessary here for the
367 # following commands anyway.
368 if p.is_delete:
369 stdout.append(self._check_output_svn(
370 ['delete', p.filename, '--force'], credentials=False))
371 assert(not os.path.exists(filepath))
372 stdout.append('Deleted.')
373 else:
374 # svn add while creating directories otherwise svn add on the
375 # contained files will silently fail.
376 # First, find the root directory that exists.
377 dirname = os.path.dirname(p.filename)
378 dirs_to_create = []
379 while (dirname and
380 not os.path.isdir(os.path.join(self.project_path, dirname))):
381 dirs_to_create.append(dirname)
382 dirname = os.path.dirname(dirname)
383 for dir_to_create in reversed(dirs_to_create):
384 os.mkdir(os.path.join(self.project_path, dir_to_create))
385 stdout.append(
386 self._check_output_svn(
387 ['add', dir_to_create, '--force'], credentials=False))
388 stdout.append('Created missing directory %s.' % dir_to_create)
389
390 if p.is_binary:
391 content = p.get()
392 with open(filepath, 'wb') as f:
393 f.write(content)
394 stdout.append('Added binary file %d bytes.' % len(content))
395 else:
396 if p.source_filename:
397 if not p.is_new:
398 raise PatchApplicationFailed(
399 p,
400 'File has a source filename specified but is not new')
401 # Copy the file first.
402 if os.path.isfile(filepath):
403 raise PatchApplicationFailed(
404 p, 'File exist but was about to be overwriten')
405 stdout.append(
406 self._check_output_svn(
407 ['copy', p.source_filename, p.filename]))
408 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
409 if p.diff_hunks:
410 cmd = [
411 'patch',
412 '-p%s' % p.patchlevel,
413 '--forward',
414 '--force',
415 '--no-backup-if-mismatch',
416 ]
417 env = os.environ.copy()
418 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
419 try:
420 stdout.append(
421 subprocess2.check_output(
422 cmd,
423 stdin=p.get(False),
424 cwd=self.project_path,
425 timeout=GLOBAL_TIMEOUT,
426 env=env))
427 finally:
428 shutil.rmtree(env['TMPDIR'])
429
430 elif p.is_new and not os.path.exists(filepath):
431 # There is only a header. Just create the file if it doesn't
432 # exist.
433 open(filepath, 'w').close()
434 stdout.append('Created an empty file.')
435 if p.is_new and not p.source_filename:
436 # Do not run it if p.source_filename is defined, since svn copy was
437 # using above.
438 stdout.append(
439 self._check_output_svn(
440 ['add', p.filename, '--force'], credentials=False))
441 for name, value in p.svn_properties:
442 if value is None:
443 stdout.append(
444 self._check_output_svn(
445 ['propdel', '--quiet', name, p.filename],
446 credentials=False))
447 stdout.append('Property %s deleted.' % name)
448 else:
449 stdout.append(
450 self._check_output_svn(
451 ['propset', name, value, p.filename], credentials=False))
452 stdout.append('Property %s=%s' % (name, value))
453 for prop, values in self.svn_config.auto_props.iteritems():
454 if fnmatch.fnmatch(p.filename, prop):
455 for value in values.split(';'):
456 if '=' not in value:
457 params = [value, '.']
458 else:
459 params = value.split('=', 1)
460 if params[1] == '*':
461 # Works around crbug.com/150960 on Windows.
462 params[1] = '.'
463 stdout.append(
464 self._check_output_svn(
465 ['propset'] + params + [p.filename], credentials=False))
466 stdout.append('Property (auto) %s' % '='.join(params))
467 for post in post_processors:
468 post(self, p)
469 if verbose:
470 print p.filename
471 print align_stdout(stdout)
472 except OSError, e:
473 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
474 except subprocess.CalledProcessError, e:
475 raise PatchApplicationFailed(
476 p,
477 'While running %s;\n%s%s' % (
478 ' '.join(e.cmd),
479 align_stdout(stdout),
480 align_stdout([getattr(e, 'stdout', '')])))
481
482 def commit(self, commit_message, user):
483 logging.info('Committing patch for %s' % user)
484 assert self.commit_user
485 assert isinstance(commit_message, unicode)
486 handle, commit_filename = tempfile.mkstemp(text=True)
487 try:
488 # Shouldn't assume default encoding is UTF-8. But really, if you are using
489 # anything else, you are living in another world.
490 os.write(handle, commit_message.encode('utf-8'))
491 os.close(handle)
492 # When committing, svn won't update the Revision metadata of the checkout,
493 # so if svn commit returns "Committed revision 3.", svn info will still
494 # return "Revision: 2". Since running svn update right after svn commit
495 # creates a race condition with other committers, this code _must_ parse
496 # the output of svn commit and use a regexp to grab the revision number.
497 # Note that "Committed revision N." is localized but subprocess2 forces
498 # LANGUAGE=en.
499 args = ['commit', '--file', commit_filename]
500 # realauthor is parsed by a server-side hook.
501 if user and user != self.commit_user:
502 args.extend(['--with-revprop', 'realauthor=%s' % user])
503 out = self._check_output_svn(args)
504 finally:
505 os.remove(commit_filename)
506 lines = filter(None, out.splitlines())
507 match = re.match(r'^Committed revision (\d+).$', lines[-1])
508 if not match:
509 raise PatchApplicationFailed(
510 None,
511 'Couldn\'t make sense out of svn commit message:\n' + out)
512 return int(match.group(1))
513
514 def _revert(self, revision):
515 """Reverts local modifications or checks out if the directory is not
516 present. Use depot_tools's functionality to do this.
517 """
518 flags = ['--ignore-externals']
519 if revision:
520 flags.extend(['--revision', str(revision)])
521 if os.path.isdir(self.project_path):
522 # This may remove any part (or all) of the checkout.
523 scm.SVN.Revert(self.project_path, no_ignore=True)
524
525 if os.path.isdir(self.project_path):
526 # Revive files that were deleted in scm.SVN.Revert().
527 self._check_call_svn(['update', '--force'] + flags,
528 timeout=FETCH_TIMEOUT)
529 else:
530 logging.info(
531 'Directory %s is not present, checking it out.' % self.project_path)
532 self._check_call_svn(
533 ['checkout', self.svn_url, self.project_path] + flags, cwd=None,
534 timeout=FETCH_TIMEOUT)
535 return self._get_revision()
536
537 def _get_revision(self):
538 out = self._check_output_svn(['info', '.'])
539 revision = int(self._parse_svn_info(out, 'revision'))
540 if revision != self._last_seen_revision:
541 logging.info('Updated to revision %d' % revision)
542 self._last_seen_revision = revision
543 return revision
544
545 def revisions(self, rev1, rev2):
546 """Returns the number of actual commits, not just the difference between
547 numbers.
548 """
549 rev2 = rev2 or 'HEAD'
550 # Revision range is inclusive and ordering doesn't matter, they'll appear in
551 # the order specified.
552 try:
553 out = self._check_output_svn(
554 ['log', '-q', self.svn_url, '-r', '%s:%s' % (rev1, rev2)])
555 except subprocess.CalledProcessError:
556 return None
557 # Ignore the '----' lines.
558 return len([l for l in out.splitlines() if l.startswith('r')]) - 1
559
560
561 class GitCheckout(CheckoutBase): 164 class GitCheckout(CheckoutBase):
562 """Manages a git checkout.""" 165 """Manages a git checkout."""
563 def __init__(self, root_dir, project_name, remote_branch, git_url, 166 def __init__(self, root_dir, project_name, remote_branch, git_url,
564 commit_user, post_processors=None): 167 commit_user, post_processors=None):
565 super(GitCheckout, self).__init__(root_dir, project_name, post_processors) 168 super(GitCheckout, self).__init__(root_dir, project_name, post_processors)
566 self.git_url = git_url 169 self.git_url = git_url
567 self.commit_user = commit_user 170 self.commit_user = commit_user
568 self.remote_branch = remote_branch 171 self.remote_branch = remote_branch
569 # The working branch where patches will be applied. It will track the 172 # The working branch where patches will be applied. It will track the
570 # remote branch. 173 # remote branch.
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
632 '--quiet']) 235 '--quiet'])
633 236
634 def _get_head_commit_hash(self): 237 def _get_head_commit_hash(self):
635 """Gets the current revision (in unicode) from the local branch.""" 238 """Gets the current revision (in unicode) from the local branch."""
636 return unicode(self._check_output_git(['rev-parse', 'HEAD']).strip()) 239 return unicode(self._check_output_git(['rev-parse', 'HEAD']).strip())
637 240
638 def apply_patch(self, patches, post_processors=None, verbose=False): 241 def apply_patch(self, patches, post_processors=None, verbose=False):
639 """Applies a patch on 'working_branch' and switches to it. 242 """Applies a patch on 'working_branch' and switches to it.
640 243
641 The changes remain staged on the current branch. 244 The changes remain staged on the current branch.
642
643 Ignores svn properties and raise an exception on unexpected ones.
644 """ 245 """
645 post_processors = post_processors or self.post_processors or [] 246 post_processors = post_processors or self.post_processors or []
646 # It this throws, the checkout is corrupted. Maybe worth deleting it and 247 # It this throws, the checkout is corrupted. Maybe worth deleting it and
647 # trying again? 248 # trying again?
648 if self.remote_branch: 249 if self.remote_branch:
649 self._check_call_git( 250 self._check_call_git(
650 ['checkout', '-b', self.working_branch, '-t', self.remote_branch, 251 ['checkout', '-b', self.working_branch, '-t', self.remote_branch,
651 '--quiet']) 252 '--quiet'])
652 253
653 for index, p in enumerate(patches): 254 for index, p in enumerate(patches):
(...skipping 25 matching lines...) Expand all
679 if verbose: 280 if verbose:
680 cmd.append('--verbose') 281 cmd.append('--verbose')
681 stdout.append(self._check_output_git(cmd)) 282 stdout.append(self._check_output_git(cmd))
682 else: 283 else:
683 # No need to do anything special with p.is_new or if not 284 # No need to do anything special with p.is_new or if not
684 # p.diff_hunks. git apply manages all that already. 285 # p.diff_hunks. git apply manages all that already.
685 cmd = ['apply', '--index', '-3', '-p%s' % p.patchlevel] 286 cmd = ['apply', '--index', '-3', '-p%s' % p.patchlevel]
686 if verbose: 287 if verbose:
687 cmd.append('--verbose') 288 cmd.append('--verbose')
688 stdout.append(self._check_output_git(cmd, stdin=p.get(True))) 289 stdout.append(self._check_output_git(cmd, stdin=p.get(True)))
689 for key, value in p.svn_properties:
690 # Ignore some known auto-props flags through .subversion/config,
691 # bails out on the other ones.
692 # TODO(maruel): Read ~/.subversion/config and detect the rules that
693 # applies here to figure out if the property will be correctly
694 # handled.
695 stdout.append('Property %s=%s' % (key, value))
696 if not key in (
697 'svn:eol-style', 'svn:executable', 'svn:mime-type'):
698 raise patch.UnsupportedPatchFormat(
699 p.filename,
700 'Cannot apply svn property %s to file %s.' % (
701 key, p.filename))
702 for post in post_processors: 290 for post in post_processors:
703 post(self, p) 291 post(self, p)
704 if verbose: 292 if verbose:
705 print p.filename 293 print p.filename
706 print align_stdout(stdout) 294 print align_stdout(stdout)
707 except OSError, e: 295 except OSError, e:
708 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e)) 296 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
709 except subprocess.CalledProcessError, e: 297 except subprocess.CalledProcessError, e:
710 raise PatchApplicationFailed( 298 raise PatchApplicationFailed(
711 p, 299 p,
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after
835 def revisions(self, rev1, rev2): 423 def revisions(self, rev1, rev2):
836 return self.checkout.revisions(rev1, rev2) 424 return self.checkout.revisions(rev1, rev2)
837 425
838 @property 426 @property
839 def project_name(self): 427 def project_name(self):
840 return self.checkout.project_name 428 return self.checkout.project_name
841 429
842 @property 430 @property
843 def project_path(self): 431 def project_path(self):
844 return self.checkout.project_path 432 return self.checkout.project_path
OLDNEW
« no previous file with comments | « apply_issue.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698