[Findit] Add local_git_repository

appengine/findit/lib/gitiles/local_git_repository.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+from urlparse import urlparse
+import subprocess
+import threading
+
+from lib.gitiles import local_git_parsers
+from lib.gitiles import repo_util
+from lib.gitiles.git_repository import GitRepository
+
+_CHANGELOG_FORMAT_STRING = ('commit %H%n'
+                            'author %an%n'
+                            'author-mail %ae%n'
+                            'author-time %ad%n%n'
+                            'committer %cn%n'
+                            'committer-mail %ce%n'
+                            'committer-time %cd%n%n'
+                            '--Message start--%n%B%n--Message end--%n')
+_CHANGELOGS_FORMAT_STRING = ('**Changelog start**%%n%s' %
+                             _CHANGELOG_FORMAT_STRING)
+CHECKOUT_ROOT_DIR = os.path.join(os.path.expanduser('~'), '.local_checkouts')
+
+
+class LocalGitRepository(GitRepository):
+  """Represents local checkout of git repository on chromium host.
+
+  Note, to checkout internal repos automatically which you have access to,
+  follow the instructions in ('https://g3doc.corp.google.com/company/teams/
+  chrome/chrome_build_instructions.md?
+  cl=head#authentication-to-git-servers-chrome-internalgooglesourcecom') first.
+  """
+  lock = threading.Lock()
+  # Keep track all the updated repos, so every repo only get updated once.
+  _updated_repos = set()
+  def __init__(self, repo_url=None):
+    self._host = None
+    self._repo_path = None
+    self._repo_url = None
+    self.repo_url = repo_url

[wrengr, 2016/11/01 20:26:52]

I still think you should rename the repo_url and _repo_url fields to make clear
what the difference is. Moreover, the repo_url property and the repo_url field
clash with one another. Since the _repo_url field is associated with the
repo_url property (as it should be), the repo_url field is the one that needs to
be renamed/removed.

[Sharu Jiang, 2016/11/05 01:18:15]

Done.

+
+  @property
+  def repo_path(self):
+    return self._repo_path
+
+  @property
+  def real_repo_path(self):
+    """Absolute path of the local repository."""
+    return os.path.join(CHECKOUT_ROOT_DIR, self._host, self.repo_path)
+
+  @property
+  def repo_url(self):
+    """Url of remote repository which the local repo checks out from."""
+    return self._repo_url
+
+  @repo_url.setter
+  def repo_url(self, repo_url):
+    if self._repo_url == repo_url:
+      return
+
+    self._repo_url = repo_url
+    if not self._repo_url:
+      return
+
+    parsed_url = urlparse(repo_url)
+    self._host = parsed_url.netloc
+    # Remove the / in the front of path.
+    self._repo_path = parsed_url.path[1:]
+
+    self._CloneOrUpdateRepoIfNeeded()
+
+  def _CloneOrUpdateRepoIfNeeded(self):
+    """Clones repo, or update it if it didn't got updated before."""
+    if self.repo_url in LocalGitRepository._updated_repos:
+      return
+
+    with LocalGitRepository.lock:
+      # Clone the repo if needed.
+      if not os.path.exists(self.real_repo_path):

[wrengr, 2016/11/01 20:26:53]

In general it's bad to check whether a file/directory exists and then mess with
that file based on the result, since this involves race conditions. The
race-free approach is to assume it already exists, and catch the error to do
whatever else when it doesn't.

[Sharu Jiang, 2016/11/05 01:18:15]

what if one process is git cloning the repo, and another process find out the
path exists and do a git pull? Would there be race condition?

[wrengr, 2016/11/08 19:32:37]

Probably. Depends on whether git is smart enough to lock things while it's
working. I think it is? but not sure. Needs some research. If it turns out that
git isn't smart enough to handle it, we can always generate our own lockfile
just before doing the clone and then remove it once the clone is finished.

Feel free to deal with this in a separate cl; just be sure to make a ticket and
reference it here

+        try:
+          subprocess.check_call(['git', 'clone',
+                                 self.repo_url, self.real_repo_path])
+        except subprocess.CalledProcessError as e:  # pragma: no cover.
+          logging.error('Exception while cloning %s: %s', self.repo_url, e)
+          return
+      # Update repo if it's already cloned.
+      else:
+        try:
+          # Disable verbose of cd and git pull.
+          with open(os.devnull, 'w') as null_handle:
+            subprocess.check_call(
+                'cd %s && git pull' % self.real_repo_path,
+                stdout=null_handle, stderr=null_handle, shell=True)
+        except subprocess.CalledProcessError as e:  # pragma: no cover.
+          logging.error('Exception while updating %s: %s', self.repo_path, e)
+          return
+
+      LocalGitRepository._updated_repos.add(self.repo_url)
+
+  def _GetFinalCommand(self, command, format_date=False):
+    # Change local time to utc time.
+    if format_date:
+      command = 'TZ=UTC %s --date=format-local:"%s"' % (
+                command, local_git_parsers.DATETIME_FORMAT)
+    return 'cd %s && %s' % (self.real_repo_path, command)
+
+  def GetChangeLog(self, revision):
+    """Returns the change log of the given revision."""
+    revision = 'HEAD' if revision == 'master' else revision
+    command = ('git log --pretty=format:"%s" --max-count=1 --raw '
+               '--no-abbrev %s' % (_CHANGELOG_FORMAT_STRING, revision))
+    output = repo_util.GetCommandOutput(self._GetFinalCommand(command, True))
+    if not output:
+      return None
+
+    return local_git_parsers.GitChangeLogParser()(output, self.repo_url)
+
+  def GetChangeLogs(self, start_revision, end_revision):  # pylint: disable=W
+    """Returns change log list in (start_revision, end_revision]."""
+    start_revision = 'HEAD' if start_revision == 'master' else start_revision

[wrengr, 2016/11/01 20:26:53]

Since this conditional rewrite is used in a bunch of places, it'd be cleaner to
factor it out as a standalone function.

[Sharu Jiang, 2016/11/05 01:18:15]

Done.

+    end_revision = 'HEAD' if end_revision == 'master' else end_revision
+    command = ('git log --pretty=format:"%s" --raw '
+               '--no-abbrev %s' % (_CHANGELOGS_FORMAT_STRING,
+                                   '%s..%s' % (start_revision, end_revision)))
+    output = repo_util.GetCommandOutput(self._GetFinalCommand(command, True))
+    if not output:
+      return None
+
+    return local_git_parsers.GitChangeLogsParser()(output, self.repo_url)

[wrengr, 2016/11/01 20:26:52]

Presumably you want to save the result of
local_git_parsers.GitChangeLogsParser() so that you can reuse the same parser,
rather than constructing a new one every time GetChangeLogs is called. Ditto for
the other parsers.

Also, it'd clean up the code here if the parsers return None whenever ``output``
is falsy, rather than doing that check here. Especially given as the parsers
will already need to check for falsy arguments before trying to parse them.

[Sharu Jiang, 2016/11/05 01:18:15]

Acknowledged.

+
+  def GetChangeDiff(self, revision, path=None):  # pylint: disable=W
+    """Returns the diff of the given revision."""
+    revision = 'HEAD' if revision == 'master' else revision
+    command = 'git log --format="" --max-count=1 %s' % revision
+    if path:
+      command += ' -p %s' % path
+    output = repo_util.GetCommandOutput(self._GetFinalCommand(command))
+    if not output:
+      return None
+
+    return local_git_parsers.GitDiffParser()(output)
+
+  def GetBlame(self, path, revision):
+    """Returns blame of the file at ``path`` of the given revision."""
+    revision = 'HEAD' if revision == 'master' else revision
+    command = 'git blame --incremental %s %s' % (path, revision)
+    output = repo_util.GetCommandOutput(self._GetFinalCommand(command))
+    if not output:
+      return None
+
+    return local_git_parsers.GitBlameParser()(output, path, revision)
+
+  def GetSource(self, path, revision):
+    """Returns source code of the file at ``path`` of the given revision."""
+    # Check whether the requested file exist or not.
+    if not os.path.isfile(os.path.join(self.real_repo_path, path)):
+      return None
+    revision = 'HEAD' if revision == 'master' else revision
+    command = 'git show %s:%s' % (revision, path)
+    output = repo_util.GetCommandOutput(self._GetFinalCommand(command))
+    if not output:
+      return None
+
+    return local_git_parsers.GitSourceParser()(output)