[Findit] Add local_git_repository

appengine/findit/util_scripts/git_checkout/local_git_repository.py

+# Copyright 2014 The Chromium Authors. All rights reserved.

[chanli, 2016/11/08 01:56:20]

Nit: 2014 -> 2016. Same for other added files

[Sharu Jiang, 2016/11/11 00:29:06]

Done.

+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+from urlparse import urlparse
+import subprocess
+import threading
+
+from git_checkout import local_git_parsers
+from lib.gitiles.git_repository import GitRepository
+import script_util
+
+_CHANGELOG_FORMAT_STRING = ('commit %H%n'
+                            'author %an%n'
+                            'author-mail %ae%n'
+                            'author-time %ad%n%n'
+                            'committer %cn%n'
+                            'committer-mail %ce%n'
+                            'committer-time %cd%n%n'
+                            '--Message start--%n%B%n--Message end--%n')
+_CHANGELOGS_FORMAT_STRING = ('**Changelog start**%%n%s' %
+                             _CHANGELOG_FORMAT_STRING)
+CHECKOUT_ROOT_DIR = os.path.join(os.path.expanduser('~'), '.local_checkouts')
+
+
+def ConvertRemoteCommitToLocal(revision):
+  """Converts remote commit from gitile to local git checkout revision."""
+  return 'HEAD' if revision == 'master' else revision
+
+
+class LocalGitRepository(GitRepository):
+  """Represents local checkout of git repository on chromium host.
+
+  Note, to checkout internal repos automatically which you have access to,
+  follow the instructions in ('https://g3doc.corp.google.com/company/teams/

[stgao, 2016/11/08 21:56:36]

Could we use a go link instead?

[wrengr, 2016/11/10 21:27:04]

I like the idea, but iirc policy is that go links shouldn't be shared to
anywhere outside of google. Thus, since the infra repo is publicly available, it
shouldn't have go links. Perhaps we could use http://g.co/ instead?

[Sharu Jiang, 2016/11/11 00:29:06]

we needs to submit a proposal and get approved to create new shorturl in
http://g.co/ ... 
https://support.google.com/oh/answer/6210028?hl=en

And for the public shortening tool like https://goo.gl/
The internal url cannot be shortened.

[stgao, 2016/11/11 01:07:59]

go link seems fine as it is used a lot in the infra repo like
https://cs.chromium.org/chromium/infra/navbar.md

[Sharu Jiang, 2016/11/11 23:43:49]

Done.

+  chrome/chrome_build_instructions.md?
+  cl=head#authentication-to-git-servers-chrome-internalgooglesourcecom') first.
+  """
+  lock = threading.Lock()
+  # Keep track all the updated repos, so every repo only get updated once.
+  _updated_repos = set()
+  def __init__(self, repo_url=None):

[stgao, 2016/11/08 21:56:36]

empty line above.

[Sharu Jiang, 2016/11/11 00:29:05]

Done.

+    self._host = None
+    self._repo_path = None
+    self._repo_url = None
+    self._SetFieldsFromRepoUrl(repo_url)
+    self.changelog_parser = local_git_parsers.GitChangeLogParser()
+    self.changelogs_parser = local_git_parsers.GitChangeLogsParser()
+    self.blame_parser = local_git_parsers.GitBlameParser()
+    self.diff_parser = local_git_parsers.GitDiffParser()
+    self.source_parser = local_git_parsers.GitSourceParser()
+
+  @property
+  def repo_path(self):
+    return self._repo_path
+
+  @property
+  def real_repo_path(self):
+    """Absolute path of the local repository."""
+    return os.path.join(CHECKOUT_ROOT_DIR, self._host, self.repo_path)
+
+  @property
+  def repo_url(self):
+    """Url of remote repository which the local repo checks out from."""
+    return self._repo_url
+
+  @repo_url.setter
+  def repo_url(self, repo_url):
+    self._SetFieldsFromRepoUrl(repo_url)
+
+  def _SetFieldsFromRepoUrl(self, repo_url):

[wrengr, 2016/11/08 19:32:37]

Why is this factored out as a separate method? why not just inline it into the
repo_url setter?

[Sharu Jiang, 2016/11/11 00:29:05]

Because this needs to be called in __init__, and having self.repo_url = repo_url
in __init__ is confusing as you said in previous comments.

[wrengr, 2016/11/11 18:38:48]

Oh! That's what was up with things before! If that's the only reason for
factoring things out, then feel-free to use repo_url.setter directly within
__init__ ---just be sure to add a comment clarifying that it's actually a call
to the setter for the property. (Or, perhaps make things more explicit by
directly calling the assignment method on the property object backing
``self.repo_url``.) The biggest reason why it was so confusing before was
because of the lack of documentation indicating what was really going on

[Sharu Jiang, 2016/11/11 23:43:49]

According to
http://stackoverflow.com/questions/3941919/how-to-set-a-python-property-in-init
The previous implementation looks fine, I changed back and add comments to avoid
confusion :)

+    if self._repo_url == repo_url:
+      return
+
+    self._repo_url = repo_url
+    if not self._repo_url:
+      return
+
+    parsed_url = urlparse(repo_url)
+    self._host = parsed_url.netloc
+    # Remove the / in the front of path.
+    self._repo_path = parsed_url.path[1:]
+
+    self._CloneOrUpdateRepoIfNeeded()
+
+  def _CloneOrUpdateRepoIfNeeded(self):
+    """Clones repo, or update it if it didn't got updated before."""
+    if self.repo_url in LocalGitRepository._updated_repos:
+      return
+
+    with LocalGitRepository.lock:
+      # Clone the repo if needed.
+      if not os.path.exists(self.real_repo_path):
+        try:
+          subprocess.check_call(['git', 'clone',
+                                 self.repo_url, self.real_repo_path])
+        except subprocess.CalledProcessError as e:  # pragma: no cover.
+          logging.error('Exception while cloning %s: %s', self.repo_url, e)
+          return
+      # Update repo if it's already cloned.
+      else:
+        try:
+          # Disable verbose of cd and git pull.
+          with open(os.devnull, 'w') as null_handle:
+            subprocess.check_call(
+                'cd %s && git pull' % self.real_repo_path,
+                stdout=null_handle, stderr=null_handle, shell=True)
+        except subprocess.CalledProcessError as e:  # pragma: no cover.
+          logging.error('Exception while updating %s: %s', self.repo_path, e)
+          return
+
+      LocalGitRepository._updated_repos.add(self.repo_url)
+
+
+  def _GetFinalCommand(self, command, format_date=False):
+    # Change local time to utc time.
+    if format_date:
+      command = 'TZ=UTC %s --date=format-local:"%s"' % (

[stgao, 2016/11/08 21:56:36]

So it works again?

[Sharu Jiang, 2016/11/11 00:29:06]

The only case it doesn't work is 'git blame' + '--porcelain/incremental', so it
works for the rest.

+                command, local_git_parsers.DATETIME_FORMAT)
+    return 'cd %s && %s' % (self.real_repo_path, command)
+
+  def GetChangeLog(self, revision):
+    """Returns the change log of the given revision."""
+    command = ('git log --pretty=format:"%s" --max-count=1 --raw '
+               '--no-abbrev %s' % (_CHANGELOG_FORMAT_STRING,
+                                   ConvertRemoteCommitToLocal(revision)))
+    output = script_util.GetCommandOutput(self._GetFinalCommand(command, True))
+    return self.changelog_parser(output, self.repo_url)
+
+  def GetChangeLogs(self, start_revision, end_revision):  # pylint: disable=W
+    """Returns change log list in (start_revision, end_revision]."""
+    command = ('git log --pretty=format:"%s" --raw --no-abbrev %s' % (
+                   _CHANGELOGS_FORMAT_STRING,
+                   '%s..%s' % (ConvertRemoteCommitToLocal(start_revision),
+                               ConvertRemoteCommitToLocal(end_revision))))
+    output = script_util.GetCommandOutput(self._GetFinalCommand(command, True))
+    return self.changelogs_parser(output, self.repo_url)
+
+  def GetChangeDiff(self, revision, path=None):  # pylint: disable=W
+    """Returns the diff of the given revision."""
+    command = ('git log --format="" --max-count=1 %s' %
+               ConvertRemoteCommitToLocal(revision))
+    if path:
+      command += ' -p %s' % path
+    output = script_util.GetCommandOutput(self._GetFinalCommand(command))
+    return self.diff_parser(output)
+
+  def GetBlame(self, path, revision):
+    """Returns blame of the file at ``path`` of the given revision."""
+    command = 'git blame --incremental %s %s' % (
+        path, ConvertRemoteCommitToLocal(revision))
+    output = script_util.GetCommandOutput(self._GetFinalCommand(command))
+    return self.blame_parser(output, path, revision)
+
+  def GetSource(self, path, revision):
+    """Returns source code of the file at ``path`` of the given revision."""
+    # Check whether the requested file exist or not.
+    if not os.path.isfile(os.path.join(self.real_repo_path, path)):

[stgao, 2016/11/08 21:56:36]

What if the file got deleted after some revision? Is this expected?

[Sharu Jiang, 2016/11/11 00:29:06]

good catch, this should be removed since if the file doesn't exist, the command
output should be None.

[stgao, 2016/11/11 01:07:59]

This seems unexpected compared to the Gitiles.
So it is impossible to get the content of a deleted file which existed in a
previous revision?

[Sharu Jiang, 2016/11/11 23:52:38]

It is possible to get the content of a deleted file which existed in a previous
revision.

I meant if the file indeed does not exist, the command will return None.

+      return None
+    command = 'git show %s:%s' % (ConvertRemoteCommitToLocal(revision), path)
+    output = script_util.GetCommandOutput(self._GetFinalCommand(command))
+    return self.source_parser(output)