[Findit] Pull changelogs in batch.

appengine/findit/common/git_repository.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import base64
 from datetime import datetime
 from datetime import timedelta
 import json
 import re
 
@@ skipping 123 matching lines @@
     lines = message.strip().splitlines()
     if not lines[0].lower().startswith('revert'):
       return None
 
     for line in reversed(lines):  # pragma: no cover
       # TODO: Handle cases where no reverted_revision in reverting message.
       reverted_revision_match = REVERTED_REVISION_PATTERN.match(line)
       if reverted_revision_match:
         return reverted_revision_match.group(1)
 
-  def GetChangeLog(self, revision):
-    url, data = self._DownloadChangeLogData(revision)
-    if not data:
-      return None
-
+  def _ParseChangeLogFromLogData(self, data):
     commit_position, code_review_url = (
         self.ExtractCommitPositionAndCodeReviewUrl(data['message']))
 
     touched_files = []
     for file_diff in data['tree_diff']:
       change_type = file_diff['type'].lower()
       if not diff.IsKnownChangeType(change_type):
         raise Exception('Unknown change type "%s"' % change_type)
       touched_files.append(
           FileChangeInfo(
               change_type, file_diff['old_path'], file_diff['new_path']))
 
     author_time = self._GetDateTimeFromString(data['author']['time'])
     committer_time = self._GetDateTimeFromString(data['committer']['time'])
     reverted_revision = self.GetRevertedRevision(data['message'])
+    url = '%s/+/%s' % (self.repo_url, data['commit'])
 
     return ChangeLog(
         data['author']['name'], self._NormalizeEmail(data['author']['email']),
         author_time,
         data['committer']['name'],
         self._NormalizeEmail(data['committer']['email']),
         committer_time, data['commit'], commit_position,
         data['message'], touched_files, url, code_review_url,
         reverted_revision)
 
+  def GetChangeLog(self, revision):
+    _, data = self._DownloadChangeLogData(revision)
+    if not data:
+      return None
+
+    return self._ParseChangeLogFromLogData(data)
+
   def GetCommitsBetweenRevisions(self, start_revision, end_revision, n=1000):
     """Gets a list of commit hashes between start_revision and end_revision.
 
     Args:
       start_revision: The oldest revision in the range.
       end_revision: The latest revision in the range.
       n: The maximum number of revisions to request at a time.
 
     Returns:
       A list of commit hashes made since start_revision through and including
@@ skipping 45 matching lines @@
                  self._NormalizeEmail(region['author']['email']), author_time))
 
     return blame
 
   def GetSource(self, path, revision):
     """Returns source code of the file at ``path`` of the given revision."""
     url = '%s/+/%s/%s' % (self.repo_url, revision, path)
     return self._SendRequestForTextResponse(url)
 
   def GetChangeLogs(self, start_revision, end_revision, n=1000):
-    """Gets a list of ChangeLogs in revision range.
+    """Gets a list of ChangeLogs in revision range by batch.
 
     Args:
       start_revision (str): The oldest revision in the range.
       end_revision (str): The latest revision in the range.
-      n (int): The maximum number of revisions to request at a time.
+      n (int): The maximum number of revisions to request at a time (default
+        to 1000).
 
     Returns:
       A list of changelogs in (start_revision, end_revision].
     """
-    revisions = self.GetCommitsBetweenRevisions(start_revision, end_revision, n)
+    next_end_revision = end_revision
     changelogs = []
 
-    for revision in revisions:
-      changelog = self.GetChangeLog(revision)
-      if not changelog:
-        raise Exception('Failed to pull changelog for revision %s' % revision)
+    while next_end_revision:
+      url = '%s/+log/%s..%s' % (self.repo_url,
+                                start_revision, next_end_revision)
+      data = self._SendRequestForJsonResponse(url, {'n': str(n),
+                                                    'name-status': '1'})

[stgao, 2016/05/06 06:59:18]

Will this include the file change?

[Sharu Jiang, 2016/05/06 17:46:12]

Yes, it includes the 'tree-diff's 

+      for log in data['log']:
+        changelogs.append(self._ParseChangeLogFromLogData(log))
 
-      changelogs.append(changelog)
+      if 'next' in data:
+        next_end_revision = data['next']
+      else:
+        next_end_revision = None
 
     return changelogs