Extensions Docs Server: Large performance increase

chrome/common/extensions/docs/server2/handler.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
+from StringIO import StringIO
 import sys
 
 from appengine_wrappers import webapp
 from appengine_wrappers import memcache
 from appengine_wrappers import urlfetch
 
 from api_data_source import APIDataSource
 from api_list_data_source import APIListDataSource
 from appengine_blobstore import AppEngineBlobstore
 from appengine_memcache import AppEngineMemcache
 from appengine_url_fetcher import AppEngineUrlFetcher
 from branch_utility import BranchUtility
 from example_zipper import ExampleZipper
 from file_system_cache import FileSystemCache
+import file_system_cache as fs_cache
 from github_file_system import GithubFileSystem
 from intro_data_source import IntroDataSource
 from local_file_system import LocalFileSystem
 from memcache_file_system import MemcacheFileSystem
 from samples_data_source import SamplesDataSource
 from server_instance import ServerInstance
 from subversion_file_system import SubversionFileSystem
 from template_data_source import TemplateDataSource
 import url_constants
 
 # The branch that the server will default to when no branch is specified in the
 # URL. This is necessary because it is not possible to pass flags to the script
 # handler.
 DEFAULT_BRANCH = 'local'
 
 BRANCH_UTILITY_MEMCACHE = AppEngineMemcache('branch_utility')
 BRANCH_UTILITY = BranchUtility(url_constants.OMAHA_PROXY_URL,
                                DEFAULT_BRANCH,
                                AppEngineUrlFetcher(None),
                                BRANCH_UTILITY_MEMCACHE)
 
+GITHUB_MEMCACHE = AppEngineMemcache('github')
 GITHUB_FILE_SYSTEM = GithubFileSystem(
     AppEngineUrlFetcher(url_constants.GITHUB_URL),
-    AppEngineMemcache('github'),
+    GITHUB_MEMCACHE,
     AppEngineBlobstore())
-GITHUB_CACHE_BUILDER = FileSystemCache.Builder(GITHUB_FILE_SYSTEM)
+GITHUB_CACHE_BUILDER = FileSystemCache.Builder(GITHUB_FILE_SYSTEM,
+                                               GITHUB_MEMCACHE)
 
 STATIC_DIR_PREFIX = 'docs/server2'
 EXTENSIONS_PATH = 'chrome/common/extensions'
 DOCS_PATH = 'docs'
 API_PATH = 'api'
-INTRO_PATH = DOCS_PATH + '/server2/templates/intros'
-ARTICLE_PATH = DOCS_PATH + '/server2/templates/articles'
-PUBLIC_TEMPLATE_PATH = DOCS_PATH + '/server2/templates/public'
-PRIVATE_TEMPLATE_PATH = DOCS_PATH + '/server2/templates/private'
+TEMPLATE_PATH = DOCS_PATH + '/server2/templates'
+INTRO_PATH = TEMPLATE_PATH + '/intros'
+ARTICLE_PATH = TEMPLATE_PATH + '/articles'
+PUBLIC_TEMPLATE_PATH = TEMPLATE_PATH + '/public'
+PRIVATE_TEMPLATE_PATH = TEMPLATE_PATH + '/private'
 EXAMPLES_PATH = DOCS_PATH + '/examples'
-FULL_EXAMPLES_PATH = DOCS_PATH + '/' + EXAMPLES_PATH
 
 # Global cache of instances because Handler is recreated for every request.
 SERVER_INSTANCES = {}
 
+def _CreateMemcacheFileSystem(branch, branch_memcache):
+  svn_url = _GetURLFromBranch(branch) + '/' + EXTENSIONS_PATH
+  stat_fetcher = AppEngineUrlFetcher(
+      svn_url.replace(url_constants.SVN_URL, url_constants.VIEWVC_URL))
+  fetcher = AppEngineUrlFetcher(svn_url)
+  return MemcacheFileSystem(SubversionFileSystem(fetcher, stat_fetcher),
+                            branch_memcache)
+
 def _GetInstanceForBranch(channel_name, local_path):
   branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel_name)
   if branch in SERVER_INSTANCES:
     return SERVER_INSTANCES[branch]
+  branch_memcache = AppEngineMemcache(branch)
   if branch == 'local':
     file_system = LocalFileSystem(local_path)
   else:
-    svn_url = _GetURLFromBranch(branch) + '/' + EXTENSIONS_PATH
-    stat_fetcher = AppEngineUrlFetcher(
-        svn_url.replace(url_constants.SVN_URL, url_constants.VIEWVC_URL))
-    fetcher = AppEngineUrlFetcher(svn_url)
-    file_system = MemcacheFileSystem(
-        SubversionFileSystem(fetcher, stat_fetcher),
-        AppEngineMemcache(branch))
+    file_system = _CreateMemcacheFileSystem(branch, branch_memcache)
 
-  cache_builder = FileSystemCache.Builder(file_system)
-  api_list_data_source = APIListDataSource(cache_builder,
-                                           file_system,
-                                           API_PATH,
-                                           PUBLIC_TEMPLATE_PATH)
-  intro_data_source = IntroDataSource(cache_builder,
-                                      [INTRO_PATH, ARTICLE_PATH])
+  cache_builder = FileSystemCache.Builder(file_system, branch_memcache)
+  api_list_data_source_factory = APIListDataSource.Factory(cache_builder,
+                                                           file_system,
+                                                           API_PATH,
+                                                           PUBLIC_TEMPLATE_PATH)
+  intro_data_source_factory = IntroDataSource.Factory(
+      cache_builder,
+      [INTRO_PATH, ARTICLE_PATH])
   samples_data_source_factory = SamplesDataSource.Factory(branch,
                                                           file_system,
                                                           GITHUB_FILE_SYSTEM,
                                                           cache_builder,
                                                           GITHUB_CACHE_BUILDER,
                                                           EXAMPLES_PATH)
   api_data_source_factory = APIDataSource.Factory(cache_builder,
                                                   API_PATH,
                                                   samples_data_source_factory)
   template_data_source_factory = TemplateDataSource.Factory(
       channel_name,
       api_data_source_factory,
-      api_list_data_source,
-      intro_data_source,
+      api_list_data_source_factory,
+      intro_data_source_factory,
       samples_data_source_factory,
       cache_builder,
       PUBLIC_TEMPLATE_PATH,
       PRIVATE_TEMPLATE_PATH)
   example_zipper = ExampleZipper(file_system,
                                  cache_builder,
                                  DOCS_PATH,
                                  EXAMPLES_PATH)
   SERVER_INSTANCES[branch] = ServerInstance(
       template_data_source_factory,
       example_zipper,
       cache_builder)
   return SERVER_INSTANCES[branch]
 
 def _GetURLFromBranch(branch):
     if branch == 'trunk':
       return url_constants.SVN_TRUNK_URL + '/src'
     return url_constants.SVN_BRANCH_URL + '/' + branch + '/src'
 
 def _CleanBranches():
   numbers = BRANCH_UTILITY.GetAllBranchNumbers()
   for key in SERVER_INSTANCES.keys():
     if key not in numbers:
       SERVER_INSTANCES.pop(key)
 
+class _MockResponse(object):
+  def __init__(self):
+    self.status = 200
+    self.out = StringIO()
+
+  def set_status(self, status):
+    self.status = status
+
+class _MockRequest(object):
+  def __init__(self, path):
+    self.headers = {}
+    self.path = path
+
 class Handler(webapp.RequestHandler):
   def __init__(self, request, response, local_path=EXTENSIONS_PATH):
     self._local_path = local_path
     super(Handler, self).__init__(request, response)
 
   def _NavigateToPath(self, path):

[not at google - send to devlin, 2012/08/20 05:27:10]

this name always trips me up. Can we call it _HandleGet or something, to
distinguish it from the cron case?

[cduvall, 2012/08/20 21:28:09]

Done.

     channel_name, real_path = BRANCH_UTILITY.SplitChannelNameFromPath(path)
     # TODO: Detect that these are directories and serve index.html out of them.
     if real_path.strip('/') == 'apps':
       real_path = 'apps/index.html'
     if real_path.strip('/') == 'extensions':
       real_path = 'extensions/index.html'
     _CleanBranches()
     _GetInstanceForBranch(channel_name, self._local_path).Get(real_path,
                                                               self.request,
                                                               self.response)
 
+  def _Render(self, files, branch):
+    for f in files:
+      path = branch + f.split(PUBLIC_TEMPLATE_PATH)[-1]
+      self.request = _MockRequest(path)
+      self.response = _MockResponse()
+      self._NavigateToPath(path)
+
+  def _Cron(self, files):
+    self._file_system.Read(files).Get()
+
   def get(self):
     path = self.request.path
-    if '_ah/warmup' in path:
-      logging.info('Warmup request.')
-      self._NavigateToPath('trunk/extensions/samples.html')
-      self._NavigateToPath('dev/extensions/samples.html')
-      self._NavigateToPath('beta/extensions/samples.html')
-      self._NavigateToPath('stable/extensions/samples.html')
-      # Only do this request if we are on the deployed server.
-      # Bug: http://crbug.com/141910
-      if DEFAULT_BRANCH != 'local':
-        self._NavigateToPath('apps/samples.html')
+    if path.startswith('/cron'):

[not at google - send to devlin, 2012/08/20 05:27:10]

nit: can we split up these two distinct cases (cron vs get) into two distinct
functions, like,

path = self.request.path
if path.startswith('/cron'):
  self._HandleCron(path.split('/')[-1])
else:
  # Redirect paths...
  path = ...
  self._HandleGet(self.path)

[cduvall, 2012/08/20 21:28:09]

Done.

+      branch = path.split('/')[-1]
+      logging.info('Running cron job for %s.' % branch)
+      branch_memcache = AppEngineMemcache(branch)
+      self._file_system = _CreateMemcacheFileSystem(branch, branch_memcache)

[not at google - send to devlin, 2012/08/20 05:27:10]

file_system doesn't need to be on self, we can define _Cron as a local function,
or even as a lamba.

[cduvall, 2012/08/20 21:28:09]

Done.

+      builder = FileSystemCache.Builder(self._file_system, branch_memcache)
+      cache = builder.build(self._Cron, fs_cache.FS_CACHE_CRON)
+      render_cache = builder.build(lambda x: self._Render(x, branch),
+                                   fs_cache.FS_CACHE_RENDER)
+      cache.GetFromFileListing(TEMPLATE_PATH)
+      cache.GetFromFileListing(API_PATH)
+      cache.GetFromFileListing(EXAMPLES_PATH)
+      render_cache.GetFromFileListing(PUBLIC_TEMPLATE_PATH)

[not at google - send to devlin, 2012/08/20 05:27:10]

why does rendering all files from the public template path not pull in
everything that the server needs? I.e. why all the cache.GetFromFileListing
stuff above?

I thought maybe it was an optimisation to prevent needing to render anything (we
really just need to compile things), but yeah, this seems non-trivial. Easier to
render everything even though all that string joining is a bit wasted.

[cduvall, 2012/08/20 21:28:09]

I just thought it would be a little faster getting all the files ahead of time
in one large parallel fetch, so all the rendering could just use the files from
memcache.

[not at google - send to devlin, 2012/08/21 00:30:11]

Interesting. Did you find that to be the case? If it's faster to get this stuff
ahead of time then we should do it :\ sorry. Though comment that's why you're
doing it. And make sure it is actually faster.

[cduvall, 2012/08/21 01:33:33]

I don't think it makes too much of a difference, so I'm going with simpler is
better.

       return
 
     # Redirect paths like "directory" to "directory/". This is so relative file
     # paths will know to treat this as a directory.
     if os.path.splitext(path)[1] == '' and path[-1] != '/':
       self.redirect(path + '/')
     path = path.replace('/chrome/', '')
     path = path.strip('/')
     self._NavigateToPath(path)