Extensions Docs Server: Large performance increase

chrome/common/extensions/docs/server2/handler.py

Index: chrome/common/extensions/docs/server2/handler.py
diff --git a/chrome/common/extensions/docs/server2/handler.py b/chrome/common/extensions/docs/server2/handler.py
index 9cf6c0f24939f14df82d82990d6f89c974175fdc..f6af7acd473d16f80d25cb2cdc2a3c1b0c04b8a7 100644
--- a/chrome/common/extensions/docs/server2/handler.py
+++ b/chrome/common/extensions/docs/server2/handler.py
@@ -4,6 +4,7 @@
 
 import logging
 import os
+from StringIO import StringIO
 import sys
 
 from appengine_wrappers import webapp
@@ -18,6 +19,7 @@ from appengine_url_fetcher import AppEngineUrlFetcher
 from branch_utility import BranchUtility
 from example_zipper import ExampleZipper
 from file_system_cache import FileSystemCache
+import file_system_cache as fs_cache
 from github_file_system import GithubFileSystem
 from intro_data_source import IntroDataSource
 from local_file_system import LocalFileSystem
@@ -39,48 +41,54 @@ BRANCH_UTILITY = BranchUtility(url_constants.OMAHA_PROXY_URL,
                                AppEngineUrlFetcher(None),
                                BRANCH_UTILITY_MEMCACHE)
 
+GITHUB_MEMCACHE = AppEngineMemcache('github')
 GITHUB_FILE_SYSTEM = GithubFileSystem(
     AppEngineUrlFetcher(url_constants.GITHUB_URL),
-    AppEngineMemcache('github'),
+    GITHUB_MEMCACHE,
     AppEngineBlobstore())
-GITHUB_CACHE_BUILDER = FileSystemCache.Builder(GITHUB_FILE_SYSTEM)
+GITHUB_CACHE_BUILDER = FileSystemCache.Builder(GITHUB_FILE_SYSTEM,
+                                               GITHUB_MEMCACHE)
 
 STATIC_DIR_PREFIX = 'docs/server2'
 EXTENSIONS_PATH = 'chrome/common/extensions'
 DOCS_PATH = 'docs'
 API_PATH = 'api'
-INTRO_PATH = DOCS_PATH + '/server2/templates/intros'
-ARTICLE_PATH = DOCS_PATH + '/server2/templates/articles'
-PUBLIC_TEMPLATE_PATH = DOCS_PATH + '/server2/templates/public'
-PRIVATE_TEMPLATE_PATH = DOCS_PATH + '/server2/templates/private'
+TEMPLATE_PATH = DOCS_PATH + '/server2/templates'
+INTRO_PATH = TEMPLATE_PATH + '/intros'
+ARTICLE_PATH = TEMPLATE_PATH + '/articles'
+PUBLIC_TEMPLATE_PATH = TEMPLATE_PATH + '/public'
+PRIVATE_TEMPLATE_PATH = TEMPLATE_PATH + '/private'
 EXAMPLES_PATH = DOCS_PATH + '/examples'
-FULL_EXAMPLES_PATH = DOCS_PATH + '/' + EXAMPLES_PATH
 
 # Global cache of instances because Handler is recreated for every request.
 SERVER_INSTANCES = {}
 
+def _CreateMemcacheFileSystem(branch, branch_memcache):
+  svn_url = _GetURLFromBranch(branch) + '/' + EXTENSIONS_PATH
+  stat_fetcher = AppEngineUrlFetcher(
+      svn_url.replace(url_constants.SVN_URL, url_constants.VIEWVC_URL))
+  fetcher = AppEngineUrlFetcher(svn_url)
+  return MemcacheFileSystem(SubversionFileSystem(fetcher, stat_fetcher),
+                            branch_memcache)
+
 def _GetInstanceForBranch(channel_name, local_path):
   branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel_name)
   if branch in SERVER_INSTANCES:
     return SERVER_INSTANCES[branch]
+  branch_memcache = AppEngineMemcache(branch)
   if branch == 'local':
     file_system = LocalFileSystem(local_path)
   else:
-    svn_url = _GetURLFromBranch(branch) + '/' + EXTENSIONS_PATH
-    stat_fetcher = AppEngineUrlFetcher(
-        svn_url.replace(url_constants.SVN_URL, url_constants.VIEWVC_URL))
-    fetcher = AppEngineUrlFetcher(svn_url)
-    file_system = MemcacheFileSystem(
-        SubversionFileSystem(fetcher, stat_fetcher),
-        AppEngineMemcache(branch))
-
-  cache_builder = FileSystemCache.Builder(file_system)
-  api_list_data_source = APIListDataSource(cache_builder,
-                                           file_system,
-                                           API_PATH,
-                                           PUBLIC_TEMPLATE_PATH)
-  intro_data_source = IntroDataSource(cache_builder,
-                                      [INTRO_PATH, ARTICLE_PATH])
+    file_system = _CreateMemcacheFileSystem(branch, branch_memcache)
+
+  cache_builder = FileSystemCache.Builder(file_system, branch_memcache)
+  api_list_data_source_factory = APIListDataSource.Factory(cache_builder,
+                                                           file_system,
+                                                           API_PATH,
+                                                           PUBLIC_TEMPLATE_PATH)
+  intro_data_source_factory = IntroDataSource.Factory(
+      cache_builder,
+      [INTRO_PATH, ARTICLE_PATH])
   samples_data_source_factory = SamplesDataSource.Factory(branch,
                                                           file_system,
                                                           GITHUB_FILE_SYSTEM,
@@ -93,8 +101,8 @@ def _GetInstanceForBranch(channel_name, local_path):
   template_data_source_factory = TemplateDataSource.Factory(
       channel_name,
       api_data_source_factory,
-      api_list_data_source,
-      intro_data_source,
+      api_list_data_source_factory,
+      intro_data_source_factory,
       samples_data_source_factory,
       cache_builder,
       PUBLIC_TEMPLATE_PATH,
@@ -120,6 +128,19 @@ def _CleanBranches():
     if key not in numbers:
       SERVER_INSTANCES.pop(key)
 
+class _MockResponse(object):
+  def __init__(self):
+    self.status = 200
+    self.out = StringIO()
+
+  def set_status(self, status):
+    self.status = status
+
+class _MockRequest(object):
+  def __init__(self, path):
+    self.headers = {}
+    self.path = path
+
 class Handler(webapp.RequestHandler):
   def __init__(self, request, response, local_path=EXTENSIONS_PATH):
     self._local_path = local_path
@@ -137,18 +158,31 @@ class Handler(webapp.RequestHandler):
                                                               self.request,
                                                               self.response)
 
+  def _Render(self, files, branch):
+    for f in files:
+      path = branch + f.split(PUBLIC_TEMPLATE_PATH)[-1]
+      self.request = _MockRequest(path)
+      self.response = _MockResponse()
+      self._NavigateToPath(path)
+
+  def _Cron(self, files):
+    self._file_system.Read(files).Get()
+
   def get(self):
     path = self.request.path
-    if '_ah/warmup' in path:
-      logging.info('Warmup request.')
-      self._NavigateToPath('trunk/extensions/samples.html')
-      self._NavigateToPath('dev/extensions/samples.html')
-      self._NavigateToPath('beta/extensions/samples.html')
-      self._NavigateToPath('stable/extensions/samples.html')
-      # Only do this request if we are on the deployed server.
-      # Bug: http://crbug.com/141910
-      if DEFAULT_BRANCH != 'local':
-        self._NavigateToPath('apps/samples.html')
+    if path.startswith('/cron'):

[not at google - send to devlin, 2012/08/20 05:27:10]

nit: can we split up these two distinct cases (cron vs get) into two distinct
functions, like,

path = self.request.path
if path.startswith('/cron'):
  self._HandleCron(path.split('/')[-1])
else:
  # Redirect paths...
  path = ...
  self._HandleGet(self.path)

[cduvall, 2012/08/20 21:28:09]

Done.

+      branch = path.split('/')[-1]
+      logging.info('Running cron job for %s.' % branch)
+      branch_memcache = AppEngineMemcache(branch)
+      self._file_system = _CreateMemcacheFileSystem(branch, branch_memcache)

[not at google - send to devlin, 2012/08/20 05:27:10]

file_system doesn't need to be on self, we can define _Cron as a local function,
or even as a lamba.

[cduvall, 2012/08/20 21:28:09]

Done.

+      builder = FileSystemCache.Builder(self._file_system, branch_memcache)
+      cache = builder.build(self._Cron, fs_cache.FS_CACHE_CRON)
+      render_cache = builder.build(lambda x: self._Render(x, branch),
+                                   fs_cache.FS_CACHE_RENDER)
+      cache.GetFromFileListing(TEMPLATE_PATH)
+      cache.GetFromFileListing(API_PATH)
+      cache.GetFromFileListing(EXAMPLES_PATH)
+      render_cache.GetFromFileListing(PUBLIC_TEMPLATE_PATH)

[not at google - send to devlin, 2012/08/20 05:27:10]

why does rendering all files from the public template path not pull in
everything that the server needs? I.e. why all the cache.GetFromFileListing
stuff above?

I thought maybe it was an optimisation to prevent needing to render anything (we
really just need to compile things), but yeah, this seems non-trivial. Easier to
render everything even though all that string joining is a bit wasted.

[cduvall, 2012/08/20 21:28:09]

I just thought it would be a little faster getting all the files ahead of time
in one large parallel fetch, so all the rendering could just use the files from
memcache.

[not at google - send to devlin, 2012/08/21 00:30:11]

Interesting. Did you find that to be the case? If it's faster to get this stuff
ahead of time then we should do it :\ sorry. Though comment that's why you're
doing it. And make sure it is actually faster.

[cduvall, 2012/08/21 01:33:33]

I don't think it makes too much of a difference, so I'm going with simpler is
better.

       return
 
     # Redirect paths like "directory" to "directory/". This is so relative file