Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(145)

Unified Diff: chrome/common/extensions/docs/server2/handler.py

Issue 13470005: Refactor the devserver to make it easier to control caching (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: cduvall, rebase Created 7 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: chrome/common/extensions/docs/server2/handler.py
diff --git a/chrome/common/extensions/docs/server2/handler.py b/chrome/common/extensions/docs/server2/handler.py
index 8fbad1695bba8a2cc354c114712f9b798e414b10..24ceb226476365efbdfa0231d691520d12ead58b 100644
--- a/chrome/common/extensions/docs/server2/handler.py
+++ b/chrome/common/extensions/docs/server2/handler.py
@@ -5,302 +5,105 @@
import logging
import os
from StringIO import StringIO
-import sys
from appengine_wrappers import webapp
from appengine_wrappers import memcache
from appengine_wrappers import urlfetch
-
-from api_data_source import APIDataSource
-from api_list_data_source import APIListDataSource
-from appengine_blobstore import AppEngineBlobstore
-from in_memory_object_store import InMemoryObjectStore
-from appengine_url_fetcher import AppEngineUrlFetcher
from branch_utility import BranchUtility
-from example_zipper import ExampleZipper
-from compiled_file_system import CompiledFileSystem
-import compiled_file_system as compiled_fs
-from github_file_system import GithubFileSystem
-from intro_data_source import IntroDataSource
-from known_issues_data_source import KnownIssuesDataSource
-from local_file_system import LocalFileSystem
-from memcache_file_system import MemcacheFileSystem
-from reference_resolver import ReferenceResolver
-from samples_data_source import SamplesDataSource
from server_instance import ServerInstance
-from sidenav_data_source import SidenavDataSource
-from subversion_file_system import SubversionFileSystem
-from template_data_source import TemplateDataSource
-from third_party.json_schema_compiler.model import UnixName
-import url_constants
-
-# Increment this version to force the server to reload all pages in the first
-# cron job that is run.
-_VERSION = 1
+import svn_constants
+import time
# The default channel to serve docs for if no channel is specified.
_DEFAULT_CHANNEL = 'stable'
-BRANCH_UTILITY_MEMCACHE = InMemoryObjectStore('branch_utility')
-BRANCH_UTILITY = BranchUtility(url_constants.OMAHA_PROXY_URL,
- AppEngineUrlFetcher(None),
- BRANCH_UTILITY_MEMCACHE)
-
-GITHUB_MEMCACHE = InMemoryObjectStore('github')
-GITHUB_FILE_SYSTEM = GithubFileSystem(
- AppEngineUrlFetcher(url_constants.GITHUB_URL),
- GITHUB_MEMCACHE,
- AppEngineBlobstore())
-GITHUB_COMPILED_FILE_SYSTEM = CompiledFileSystem.Factory(GITHUB_FILE_SYSTEM,
- GITHUB_MEMCACHE)
-
-EXTENSIONS_PATH = 'chrome/common/extensions'
-DOCS_PATH = 'docs'
-API_PATH = 'api'
-TEMPLATE_PATH = DOCS_PATH + '/templates'
-INTRO_PATH = TEMPLATE_PATH + '/intros'
-ARTICLE_PATH = TEMPLATE_PATH + '/articles'
-PUBLIC_TEMPLATE_PATH = TEMPLATE_PATH + '/public'
-PRIVATE_TEMPLATE_PATH = TEMPLATE_PATH + '/private'
-EXAMPLES_PATH = DOCS_PATH + '/examples'
-JSON_PATH = TEMPLATE_PATH + '/json'
-
-# Global cache of instances because Handler is recreated for every request.
-SERVER_INSTANCES = {}
-
-def _GetURLFromBranch(branch):
- if branch == 'trunk':
- return url_constants.SVN_TRUNK_URL + '/src'
- return url_constants.SVN_BRANCH_URL + '/' + branch + '/src'
-
-def _SplitFilenameUnix(base_dir, files):
- return [UnixName(os.path.splitext(f.split('/')[-1])[0]) for f in files]
-
-def _CreateMemcacheFileSystem(branch, branch_memcache):
- svn_url = _GetURLFromBranch(branch) + '/' + EXTENSIONS_PATH
- stat_fetcher = AppEngineUrlFetcher(
- svn_url.replace(url_constants.SVN_URL, url_constants.VIEWVC_URL))
- fetcher = AppEngineUrlFetcher(svn_url)
- return MemcacheFileSystem(SubversionFileSystem(fetcher, stat_fetcher),
- branch_memcache)
-
-_default_branch = BRANCH_UTILITY.GetBranchNumberForChannelName(_DEFAULT_CHANNEL)
-APPS_MEMCACHE = InMemoryObjectStore(_default_branch)
-APPS_FILE_SYSTEM = _CreateMemcacheFileSystem(_default_branch, APPS_MEMCACHE)
-APPS_COMPILED_FILE_SYSTEM = CompiledFileSystem.Factory(
- APPS_FILE_SYSTEM,
- APPS_MEMCACHE).Create(_SplitFilenameUnix, compiled_fs.APPS_FS)
-
-EXTENSIONS_MEMCACHE = InMemoryObjectStore(_default_branch)
-EXTENSIONS_FILE_SYSTEM = _CreateMemcacheFileSystem(_default_branch,
- EXTENSIONS_MEMCACHE)
-EXTENSIONS_COMPILED_FILE_SYSTEM = CompiledFileSystem.Factory(
- EXTENSIONS_FILE_SYSTEM,
- EXTENSIONS_MEMCACHE).Create(_SplitFilenameUnix, compiled_fs.EXTENSIONS_FS)
-
-KNOWN_ISSUES_DATA_SOURCE = KnownIssuesDataSource(
- InMemoryObjectStore('KnownIssues'),
- AppEngineUrlFetcher(None))
-
-def _MakeInstanceKey(branch, number):
- return '%s/%s' % (branch, number)
-
-def _GetInstanceForBranch(channel_name, local_path):
- branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel_name)
-
- # The key for the server is a tuple of |channel_name| with |branch|, since
- # sometimes stable and beta point to the same branch.
- instance_key = _MakeInstanceKey(channel_name, branch)
- instance = SERVER_INSTANCES.get(instance_key, None)
- if instance is not None:
- return instance
-
- branch_memcache = InMemoryObjectStore(branch)
- file_system = _CreateMemcacheFileSystem(branch, branch_memcache)
- cache_factory = CompiledFileSystem.Factory(file_system, branch_memcache)
- api_list_data_source_factory = APIListDataSource.Factory(cache_factory,
- file_system,
- API_PATH,
- PUBLIC_TEMPLATE_PATH)
- api_data_source_factory = APIDataSource.Factory(
- cache_factory,
- API_PATH)
-
- # Give the ReferenceResolver a memcache, to speed up the lookup of
- # duplicate $refs.
- ref_resolver_factory = ReferenceResolver.Factory(
- api_data_source_factory,
- api_list_data_source_factory,
- branch_memcache)
- api_data_source_factory.SetReferenceResolverFactory(ref_resolver_factory)
- samples_data_source_factory = SamplesDataSource.Factory(
- channel_name,
- file_system,
- GITHUB_FILE_SYSTEM,
- cache_factory,
- GITHUB_COMPILED_FILE_SYSTEM,
- ref_resolver_factory,
- EXAMPLES_PATH)
- api_data_source_factory.SetSamplesDataSourceFactory(
- samples_data_source_factory)
- intro_data_source_factory = IntroDataSource.Factory(
- cache_factory,
- ref_resolver_factory,
- [INTRO_PATH, ARTICLE_PATH])
- sidenav_data_source_factory = SidenavDataSource.Factory(cache_factory,
- JSON_PATH)
- template_data_source_factory = TemplateDataSource.Factory(
- channel_name,
- api_data_source_factory,
- api_list_data_source_factory,
- intro_data_source_factory,
- samples_data_source_factory,
- KNOWN_ISSUES_DATA_SOURCE,
- sidenav_data_source_factory,
- cache_factory,
- ref_resolver_factory,
- PUBLIC_TEMPLATE_PATH,
- PRIVATE_TEMPLATE_PATH)
- example_zipper = ExampleZipper(file_system,
- cache_factory,
- DOCS_PATH)
-
- instance = ServerInstance(template_data_source_factory,
- example_zipper,
- cache_factory)
- SERVER_INSTANCES[instance_key] = instance
- return instance
-
-def _CleanBranches():
- keys = [_MakeInstanceKey(branch, number)
- for branch, number in BRANCH_UTILITY.GetAllBranchNumbers()]
- for key in SERVER_INSTANCES.keys():
- if key not in keys:
- SERVER_INSTANCES.pop(key)
-
-class _MockResponse(object):
- def __init__(self):
- self.status = 200
- self.out = StringIO()
- self.headers = {}
-
- def set_status(self, status):
- self.status = status
-
- def clear(self, *args):
- pass
-
-class _MockRequest(object):
- def __init__(self, path):
- self.headers = {}
- self.path = path
- self.url = 'http://localhost' + path
-
class Handler(webapp.RequestHandler):
- def __init__(self, request, response, local_path=EXTENSIONS_PATH):
- self._local_path = local_path
+ def __init__(self, request, response):
super(Handler, self).__init__(request, response)
def _HandleGet(self, path):
- channel_name, real_path = BRANCH_UTILITY.SplitChannelNameFromPath(path)
+ channel_name, real_path = BranchUtility.SplitChannelNameFromPath(path)
if channel_name == _DEFAULT_CHANNEL:
self.redirect('/%s' % real_path)
return
- # TODO: Detect that these are directories and serve index.html out of them.
+ if channel_name is None:
+ channel_name = _DEFAULT_CHANNEL
+
+ # TODO(kalman): Check if |path| is a directory and serve path/index.html
+ # rather than special-casing apps/extensions.
if real_path.strip('/') == 'apps':
real_path = 'apps/index.html'
if real_path.strip('/') == 'extensions':
real_path = 'extensions/index.html'
- if (not real_path.startswith('extensions/') and
- not real_path.startswith('apps/') and
- not real_path.startswith('static/')):
- if self._RedirectBadPaths(real_path, channel_name):
- return
-
- _CleanBranches()
-
- # Yes, do this after it's passed to RedirectBadPaths. That needs to know
- # whether or not a branch was specified.
- if channel_name is None:
- channel_name = _DEFAULT_CHANNEL
- _GetInstanceForBranch(channel_name, self._local_path).Get(real_path,
- self.request,
- self.response)
-
- def _Render(self, files, channel):
- original_response = self.response
- for f in files:
- if f.endswith('404.html'):
- continue
- path = channel + f.split(PUBLIC_TEMPLATE_PATH)[-1]
- self.request = _MockRequest(path)
- self.response = _MockResponse()
- try:
- self._HandleGet(path)
- except Exception as e:
- logging.error('Error rendering %s: %s' % (path, str(e)))
- self.response = original_response
-
- class _ValueHolder(object):
- """Class to allow a value to be changed within a lambda.
- """
- def __init__(self, starting_value):
- self._value = starting_value
+ server_instance = ServerInstance.GetOrCreate(channel_name)
- def Set(self, value):
- self._value = value
+ canonical_path = server_instance.path_canonicalizer.Canonicalize(real_path)
+ if real_path != canonical_path:
+ self.redirect(canonical_path)
+ return
- def Get(self):
- return self._value
+ ServerInstance.GetOrCreate(channel_name).Get(real_path,
+ self.request,
+ self.response)
def _HandleCron(self, path):
- # Cache population strategy:
- #
- # We could list all files in PUBLIC_TEMPLATE_PATH then render them. However,
- # this would be inefficient in the common case where files haven't changed
- # since the last cron.
+ # Cron strategy:
#
- # Instead, let the CompiledFileSystem give us clues when to re-render: we
- # use the CFS to check whether the templates, examples, or API folders have
- # been changed. If there has been a change, the compilation function will
- # be called. The same is then done separately with the apps samples page,
- # since it pulls its data from Github.
- channel = path.split('/')[-1]
- branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel)
- logging.info('Running cron job for %s.' % branch)
- branch_memcache = InMemoryObjectStore(branch)
- file_system = _CreateMemcacheFileSystem(branch, branch_memcache)
- factory = CompiledFileSystem.Factory(file_system, branch_memcache)
+ # Find all public template files and static files, and render them. Most of
+ # the time these won't have changed since the last cron run, so it's a
+ # little wasteful, but hopefully rendering is really fast (if it isn't we
+ # have a problem).
+ class MockResponse(object):
+ def __init__(self):
+ self.status = 200
+ self.out = StringIO()
+ self.headers = {}
+ def set_status(self, status):
+ self.status = status
+ def clear(self, *args):
+ pass
+
+ class MockRequest(object):
+ def __init__(self, path):
+ self.headers = {}
+ self.path = path
+ self.url = '//localhost/%s' % path
- needs_render = self._ValueHolder(False)
- invalidation_cache = factory.Create(lambda _, __: needs_render.Set(True),
- compiled_fs.CRON_INVALIDATION,
- version=_VERSION)
- for path in [TEMPLATE_PATH, EXAMPLES_PATH, API_PATH]:
- invalidation_cache.GetFromFile(path + '/')
-
- if needs_render.Get():
- file_listing_cache = factory.Create(lambda _, x: x,
- compiled_fs.CRON_FILE_LISTING)
- self._Render(file_listing_cache.GetFromFileListing(PUBLIC_TEMPLATE_PATH),
- channel)
+ channel = path.split('/')[-1]
+ logging.info('cron/%s: starting' % channel)
+
+ server_instance = ServerInstance.GetOrCreate(channel)
+
+ def run_cron_for_dir(d):
+ error = None
+ start_time = time.time()
+ files = [f for f in server_instance.content_cache.GetFromFileListing(d)
+ if not f.endswith('/')]
+ for f in files:
+ try:
+ server_instance.Get(f, MockRequest(f), MockResponse())
+ except error:
+ logging.error('cron/%s: error rendering %s/%s: %s' % (
+ channel, d, f, error))
+ logging.info('cron/%s: rendering %s files in %s took %s seconds' % (
+ channel, len(files), d, time.time() - start_time))
+ return error
+
+ # Don't use "or" since we want to evaluate everything no matter what.
+ was_error = any((run_cron_for_dir(svn_constants.PUBLIC_TEMPLATE_PATH),
+ run_cron_for_dir(svn_constants.STATIC_PATH)))
+
+ if was_error:
+ self.response.status = 500
+ self.response.out.write('Failure')
else:
- # If |needs_render| was True, this page was already rendered, and we don't
- # need to render again.
- github_invalidation_cache = GITHUB_COMPILED_FILE_SYSTEM.Create(
- lambda _, __: needs_render.Set(True),
- compiled_fs.CRON_GITHUB_INVALIDATION)
- if needs_render.Get():
- self._Render([PUBLIC_TEMPLATE_PATH + '/apps/samples.html'], channel)
-
- # It's good to keep the extensions samples page fresh, because if it
- # gets dropped from the cache ALL the extensions pages time out.
- self._Render([PUBLIC_TEMPLATE_PATH + '/extensions/samples.html'], channel)
+ self.response.status = 200
+ self.response.out.write('Success')
- self.response.out.write('Success')
+ logging.info('cron/%s: finished' % channel)
def _RedirectSpecialCases(self, path):
google_dev_url = 'http://developer.google.com/chrome'
@@ -314,38 +117,16 @@ class Handler(webapp.RequestHandler):
return False
- def _RedirectBadPaths(self, path, channel_name):
- if '/' in path or path == '404.html':
- return False
- apps_templates = APPS_COMPILED_FILE_SYSTEM.GetFromFileListing(
- PUBLIC_TEMPLATE_PATH + '/apps')
- extensions_templates = EXTENSIONS_COMPILED_FILE_SYSTEM.GetFromFileListing(
- PUBLIC_TEMPLATE_PATH + '/extensions')
- unix_path = UnixName(os.path.splitext(path)[0])
- if channel_name is None:
- apps_path = '/apps/%s' % path
- extensions_path = '/extensions/%s' % path
- else:
- apps_path = '/%s/apps/%s' % (channel_name, path)
- extensions_path = '/%s/extensions/%s' % (channel_name, path)
- if unix_path in extensions_templates:
- self.redirect(extensions_path)
- elif unix_path in apps_templates:
- self.redirect(apps_path)
- else:
- self.redirect(extensions_path)
- return True
-
def _RedirectFromCodeDotGoogleDotCom(self, path):
if (not self.request.url.startswith(('http://code.google.com',
'https://code.google.com'))):
return False
- newUrl = 'http://developer.chrome.com/'
+ new_url = 'http://developer.chrome.com/'
# switch to https if necessary
if (self.request.url.startswith('https')):
- newUrl = newUrl.replace('http', 'https', 1)
+ new_url = new_url.replace('http', 'https', 1)
path = path.split('/')
if len(path) > 0 and path[0] == 'chrome':
@@ -355,8 +136,8 @@ class Handler(webapp.RequestHandler):
position = path.index(channel)
path.pop(position)
path.insert(0, channel)
- newUrl += '/'.join(path)
- self.redirect(newUrl)
+ new_url += '/'.join(path)
+ self.redirect(new_url)
return True
def get(self):
@@ -375,5 +156,7 @@ class Handler(webapp.RequestHandler):
return
path = path.strip('/')
- if not self._RedirectFromCodeDotGoogleDotCom(path):
- self._HandleGet(path)
+ if self._RedirectFromCodeDotGoogleDotCom(path):
+ return
+
+ self._HandleGet(path)

Powered by Google App Engine
This is Rietveld 408576698