Doc server broken link detection

chrome/common/extensions/docs/server2/cron_servlet.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
+import posixpath
 import time
 import traceback
 
 from app_yaml_helper import AppYamlHelper
 from appengine_wrappers import (
     GetAppVersion, DeadlineExceededError, IsDevServer, logservice)
 from branch_utility import BranchUtility
 from caching_file_system import CachingFileSystem
 from compiled_file_system import CompiledFileSystem
 from empty_dir_file_system import EmptyDirFileSystem
 from github_file_system import GithubFileSystem
 from object_store_creator import ObjectStoreCreator
 from render_servlet import RenderServlet
 from server_instance import ServerInstance
 from servlet import Servlet, Request, Response
 from subversion_file_system import SubversionFileSystem
 import svn_constants
 from third_party.json_schema_compiler.memoize import memoize
 
+def CreateURLsFromPaths(file_system, dirpath, urlprefix):
+  ''' Create a mapping between all of the files in the directory of |dirpath|

[not at google - send to devlin, 2013/07/09 00:17:32]

'''Yields a tuple (url, path) for every file in |directory|, where the URL is
given relative to |urlprefix|.

Can we move this function to a file like file_system_util.py?

[jshumway, 2013/07/17 00:49:55]

Done.

+  and their corrisponding URLs, with the URLs as keys.
+  '''
+  for root, dirs, files in file_system.Walk(dirpath):

[not at google - send to devlin, 2013/07/09 00:17:32]

s/dirs/_/

[jshumway, 2013/07/17 00:49:55]

Done.

+    for f in files:
+      url = posixpath.join(urlprefix, root[len(dirpath) + 1:], f)
+      yield url, '%s/%s' % (root, f)
+
 class _SingletonRenderServletDelegate(RenderServlet.Delegate):
   def __init__(self, server_instance):
     self._server_instance = server_instance
 
   def CreateServerInstanceForChannel(self, channel):
     return self._server_instance
 
 class CronServlet(Servlet):
   '''Servlet which runs a cron job.
   '''
@@ skipping 44 matching lines @@
     server_instance = self._GetSafeServerInstance()
 
     def get_via_render_servlet(path):
       request = Request(path, self._request.host, self._request.headers)
       delegate = _SingletonRenderServletDelegate(server_instance)
       return RenderServlet(request, delegate).Get()
 
     def run_cron_for_dir(d, path_prefix=''):
       success = True
       start_time = time.time()
-      # TODO(jshumway): use server_instance.host_file_system.Walk.
       # TODO(kalman): delete me where it's set.

[not at google - send to devlin, 2013/07/09 00:17:32]

could you move this todo into server_instance on content_cache, doesn't make
sense here anymore.

[jshumway, 2013/07/17 00:49:55]

Done.

-      files = [f for f in server_instance.content_cache.GetFromFileListing(d)
-               if not f.endswith('/') and f != 'redirects.json']
+      files = dict(
+          CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix))
       logging.info('cron/%s: rendering %s files from %s...' % (
           channel, len(files), d))
       try:
-        for i, f in enumerate(files):
+        for i, path in enumerate(files):
           error = None
-          path = '%s%s' % (path_prefix, f)
           try:
             response = get_via_render_servlet(path)
             if response.status != 200:
               error = 'Got %s response' % response.status
           except DeadlineExceededError:
             logging.error(
                 'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % (
                     channel, path, i + 1, len(files), traceback.format_exc()))
             raise
           except error:
@@ skipping 121 matching lines @@
         object_store_creator)
     compiled_host_fs_factory = CompiledFileSystem.Factory(
         host_file_system,
         object_store_creator)
     return ServerInstance(channel,
                           object_store_creator,
                           host_file_system,
                           app_samples_file_system,
                           '' if channel == 'stable' else '/%s' % channel,
                           compiled_host_fs_factory)