Doc server broken link detection

chrome/common/extensions/docs/server2/cron_servlet.py

Index: chrome/common/extensions/docs/server2/cron_servlet.py
diff --git a/chrome/common/extensions/docs/server2/cron_servlet.py b/chrome/common/extensions/docs/server2/cron_servlet.py
index 2e7c62029948dc928bd37cc9e67389764e301ecf..bc169ab8eef393c32e50483e9410bb25fbeee83e 100644
--- a/chrome/common/extensions/docs/server2/cron_servlet.py
+++ b/chrome/common/extensions/docs/server2/cron_servlet.py
@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import logging
+import posixpath
 import time
 import traceback
 
@@ -22,6 +23,15 @@ from subversion_file_system import SubversionFileSystem
 import svn_constants
 from third_party.json_schema_compiler.memoize import memoize
 
+def CreateURLsFromPaths(file_system, dirpath, urlprefix):
+  ''' Create a mapping between all of the files in the directory of |dirpath|

[not at google - send to devlin, 2013/07/09 00:17:32]

'''Yields a tuple (url, path) for every file in |directory|, where the URL is
given relative to |urlprefix|.

Can we move this function to a file like file_system_util.py?

[jshumway, 2013/07/17 00:49:55]

Done.

+  and their corrisponding URLs, with the URLs as keys.
+  '''
+  for root, dirs, files in file_system.Walk(dirpath):

[not at google - send to devlin, 2013/07/09 00:17:32]

s/dirs/_/

[jshumway, 2013/07/17 00:49:55]

Done.

+    for f in files:
+      url = posixpath.join(urlprefix, root[len(dirpath) + 1:], f)
+      yield url, '%s/%s' % (root, f)
+
 class _SingletonRenderServletDelegate(RenderServlet.Delegate):
   def __init__(self, server_instance):
     self._server_instance = server_instance
@@ -86,16 +96,14 @@ class CronServlet(Servlet):
     def run_cron_for_dir(d, path_prefix=''):
       success = True
       start_time = time.time()
-      # TODO(jshumway): use server_instance.host_file_system.Walk.
       # TODO(kalman): delete me where it's set.

[not at google - send to devlin, 2013/07/09 00:17:32]

could you move this todo into server_instance on content_cache, doesn't make
sense here anymore.

[jshumway, 2013/07/17 00:49:55]

Done.

-      files = [f for f in server_instance.content_cache.GetFromFileListing(d)
-               if not f.endswith('/') and f != 'redirects.json']
+      files = dict(
+          CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix))
       logging.info('cron/%s: rendering %s files from %s...' % (
           channel, len(files), d))
       try:
-        for i, f in enumerate(files):
+        for i, path in enumerate(files):
           error = None
-          path = '%s%s' % (path_prefix, f)
           try:
             response = get_via_render_servlet(path)
             if response.status != 200: