Doc server broken link detection

chrome/common/extensions/docs/server2/integration_test.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Run build_server so that files needed by tests are copied to the local
 # third_party directory.
 import build_server
 build_server.main()
 
-import logging
 import optparse
 import os
 import sys
 import time
 import unittest
 
+from link_error_detector import LinkErrorDetector, pprint_broken_links
+from local_file_system import LocalFileSystem
 from local_renderer import LocalRenderer
 from fake_fetchers import ConfigureFakeFetchers
 from handler import Handler
 from servlet import Request
 from test_util import EnableLogging, DisableLogging
 
 # Arguments set up if __main__ specifies them.
 _EXPLICIT_TEST_FILES = None
 
 def _ToPosixPath(os_path):
@@ skipping 26 matching lines @@
 
     print('Running cron...')
     start_time = time.time()
     try:
       response = Handler(Request.ForTest('/_cron/stable')).Get()
       self.assertEqual(200, response.status)
       self.assertEqual('Success', response.content.ToString())
     finally:
       print('Took %s seconds' % (time.time() - start_time))
 
+    print "checking for broken links..."

[not at google - send to devlin, 2013/07/09 00:17:32]

print(..) not print ..., print without () is deprecated.

also Checking not checking.

[jshumway, 2013/07/17 00:49:55]

Done.

+    def renderer(path):
+      return Handler(Request.ForTest(path)).Get()
+
+    start_time = time.time()
+    link_error_detector = LinkErrorDetector(
+        LocalFileSystem(os.path.join(sys.path[0], os.pardir, os.pardir)),
+        renderer,

[not at google - send to devlin, 2013/07/09 00:17:32]

maybe make this a lambda?

[jshumway, 2013/07/17 00:49:55]

Done.

+        'templates/public',
+        ('extensions/index.html', 'apps/about_apps.html'))
+
+    broken_links = link_error_detector.GetBrokenLinks()
+    if broken_links:
+      pprint_broken_links(broken_links)
+
+    print 'finding %d broken links took %s seconds' % (

[not at google - send to devlin, 2013/07/09 00:17:32]

likewise, and here Took %s seconds.

[jshumway, 2013/07/17 00:49:55]

Done.

+        len(broken_links), time.time() - start_time)
+
+    print 'searching for orphan pages...'

[not at google - send to devlin, 2013/07/09 00:17:32]

likewise here and below.

[jshumway, 2013/07/17 00:49:55]

Done.

+    start_time = time.time()
+    orphans = link_error_detector.GetOrphanPages()
+    for page in orphans:
+      print page

[not at google - send to devlin, 2013/07/09 00:17:32]

I think we can do a better job of being informative here.

[jshumway, 2013/07/17 00:49:55]

Done.

+    print 'finding %d orphan pages took %s seconds' % (
+        len(orphans), time.time() - start_time)
+
     public_files = _GetPublicFiles()
 
     print('Rendering %s public files...' % len(public_files.keys()))
     start_time = time.time()
     try:
       for path, content in public_files.iteritems():
         if path.endswith('redirects.json'):
           continue
         def check_result(response):
           self.assertEqual(200, response.status,
@@ skipping 20 matching lines @@
     if _EXPLICIT_TEST_FILES is None:
       return
     for filename in _EXPLICIT_TEST_FILES:
       print('Rendering %s...' % filename)
       start_time = time.time()
       try:
         response = LocalRenderer.Render(_ToPosixPath(filename))
         self.assertEqual(200, response.status)
         self.assertTrue(response.content != '')
       finally:
         print('Took %s seconds' % (time.time() - start_time))

[not at google - send to devlin, 2013/07/09 00:17:32]

like I mentioned - can we make the test run here too, but just for that one
file?

(provided it doesn't take too long)

[jshumway, 2013/07/17 00:49:55]

I built out this functionality then discovered that LocalRenderer was too slow
(on the magnitude of a minute) to test a page dense with links. Thus a full cron
run would be required, which would bring the time down to 20 seconds on a good
computer which is still too slow. As we decided in IRC, I removed the check.

The error detection code is still structured in a way that it is easy (4-5
lines) to add a function that checks a single file back in, if in the future
everything is faster, or if we want to run the checks server side.

 
   @DisableLogging('warning')
   def testFileNotFound(self):
     response = Handler(Request.ForTest('/extensions/notfound.html')).Get()
     self.assertEqual(404, response.status)
 
 if __name__ == '__main__':
   parser = optparse.OptionParser()
   parser.add_option('-a', '--all', action='store_true', default=False)
   (opts, args) = parser.parse_args()
   if not opts.all:
     _EXPLICIT_TEST_FILES = args
   # Kill sys.argv because we have our own flags.
   sys.argv = [sys.argv[0]]
   unittest.main()