Doc server broken link detection

chrome/common/extensions/docs/server2/integration_test.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Run build_server so that files needed by tests are copied to the local
 # third_party directory.
 import build_server
 build_server.main()
 
-import logging
+from itertools import groupby
+from operator import itemgetter
 import optparse
 import os
 import sys
 import time
 import unittest
 
+from link_error_detector import LinkErrorDetector
+from local_file_system import LocalFileSystem
 from local_renderer import LocalRenderer
 from fake_fetchers import ConfigureFakeFetchers
 from handler import Handler
 from servlet import Request
 from test_util import EnableLogging, DisableLogging
 
 # Arguments set up if __main__ specifies them.
 _EXPLICIT_TEST_FILES = None
 
 def _ToPosixPath(os_path):
   return os_path.replace(os.sep, '/')
 
 def _GetPublicFiles():
   '''Gets all public files mapped to their contents.
   '''
   public_path = os.path.join(sys.path[0], os.pardir, 'templates', 'public')
   public_files = {}
   for path, dirs, files in os.walk(public_path, topdown=True):
     dirs[:] = [d for d in dirs if d != '.svn']
     relative_posix_path = _ToPosixPath(path[len(public_path):])
     for filename in files:
       with open(os.path.join(path, filename), 'r') as f:
         public_files['/'.join((relative_posix_path, filename))] = f.read()
   return public_files
 
+def _PrintBrokenLinks(broken_links):
+  '''Prints out broken links in a more readable format.
+  '''
+  col_width = max([len(l[0]) for l in broken_links])

[not at google - send to devlin, 2013/07/17 20:19:40]

the square brackets aren't needed (just creates redundant list).

max(len(link[0]) for link in broken_links)

and yeah, never use l as a variable name...

[jshumway, 2013/07/17 22:15:47]

Done.

+  getter = itemgetter(1)
+
+  for target, links in groupby(sorted(broken_links, key=getter), getter):
+    links = [l[0] for l in links]
+    if len(links) > 50:
+      out = "%s and %d others" % (links[0], len(links) - 1)
+      print("%s%s -> %s" % (out, (col_width - len(out)) * ' ', target))
+    else:
+      for link in links:
+        print("%s%s -> %s" % (link, (col_width - len(link)) * ' ', target))

[not at google - send to devlin, 2013/07/17 20:19:40]

can you pull this print logic into a function? Like

pretty_print(out, target)
vs
pretty_print(link, target)

with pretty_print defined under col_width like

def pretty_print(prefix, message):
  print('%s%s - %s' % (prefix, len(prefix), message)

I think that's right...

[jshumway, 2013/07/17 22:15:47]

Done.

+
 class IntegrationTest(unittest.TestCase):
   def setUp(self):
     ConfigureFakeFetchers()
 
   @EnableLogging('info')
   def testCronAndPublicFiles(self):
     '''Runs cron then requests every public file. Cron needs to be run first
     because the public file requests are offline.
     '''
     if _EXPLICIT_TEST_FILES is not None:
       return
 
     print('Running cron...')
     start_time = time.time()
     try:
       response = Handler(Request.ForTest('/_cron/stable')).Get()
       self.assertEqual(200, response.status)
       self.assertEqual('Success', response.content.ToString())
     finally:
       print('Took %s seconds' % (time.time() - start_time))
 
+    # Check for broken links.

[not at google - send to devlin, 2013/07/17 20:19:40]

comment is redundant given the next line

[jshumway, 2013/07/17 22:15:47]

Done.

+    print("Checking for broken links...")
+    start_time = time.time()
+    link_error_detector = LinkErrorDetector(
+        LocalFileSystem(os.path.join(sys.path[0], os.pardir, os.pardir)),
+        lambda path: Handler(Request.ForTest(path)).Get(),
+        'templates/public',
+        ('extensions/index.html', 'apps/about_apps.html'))
+
+    broken_links, broken_anchors = link_error_detector.GetBrokenLinks()
+    if broken_links or broken_anchors:
+      print('[%d broken links] page of occurrence\t-> href of broken link' % (
+          len(broken_links) + len(broken_anchors)))

[not at google - send to devlin, 2013/07/17 20:19:40]

logging will look a little bit awkward since alignment depends on tab width.

Let's just say:

print('Found %s broken links:' % len(...))
_PrintBrokenLinks(broken_links + broken_anchors)

or maybe the + isn't possible? I can't see what the effect of breaking this up
is...

[jshumway, 2013/07/17 22:15:47]

Done.

+      if broken_links:
+        _PrintBrokenLinks(broken_links)
+      if broken_anchors:
+        _PrintBrokenLinks(broken_anchors)

[not at google - send to devlin, 2013/07/17 20:19:40]

make sure you actually fail the test at this point or chances are nobody will
notice.

[jshumway, 2013/07/17 22:15:47]

Prints a warning.

+
+    print('Took %s seconds.' % (time.time() - start_time))
+
+    # Check for orphaned pages.

[not at google - send to devlin, 2013/07/17 20:19:40]

comment also redundant

[jshumway, 2013/07/17 22:15:47]

Done.

+    print('Searching for orphaned pages...')
+    start_time = time.time()
+    orphaned_pages = link_error_detector.GetOrphanedPages()
+    if orphaned_pages:
+      print('[%d orphaned pages]' % len(orphaned_pages))

[not at google - send to devlin, 2013/07/17 20:19:40]

Found %d orphaned pages:

[jshumway, 2013/07/17 22:15:47]

Done.

+      for page in orphaned_pages:
+        print(page)
+    print('Took %s seconds.' % (time.time() - start_time))

[not at google - send to devlin, 2013/07/17 20:19:40]

likewise, fail the test (self.fail maybe?)

[jshumway, 2013/07/17 22:15:47]

Prints a warning.

+
     public_files = _GetPublicFiles()
 
     print('Rendering %s public files...' % len(public_files.keys()))
     start_time = time.time()
     try:
       for path, content in public_files.iteritems():
         if path.endswith('redirects.json'):
           continue
         def check_result(response):
           self.assertEqual(200, response.status,
@@ skipping 22 matching lines @@
     for filename in _EXPLICIT_TEST_FILES:
       print('Rendering %s...' % filename)
       start_time = time.time()
       try:
         response = LocalRenderer.Render(_ToPosixPath(filename))
         self.assertEqual(200, response.status)
         self.assertTrue(response.content != '')
       finally:
         print('Took %s seconds' % (time.time() - start_time))
 
+    # TODO: Check page for broken links (currently prohibited by the time it

[not at google - send to devlin, 2013/07/17 20:19:40]

always name TODOs. so TODO(jshumway) here.

[jshumway, 2013/07/17 22:15:47]

Done.

+    # takes to render the pages).
+
   @DisableLogging('warning')
   def testFileNotFound(self):
     response = Handler(Request.ForTest('/extensions/notfound.html')).Get()
     self.assertEqual(404, response.status)
 
 if __name__ == '__main__':
   parser = optparse.OptionParser()
   parser.add_option('-a', '--all', action='store_true', default=False)
   (opts, args) = parser.parse_args()
   if not opts.all:
     _EXPLICIT_TEST_FILES = args
   # Kill sys.argv because we have our own flags.
   sys.argv = [sys.argv[0]]
   unittest.main()