Extensions Docs Server: Doc conversion script

chrome/common/extensions/docs/server2/converter.py

Index: chrome/common/extensions/docs/server2/converter.py
diff --git a/chrome/common/extensions/docs/server2/converter.py b/chrome/common/extensions/docs/server2/converter.py
new file mode 100755
index 0000000000000000000000000000000000000000..b048abefc979bb9dcdaba2e8a8986d85515076fe
--- /dev/null
+++ b/chrome/common/extensions/docs/server2/converter.py
@@ -0,0 +1,252 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Example run from the server2/ directory:
+# $ converter.py ../static/ ../../api templates/articles/ templates/intros/
+#   templates/public/ static/images/ -r
+
+import optparse
+import os
+import re
+import shutil
+
+from docs_server_utils import SanitizeAPIName
+
+IGNORED_FILES = [
+  # These are custom files.
+  '404',
+  'api_index',
+  'experimental',
+  'samples',
+  'index',
+  # These are APIs that should not have docs.
+  'test',
+  'experimental_idltest',
+]
+
+# These are mappings for APIs that have no intros. They are needed because the
+# names of the JSON files do not give enough information on the actual API name.

[not at google - send to devlin, 2012/08/01 20:41:04]

seems like most of these you could figure out based on camelcasing the
underscores, with the exception of experimental (programmable).

with two exception, and experimental.input.virtualKeyboard and input.ime.

[cduvall, 2012/08/02 00:54:06]

Done.

+CUSTOM_MAPPINGS = {
+  'experimental_bookmark_manager': 'experimental_bookmarkManager',
+  'experimental_input_virtual_keyboard': 'experimental_input_virtualKeyboard',
+  'experimental_media_galleries': 'experimental_mediaGalleries',
+  'experimental_offscreen_tabs': 'experimental_offscreenTabs',
+  'input_ime': 'input_ime',
+  'file_system': 'fileSystem',
+  'page_actions': 'pageActions',
+  'script_badge': 'scriptBadge'
+}
+
+def _ReadFile(filename):
+  with open(filename, 'r') as f:
+    return f.read()
+
+def _WriteFile(filename, data):
+  with open(filename, 'w+') as f:
+    f.write(data)
+
+def _UnixName(name):
+  """Returns the unix_style name for a given lowerCamelCase string.
+  Shamelessly stolen from json_schema_compiler/model.py.
+  """
+  name = os.path.splitext(name)[0]
+  s1 = re.sub('([a-z])([A-Z])', r'\1_\2', name)
+  s2 = re.sub('([A-Z]+)([A-Z][a-z])', r'\1_\2', s1)
+  return s2.replace('.', '_').lower()
+
+def _ListAllAPIs(dirname):
+  all_files = []
+  for path, dirs, files in os.walk(dirname):
+    if path == '.':
+      all_files.extend([f for f in files
+                        if f.endswith('.json') or f.endswith('.idl')])
+    else:
+      all_files.extend([os.path.join(path, f) for f in files
+                        if f.endswith('.json') or f.endswith('.idl')])
+  dirname = dirname.rstrip('/') + '/'
+  return [f[len(dirname):] for f in all_files
+          if os.path.splitext(f)[0].split('_')[-1] not in ['private',
+                                                           'internal']]
+
+def _MakeArticleTemplate(filename):
+  return '{{+partials.standard_article article:intros.%s}}' % filename
+
+def _MakeAPITemplate(intro_name, api_name, has_intro):
+  if has_intro:
+    return ('{{+partials.standard_api api:apis.%s intro:intros.%s}}' %
+            (api_name, intro_name))
+  else:
+    return '{{+partials.standard_api api:apis.%s}}' % api_name
+
+def _GetAPIPath(name, api_dir):
+  api_files = _ListAllAPIs(api_dir)
+  for filename in api_files:
+    if name == _UnixName(SanitizeAPIName(filename)):
+      return _UnixName(filename)
+
+def _CleanAPIs(source_dir, api_dir, intros_dest, template_dest, exclude):
+  source_files = os.listdir(source_dir)
+  source_files_unix = set(_UnixName(f) for f in source_files)
+  api_files = set(_UnixName(SanitizeAPIName(f)) for f in _ListAllAPIs(api_dir))
+  intros_files = set(_UnixName(f) for f in os.listdir(intros_dest))
+  to_delete = api_files - source_files_unix - set(exclude)
+  for filename in os.listdir(template_dest):
+    no_ext = os.path.splitext(filename)[0]
+    # Check for changes like appWindow -> app.window.
+    if (_UnixName(filename) in source_files_unix - set(exclude) and
+        no_ext not in source_files):
+      os.remove(os.path.join(template_dest, filename))
+    if _UnixName(filename) in to_delete:
+      try:
+        os.remove(os.path.join(intros_dest, filename))
+      except OSError:
+        pass
+      os.remove(os.path.join(template_dest, filename))
+
+def _FormatFile(contents, path, name, image_dest, replace, is_api):
+  # Copy all images referenced in the page.
+  for image in re.findall(r'src="\.\./images/([^"]*)"', contents):
+    if not replace and os.path.exists(os.path.join(image_dest, image)):
+      continue
+    if '/' in image:
+      try:
+        os.makedirs(os.path.join(image_dest, image.rsplit('/', 1)[0]))
+      except:
+        pass
+    shutil.copy(
+        os.path.join(path, os.pardir, 'images', image),
+        os.path.join(image_dest, image))
+  contents = re.sub(r'<!--.*(BEGIN|END).*-->', r'', contents)
+  contents = re.sub(r'\.\./images', r'{{static}}/images', contents)
+  contents = re.sub(r'<div.*id="pageData-showTOC".*>.*</div>', r'', contents)
+  if is_api:
+    contents = re.sub(r'<div.*id="pageData-name".*>.*</div>', r'', contents)
+  else:
+    contents = re.sub(r'<div.*id="pageData-name".*>(.*)</div>',
+                      r'<h1 class="page_title">\1</h1>',
+                      contents)
+  # Remove blank lines.
+  contents = '\n'.join([line for line in contents.split('\n') if line.strip()])
+
+  # Attempt to guess if the page has no title.
+  if '<h1' not in contents and not is_api:
+    title = _UnixName(name)
+    title = ' '.join([part[0].upper() + part[1:] for part in title.split('_')])
+    contents = ('<h1 class="page_title">%s</h1>' % title) + contents
+  return contents
+
+def _MoveAllFiles(source_dir,
+                  api_dir,
+                  articles_dest,
+                  intros_dest,
+                  template_dest,
+                  image_dest,
+                  replace=False,
+                  exclude_dir=None):
+  if exclude_dir is None:
+    exclude_files = []
+  else:
+    exclude_files = [_UnixName(f) for f in os.listdir(exclude_dir)]
+  exclude_files.extend(IGNORED_FILES)
+  api_files = _ListAllAPIs(api_dir)
+  if replace:
+    _CleanAPIs(source_dir, api_dir, intros_dest, template_dest, exclude_files)
+  files = set(os.listdir(source_dir))
+  unix_files = [_UnixName(f) for f in files]
+  for name in  [SanitizeAPIName(f)  for f in _ListAllAPIs(api_dir)]:
+    if _UnixName(name) not in unix_files:
+      files.add(name + '.html')
+  for file_ in files:
+    if (_UnixName(file_) in exclude_files or
+        file_.startswith('.') or
+        file_.startswith('_')):
+      continue
+    _MoveSingleFile(source_dir,
+                    file_,
+                    api_dir,
+                    articles_dest,
+                    intros_dest,
+                    template_dest,
+                    image_dest,
+                    replace)
+
+def _MoveSingleFile(source_dir,
+                    source_file,
+                    api_dir,
+                    articles_dest,
+                    intros_dest,
+                    template_dest,
+                    image_dest,
+                    replace=False):
+  unix_name = _UnixName(source_file)
+  is_api = unix_name in [_UnixName(SanitizeAPIName(f))
+                         for f in _ListAllAPIs(api_dir)]
+  if unix_name in CUSTOM_MAPPINGS:
+    processed_name = CUSTOM_MAPPINGS[unix_name]
+  else:
+    processed_name = os.path.splitext(source_file)[0].replace('.', '_')
+    if (is_api and
+        '_' in processed_name.replace('experimental_', '') and
+        not os.path.exists(os.path.join(source_dir, source_file))):
+      print ('*******************************************************\n'
+             'NOTICE: No new-style doc generated for %s.\n\n'
+             'Please add a custom mapping for %s to the |CUSTOM_MAPPINGS|'
+             ' section in:\n'
+             'chrome/common/extensions/docs/server2/converter.py\n'
+             '*******************************************************' %
+             (source_file, source_file))
+      return
+  try:
+    static_data = _FormatFile(_ReadFile(os.path.join(source_dir, source_file)),
+                              source_dir,
+                              source_file,
+                              image_dest,
+                              replace,
+                              is_api)
+  except IOError:
+    static_data = None
+  template_file = os.path.join(template_dest, processed_name + '.html')
+  if is_api:
+    template_data = _MakeAPITemplate(processed_name,
+                                     _GetAPIPath(unix_name, api_dir),
+                                     static_data is not None)
+    static_file = os.path.join(intros_dest, processed_name + '.html')
+  else:
+    template_data = _MakeArticleTemplate(unix_name)
+    static_file = os.path.join(articles_dest, processed_name + '.html')
+  if replace or not os.path.exists(template_file):
+    _WriteFile(template_file, template_data)
+  if static_data is not None and (replace or not os.path.exists(static_file)):
+    _WriteFile(static_file, static_data)
+
+if __name__ == '__main__':
+  parser = optparse.OptionParser(
+      description='Converts static files from the old documentation system to '
+                  'the new one. If run without -f, all the files in |src| will '
+                  'be converted.',
+      usage='usage: %prog [options] static_src_dir [-f static_src_file] '
+            'api_dir articles_dest intros_dest template_dest image_dest')
+  parser.add_option('-f',
+                    '--file',
+                    action='store_true',
+                    default=False,
+                    help='convert single file')
+  parser.add_option('-e',
+                    '--exclude',
+                    default=None,
+                    help='exclude files matching the names in this dir')
+  parser.add_option('-r',
+                    '--replace',
+                    action='store_true',
+                    default=False,
+                    help='replace existing files')
+  (opts, args) = parser.parse_args()
+  if (not opts.file and len(args) != 6) or (opts.file and len(args) != 7):
+    parser.error('incorrect number of arguments.')
+
+  if opts.file:
+    _MoveSingleFile(*args, replace=opts.replace)
+  else:
+    _MoveAllFiles(*args, replace=opts.replace, exclude_dir=opts.exclude)