Devsite doc generation tweaks following Andy's requests.

native_client_sdk/src/doc/_sphinxext/devsite_builder.py

Index: native_client_sdk/src/doc/_sphinxext/devsite_builder.py
diff --git a/native_client_sdk/src/doc/_sphinxext/devsite_builder.py b/native_client_sdk/src/doc/_sphinxext/devsite_builder.py
index df2c1d2b4feac411a696beb2303c48712ca8d88c..192534e0e294c9c31f4b97b24f98478ae35d82c4 100644
--- a/native_client_sdk/src/doc/_sphinxext/devsite_builder.py
+++ b/native_client_sdk/src/doc/_sphinxext/devsite_builder.py
@@ -48,7 +48,7 @@ ${doc_body}
 '''.lstrip())
 
 DEVSITE_PREFIX = r'''
-{% setvar pepperversion %}pepper''' + PEPPER_VERSION + ''' {% endsetvar %}
+{% setvar pepperversion %}pepper''' + PEPPER_VERSION + '''{% endsetvar %}
 {% include "native-client/_local_variables.html" %}'''
 
 DEVSITE_BUTTERBAR = '{{butterbar}}'
@@ -253,8 +253,18 @@ class DevsiteBuilder(StandaloneHTMLBuilder):
 
   def finish(self):
     super(DevsiteBuilder, self).finish()
-    self.info(bold('generating YAML table-of-contents files... '))
-    create_toc_yaml_files(self.app)
+    if self.devsite_production_mode:
+      # We decided to keep the manual _book.yaml for now;
+      # The code for auto-generating YAML TOCs from index.rst was removed in
+      # https://codereview.chromium.org/57923006/
+      self.info(bold('generating YAML table-of-contents... '))
+      subs = {
+        'version': PEPPER_VERSION,
+        'folder': self.config.devsite_foldername or ''}
+      with open(os.path.join(self.env.srcdir, '_book.yaml')) as in_f:
+        with open(os.path.join(self.outdir, '_book.yaml'), 'w') as out_f:
+          out_f.write(string.Template(in_f.read()).substitute(subs))
+    self.info()
 
   def dump_inventory(self):
     # We don't want an inventory file when building for devsite
@@ -343,175 +353,6 @@ class NaclCodeDirective(Directive):
     literal['prettyprint'] = self.options.get('prettyprint', 1)
     return [literal]
 
-# RstFileInfo and RstDirInfo are types that are used to represent a hierarchical
-# tree of directories and files. What ends up being created is something like
-# the following:
-#
-# { 'overview.rst':     RstFileInfo('overview.rst', 'Overview'),
-#   'quick-start.rst':  RstFileInfo('quick-start.rst', 'Quick Start'),
-#   'sdk':              RstDirInfo('sdk', 'SDK', {
-#                         'index.rst': RstFileInfo('sdk/index.rst', 'SDK'),
-#                         ...}),
-#   'devguide':
-#     RstDirInfo('devguide', "Developer's Guide", {
-#       'index.rst':  RstFileInfo('devguide/index.rst', 'Devguide index'),
-#       'tutorial':   RstDirInfo('devguide/tutorial', 'Tutorial', {
-#         ...}))
-# }
-#
-# Note: the containers are actually OrderedDicts.
-RstFileInfo = namedtuple('RstFileInfo', 'path title')
-RstDirInfo = namedtuple('RstDirInfo', 'path title items')
-
-def create_toc_yaml_files(app):
-  rst_tree = parse_rst_index_tree(app)
-  generate_yaml_tree(app, rst_tree, rootpath=None, title=None)
-
-def dump_rst_tree(tree, indent=0):
-  """ tree: a sequence of RstDirInfo/RstFileInfo objects.
-  """
-  for path, item in tree.items():
-    if isinstance(item, RstFileInfo):
-      print(' ' * indent, item.path)
-    else:
-      print(' ' * indent, 'Dumping subtree', item.path)
-      dump_rst_tree(item.items, indent + 4)
-
-def generate_yaml_tree(app, rst_tree, rootpath=None, title=None):
-  """Generate YAML table-of-contents files in the build directory.
-
-  Arguments:
-
-    app: Sphinx app object
-    rst_tree: the RST tree from parse_rst_index_tree
-
-  The following two arguments should be left default in the top-level call;
-  they are used for recursive calls.
-
-    rootpath: path within the full rst_tree for this tree.
-    title: title for the current tree.
-  """
-  # Template substitutions for the generated YAML data.
-  substitutions = {
-    'version': PEPPER_VERSION,
-    'folder': ''}
-  if app.config.devsite_foldername:
-    substitutions['folder'] = app.config.devsite_foldername + '/'
-
-  # toclines collects a list of lines to be generated into the YAML file.
-  # The logic here handles two cases: top-level table of contents (goes into
-  # top-level _book.yaml) and per-directory _toc.yaml files.
-  toclines = []
-  if rootpath is None:
-    # Top-level TOC.
-    out_path = os.path.join(app.outdir, '_book.yaml')
-    template_path = os.path.join(app.srcdir, '_book_template.yaml')
-    indent = 0
-  else:
-    out_path = os.path.join(app.outdir, rootpath, '_toc.yaml')
-    template_path = None
-    toclines.append('- title: "%s"' % title)
-    toclines.append('  path: /native-client/${folder}%s' % rootpath)
-    if rst_tree:
-      toclines.append('  section:')
-    indent = 4
-
-  def add_tocline(s):
-    toclines.append((' ' * indent) + s)
-
-  # A list of discovered subtrees that need to be processed after we're done
-  # with this one.
-  subtrees = []
-
-  for name, item in rst_tree.items():
-    if isinstance(item, RstFileInfo):
-      add_tocline('- title: "%s"' % item.title)
-      add_tocline('  path: /native-client/${folder}%s' % (
-        os.path.splitext(item.path)[0]))
-    else:
-      assert isinstance(item, RstDirInfo)
-      add_tocline('- include: /native-client/${folder}%s/_toc.yaml' % item.path)
-      subtrees.append(item)
-
-  # ensure a \n in the end
-  toclines.append('')
-
-  if template_path is not None:
-    toc = '\n'.join(toclines)
-    # When we have a template file, read the template and substitute the
-    # contents in the '${toc}' variable (and then apply substitutions on that
-    # recursively...)
-    with open(template_path) as tfile:
-      toc = string.Template(tfile.read()).substitute(toc=toc, **substitutions)
-      toc = string.Template(toc).substitute(substitutions)
-  else:
-    toc = '\n'.join(['toc:'] + toclines)
-    toc = string.Template(toc).substitute(substitutions)
-
-  with open(out_path, 'w') as outfile:
-    outfile.write(toc)
-  app.info('Wrote %s' % out_path)
-
-  for subtree in subtrees:
-    # Call self recursively on subtrees.
-    generate_yaml_tree(app, subtree.items, rootpath=subtree.path,
-                       title=subtree.title)
-
-def parse_rst_index_tree(app):
-  """Parse the top-level index.rst and build a tree of RST files and dirs.
-
-  Return the top-level tree, which is an OrderedDict. See the comment above
-  RstFileInfo for more details.
-  """
-  # Parse index.rst to create a list of all known .rst files - their paths
-  # within the source tree and the titles of the documents they contain. The
-  # result is rst_files - an ordered dict mapping path->title.
-  with open(os.path.join(app.srcdir, 'index.rst')) as f:
-    paths = [line.strip() for line in f if line.strip().endswith('.rst')]
-  rst_files = OrderedDict()
-  for path in paths:
-    path_noext = os.path.splitext(path)[0]
-    if path_noext in app.env.titles:
-      rst_files[path] = app.env.titles[path_noext].astext()
-    else:
-      app.warn('unable to get title for path %s' % path)
-      continue
-  # rst_files represent a file/dir hierarchy which has to be unraveled into a
-  # hierarchical data structure.
-  rst_tree = OrderedDict()
-  for path in rst_files:
-    path_components = path.split('/')
-    cur_dir_itemlist = rst_tree
-    # Figure out the actual tree node this file has to be attached to (creating
-    # the nodes on-demand if needed). Note that when we see a file, the
-    # RstDirInfo representing the directory it's in may not exist yet (and the
-    # same for the parent directories...). This loop makes sure that the whole
-    # hierarchy gets created if it doesn't already exist.
-    # Throughout the loop, cur_dir_itemlist points to the OrderedDict that
-    # represents the directory containing this path.
-    for i in range(len(path_components) - 1):
-      dir = path_components[i]
-      if dir not in cur_dir_itemlist:
-        newpath = os.path.join(*path_components[:i+1])
-        # We assume every dir has an index.rst, the title of which is used as
-        # the title of the dir.
-        newtitle = rst_files[os.path.join(newpath, 'index.rst')]
-        newitems = OrderedDict()
-        cur_dir_itemlist[dir] = RstDirInfo(path=newpath, title=newtitle,
-                                           items=newitems)
-        cur_dir_itemlist = newitems
-      else:
-        cur_dir_itemlist = cur_dir_itemlist[dir].items
-    filename = path_components[-1]
-    if filename != 'index.rst':
-      # Now that the path through the tree is blazed through, we can finally
-      # add the file.
-      # But don't add index.rst - because the directory is already described
-      # by a title.
-      cur_dir_itemlist[path_components[-1]] = RstFileInfo(
-          path=filename, title=rst_files[path])
-  return rst_tree
-
 def setup(app):
   """ Extension registration hook.
   """