HTML human writable docs working end to end!... mostly...

tools/html_json_doc/lib/html_to_json.dart

Index: tools/html_json_doc/lib/html_to_json.dart
diff --git a/tools/html_json_doc/lib/html_to_json.dart b/tools/html_json_doc/lib/html_to_json.dart
new file mode 100644
index 0000000000000000000000000000000000000000..ddfb5eb2ef2a6fc7c7b6b10a63464e56ee489b5a
--- /dev/null
+++ b/tools/html_json_doc/lib/html_to_json.dart
@@ -0,0 +1,314 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/**
+ * Library for extracting the documentation comments from files generated by
+ * the HTML library. The comments are stored in a JSON file.
+ *
+ * Comments must be in either the block style with leading *s:
+ *
+ *     /**
+ *      * Comment here.
+ *      */
+ *
+ * Or the triple-slash style:
+ *
+ *     /// Docs go here.
+ *     /// And here.
+ *
+ * Each member that is to be documented should be preceeded by a meta-comment
+ * containing the string `@docsEditable` such as:
+ *
+ *     /// @docsEditable
+ */
+library html_to_json;
+
+import 'dart:json';
+import 'dart:io';
+
+
+/// True if any errors were triggered through the conversion.
+bool _anyErrors = false;
+
+
+/**
+ * Convert files on [htmlPath] and write JSON to [jsonPath].
+ */
+Future<bool> convert(Path htmlPath, Path jsonPath) {
+  var completer = new Completer();
+  
+  // TODO(amouravski): make this transform once I know what I want this file to
+  // return.
+  _convertFiles(htmlPath).then((convertedJson) {
+    final jsonFile = new File.fromPath(jsonPath);
+    var writeJson = convertedJson;
+
+    if (jsonFile.existsSync()) {
+      writeJson = _mergeJsonAndFile(convertedJson, jsonFile);
+    }
+
+    var outputStream = jsonFile.openOutputStream();
+    outputStream.writeString(prettyPrintJson(writeJson));
+
+    outputStream.onNoPendingWrites = () {
+      completer.complete(_anyErrors);
+    };
+
+    outputStream.onClosed = () {
+      completer.complete(_anyErrors);
+    };
+    
+    outputStream.onError = completer.completeException;
+  });
+
+  return completer.future;
+}
+
+
+/**
+ * Convert all files on [htmlPath].
+ *
+ * Returns a future that completes to the converted JSON object.
+ */
+Future<Object> _convertFiles(Path htmlPath) {
+  var completer = new Completer();
+
+  List<Future> fileFutures = [];
+
+  // Get a list of all HTML dart files.
+  // TODO(amouravski): discriminate .dart files.
+  final htmlDir = new Directory.fromPath(htmlPath);
+  final lister = htmlDir.list(recursive: false);
+
+  lister.onFile = (String path) {
+    final name = new Path.fromNative(path).filename;
+
+    // Ignore private classes.
+    if (name.startsWith('_')) return;
+
+    // Ignore non-dart files.
+    if (!name.endsWith('.dart')) return;
+
+    File file = new File(path);
+
+    // TODO(amouravski): Handle missing file.
+    if (!file.existsSync()) {
+      print('ERROR: cannot find file $path');
+      _anyErrors = true;
+      return;
+    }
+
+    fileFutures.add(_convertFile(file));
+  };
+
+
+  // Combine all JSON objects
+  lister.onDone = (_) {
+    Futures.wait(fileFutures).then((jsonList) {
+      var convertedJson = {};
+      jsonList.forEach((json) {
+        final k = json.keys[0];
+        convertedJson.putIfAbsent(k, () => json[k]);
+      });
+      completer.complete(convertedJson);
+    });
+  };
+
+  // TODO(amouravski): add more error handling.
+
+  return completer.future;
+}
+
+
+/**
+ * Convert a single file to JSON docs.
+ *
+ * Returns a map with one entry whose key is the file name and whose value is
+ * the list of comment lines.
+ */
+Future<Map> _convertFile(File file) {
+  var completer = new Completer();
+
+  var comments = {};
+
+  // Find all /// @docsEditable annotations.
+  InputStream file_stream = file.openInputStream();
+  StringInputStream inputLines = new StringInputStream(file_stream);
+
+  // TODO(amouravski): Re-write as file.readAsLine().thin((lines) {...}
+  inputLines.onLine = () {
+    var comment = <String>[];
+
+    var docCommentFound = false;
+    String line;
+    while ((line = inputLines.readLine()) != null) {
+      var trimmedLine = line.trim();
+
+      // Sentinel found. Process the comment block.
+      if (trimmedLine.startsWith('///') &&
+          trimmedLine.contains('@docsEditable')) {
+        if (docCommentFound == true) {
+          var nextLine = inputLines.readLine();
+
+          if (nextLine == null) return false;
+
+          var lineObject = {};
+
+          if (comments[nextLine] != null) {
+            print('WARNING: duplicate line ${nextLine} found in'
+                '${new Path(file.fullPathSync()).filename}');
+          }
+          comments.putIfAbsent(nextLine, () => comment);
+        }
+
+        // Reset.
+        docCommentFound = false;
+        comment = <String>[];
+      } else if ( // Start a comment block.
+          trimmedLine.startsWith('/**') ||
+          trimmedLine.startsWith('///')) {
+        docCommentFound = true;
+        comment.add(line);
+      } else if (docCommentFound &&
+          // TODO(amouravski): This will barf on:
+          // /// blah
+          // *
+          (trimmedLine.startsWith('*') || trimmedLine.startsWith('///'))) {
+        comment.add(line);
+      } else {
+        // Reset if we're not in a comment.
+        docCommentFound = false;
+        comment = <String>[];
+      }
+    }
+  };
+
+  inputLines.onClosed = () {
+    var jsonObject = {};
+    jsonObject[new Path(file.fullPathSync()).filename] = comments;
+    completer.complete(jsonObject);
+  };
+
+  // TODO(amouravski): better error handling.
+
+  return completer.future;
+}
+
+
+/**
+ * Merge the new JSON object and the existing file.
+ */
+Object _mergeJsonAndFile(Object json, File file) {
+  var completer = new Completer();
+
+  var fileJson = {};
+  var jsonRead = file.readAsStringSync();
+
+  if (jsonRead == '') {
+    print('WARNING: no data read from '
+        '${new Path(file.fullPathSync()).filename}');
+    _anyErrors = true;
+  } else {
+    fileJson = JSON.parse(jsonRead);
+  }
+  return _mergeJson(json, fileJson);
+}
+
+
+/**
+ * Merge two JSON objects, such that the returned JSON object is the
+ * union of both.
+ *
+ * Each JSON must be a map, with each value being a map.
+ */
+Object _mergeJson(Object json1, Object json2) {
+  if (json1 is Map && json2 is Map) {
+    // Then check if [json2] contains any key form [json1], in which case
+    // add all of the values from [json2] to the values of [json1].
+    json2.forEach((k, v) {
+      if (json1.containsKey(k)) {
+        v.forEach((vk, vv) {
+          if (json1[k].containsKey(vk) &&
+              !_listsEqual(json1[k][vk],vv)) {
+            // Assume that json1 is more current and take its data as opposed
+            // to json2's.
+            // TODO(amouravski): add better warning message and only if there's
+            // a conflict.
+            print('INFO: duplicate keys.');
+            _anyErrors = false;
+          } else {
+            json1[k].putIfAbsent(vk, () => vv);
+          }
+        });
+      } else {
+        json1.putIfAbsent(k, () => v);
+      }
+    });
+  } else {
+    throw new ArgumentError('JSON objects must both be Maps');
+  }
+
+  // TODO(amouravski): more error handling.
+
+  return json1;
+}
+
+
+/**
+ * Tests for equality between two lists.
+ *
+ * This checks the first level of depth, so does not work for nested lists.
+ */
+bool _listsEqual(List list1, List list2) {
+  return list1.every((e) => list2.contains(e)) &&
+      list2.every((e) => list1.contains(e));
+}
+
+
+/**
+ * Print JSON in a much nicer format.
+ *
+ * For example:
+ * 
+ *     {"foo":["bar","baz"],"boo":{"far:"faz"}}
+ *
+ * becomes:
+ *
+ *     {
+ *       "foo":
+ *         [
+ *           "bar",
+ *           "baz"
+ *         ],
+ *       "boo":
+ *         {
+ *           "far":
+ *             "faz"
+ *         }
+ *     }
+ */
+String prettyPrintJson(Object json, [String indentation = '']) {
+  var output;
+
+  if (json is List) {
+    var recursiveOutput =
+        Strings.join(json.map((e) => 
+            prettyPrintJson(e, '$indentation  ')), ',\n');
+    output = '$indentation[\n'
+      '$recursiveOutput'
+      '\n$indentation]';
+  } else if (json is Map) {
+    // TODO(amouravski): No newline after :
+    var mapList = json.keys.map((key) =>
+        '$indentation${JSON.stringify(key)}:\n'
+        '${prettyPrintJson(json[key], '$indentation  ')}');
+    var recursiveOutput = Strings.join(mapList, ',\n');
+    output = '$indentation{\n'
+      '$recursiveOutput'
+      '\n$indentation}';
+  } else {
+    output = '$indentation${JSON.stringify(json)}';
+  }
+  return output;
+}