HTML human writable docs working end to end!... mostly...

tools/html-json-doc/lib/HtmlToJson.dart

+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/**
+ * Library for extracting the documentation comments from files generated by
+ * the HTML library. The comments are stored in a JSON file.
+ *
+ * Comments must be in either the block style with leading *s:
+ *
+ *     /**
+ *      * Comment here.
+ *      */
+ *
+ * Or the triple-slash style:
+ *
+ *     /// Docs go here.
+ *     /// And here.
+ *
+ * Each member that is to be documented should be preceeded by a meta-comment
+ * containing the string `@docsEditable` such as:
+ *
+ *     /// @docsEditable
+ */
+library htmlToJson;

[blois, 2012/11/26 21:50:44]

library names should be html_to_json.

[Andrei Mouravski, 2012/11/26 23:24:11]

Done.

+
+import 'dart:json';
+import 'dart:io';
+
+
+/// True if any errors were triggered through the conversion.
+bool _anyErrors = false;
+
+
+/**
+ * Convert files on [htmlPath] and write JSON to [jsonPath].
+ */
+Future<bool> convert(Path htmlPath, Path jsonPath) {
+  var completer = new Completer();
+
+  var futureConvert = _convertFiles(htmlPath);
+
+  // TODO(amouravski): make this transform once I know what I want this file to
+  // return.
+  futureConvert.then((convertedJson) {
+    final jsonFile = new File.fromPath(jsonPath);
+    var writeJson = convertedJson;
+
+    if (jsonFile.existsSync()) {
+      var writeJson = _mergeJsonAndFile(convertedJson, jsonFile);
+    } else {
+      jsonFile.createSync();
+    }
+
+    var outputStream = jsonFile.openOutputStream();
+    outputStream.writeString(prettyPrintJson(writeJson));
+
+    outputStream.onNoPendingWrites = () {
+      completer.complete(_anyErrors);
+    };
+    
+    outputStream.onClosed = () {
+      completer.complete(_anyErrors);
+    };
+  });
+
+  return completer.future;
+}
+
+
+/**
+ * Convert all files on [htmlPath].
+ *
+ * Returns a future that completes to the converted JSON object.
+ */
+Future<Object> _convertFiles(Path htmlPath) {
+  var completer = new Completer();
+
+  List<Future> fileFutures = [];
+
+  // Get a list of all HTML dart files.
+  // TODO(amouravski): discriminate .dart files.
+  final htmlDir = new Directory.fromPath(htmlPath);
+  final lister = htmlDir.list(recursive: false);
+
+  lister.onFile = (String path) {
+    final name = new Path.fromNative(path).filename;
+
+    // Ignore private classes.
+    if (name.startsWith('_')) return;
+
+    // Ignore non-dart files.
+    if (!name.endsWith('.dart')) return;
+
+    File file = new File(path);
+
+    // TODO(amouravski): Handle missing file.
+    if (!file.existsSync()) {
+      print('ERROR: cannot find file $path');
+      _anyErrors = true;
+      return;
+    }
+
+    fileFutures.add(_convertFile(file));
+  };
+
+
+  // Combine all JSON objects
+  lister.onDone = (_) {
+    Futures.wait(fileFutures).then((jsonList) {
+      var convertedJson = {};
+      jsonList.forEach((json) {
+        final k = json.keys[0];
+        convertedJson.putIfAbsent(k, () => json[k]);
+      });
+      completer.complete(convertedJson);
+    });
+  };
+
+  // TODO(amouravski): add more error handling.
+
+  return completer.future;
+}
+
+
+/**
+ * Convert a single file to JSON docs.
+ *
+ * Output is a single object which with one element that maps the file name
+ * to the list of comment lines.
+ */
+Future<Object> _convertFile(File file) {
+  var completer = new Completer();
+
+  var comments = {};
+
+  // Find all /// @docsEditable annotations.
+  InputStream file_stream = file.openInputStream();
+  StringInputStream inputLines = new StringInputStream(file_stream);
+
+  inputLines.onLine = () {
+    var comment = <String>[];
+
+    var docCommentFound = false;
+    String line;
+    while ((line = inputLines.readLine()) != null) {
+      var trimmedLine = line.trim();
+
+      // Sentinel found. Process the comment block.
+      if (trimmedLine.startsWith('///') &&
+          trimmedLine.contains('@docsEditable')) {
+        if (docCommentFound == true) {
+          var nextLine = inputLines.readLine();
+
+          if (nextLine == null) return false;
+
+          var lineObject = {};
+          //lineObject[nextLine] = comment;

[blois, 2012/11/26 20:07:43]

I assume this is where duplicated lines are being hit?

[Andrei Mouravski, 2012/11/26 23:24:11]

Yes. I'll print a warning.

+          comments.putIfAbsent(nextLine, () => comment);
+        }
+
+        // Reset.
+        docCommentFound = false;
+        comment = <String>[];
+      } else if ( // Start a comment block.
+          trimmedLine.startsWith('/**') ||
+          trimmedLine.startsWith('///')) {
+        docCommentFound = true;
+        comment.add(line);
+      } else if (docCommentFound &&
+          (trimmedLine.startsWith('*') || trimmedLine.startsWith('///'))) {
+        comment.add(line);
+      } else {
+        // Reset if we're not in a comment.
+        docCommentFound = false;
+        comment = <String>[];
+      }
+    }
+  };
+
+  inputLines.onClosed = () {
+    var jsonObject = {};
+    jsonObject[new Path(file.fullPathSync()).filename] = comments;
+    completer.complete(jsonObject);
+  };
+
+  // TODO(amouravski): better error handling.
+
+  return completer.future;
+}
+
+
+/**
+ * Merge the new JSON object and the existing file.
+ */
+Object _mergeJsonAndFile(Object json, File file) {
+  var completer = new Completer();
+
+  var fileJson = {};
+  var jsonRead = Strings.join(file.readAsLinesSync(), '\n');
+
+  if (jsonRead == '') {
+    print('WARNING: no data read from '
+        '${new Path(file.fullPathSync()).filename}');
+    _anyErrors = true;
+  } else {
+    fileJson = JSON.parse(jsonRead);
+  }
+  return _mergeJson(json, fileJson);
+}
+
+
+/**
+ * Merge two JSON objects, such that the returned JSON object is the
+ * union of both.
+ *
+ * Each JSON must be a map, with each value being a map.
+ */
+Object _mergeJson(Object json1, Object json2) {
+  if (json1 is Map && json2 is Map) {
+    // Then check if [json2] contains any key form [json1], in which case
+    // add all of the values from [json2] to the values of [json1].
+    json2.forEach((k, v) {
+      if (json1.containsKey(k)) {
+        v.forEach((vk, vv) {
+          if (json1[k].containsKey(vk) &&
+              !_listsEqual(json1[k][vk],vv)) {
+            // TODO(amouravski): add better warning message and only if there's
+            // a conflict.
+            print('WARNING: duplicate keys.');
+            _anyErrors = true;
+            
+            // Assume that json2 is more current and take its data as opposed
+            // to json1's.
+            json1[k][vk] = json2[k][vk];
+          } else {
+            json1[k].putIfAbsent(vk, () => vv);
+          }
+        });
+      } else {
+        json1.putIfAbsent(k, () => v);
+      }
+    });
+  } else {
+    throw new ArgumentError('JSON objects must both be Maps');
+  }
+
+  // TODO(amouravski): more error handling.
+
+  return json1;
+}
+
+
+/**
+ * Tests for equality between two lists.
+ *
+ * This checks the first level of depth, so does not work for nested lists.
+ */
+bool _listsEqual(List list1, List list2) {
+  return list1.every((e) => list2.contains(e)) &&
+      list2.every((e) => list1.contains(e));
+}
+
+
+/**
+ * Print JSON in a much nicer format.
+ *
+ * For example:
+ *     {"foo":["bar","baz"],"boo":{"far:"faz"}}
+ *
+ * becomes:
+ *
+ *     {
+ *       "foo":
+ *         [
+ *           "bar",
+ *           "baz"
+ *         ],
+ *       "boo":
+ *         {
+ *           "far":
+ *             "faz"
+ *         }
+ *     }
+ */
+String prettyPrintJson(Object json, [depth = 0]) {

[blois, 2012/11/26 20:07:43]

Probably (unfortunately) not now, but this should really be pushed into the JSON
serializer.

[Andrei Mouravski, 2012/11/26 23:24:11]

Agreed.

+  var output;
+
+  var printSpaces = Strings.join(
+      new List<String>(depth + 1).map((e) => ''), '  ');

[sra1, 2012/11/26 20:36:57]

Call it 'indentation'.
printSpaces looks like a verb so I expect it to be a function.

The +1 bothers me.
Strings.join(new List(depth).map((e) => '  '), '');

[Andrei Mouravski, 2012/11/26 23:24:11]

Done.

+
+  if (json is List) {
+    var recursiveOutput =
+        Strings.join(json.map((e) => prettyPrintJson(e, depth + 1)), ',\n');

[blois, 2012/11/26 20:07:43]

StringBuffer-ize?

[Andrei Mouravski, 2012/11/26 23:24:11]

In the interest of getting everything in, I'm putting this off for now.

+    output = '${printSpaces}[\n'
+      '${recursiveOutput}'
+      '\n${printSpaces}]';
+  } else if (json is Map) {
+    var mapList = json.keys.map((key) =>
+        '${printSpaces}${JSON.stringify(key)}:\n${prettyPrintJson(json[key], depth + 1)}');
+    var recursiveOutput =
+        Strings.join(mapList, ',\n');
+    output = '${printSpaces}{\n'
+      '${recursiveOutput}'
+      '\n${printSpaces}}';
+  } else {
+    output = '${printSpaces}${JSON.stringify(json)}';
+  }
+  return output;
+}