added Command Line Arguments, support for directories, hiding private data, not parsing the SDK, rem

pkg/docgen/bin/docgen.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * The docgen tool takes in a library as input and produces documentation
  * for the library as well as all libraries it imports and uses. The tool can
  * be run by passing in the path to a .dart file like this:
  *
  *   ./dart docgen.dart path/to/file.dart
  *
  * This outputs information about all classes, variables, functions, and
  * methods defined in the library and its imported libraries.
  */
 library docgen;
 
 // TODO(tmandel): Use 'package:' references for imports with relative paths.
 import 'dart:io';
 import 'dart:json';
 import 'dart:async';
-import '../lib/dart2yaml.dart';
+import 'package:docgen/dart2yaml.dart';
 import '../lib/src/dart2js_mirrors.dart';
 import 'package:markdown/markdown.dart' as markdown;
-import '../../args/lib/args.dart';
+import 'package:args/args.dart';
 import '../../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors.dart';
 import '../../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors_util.dart';
 
+/// Unique ID, will get incremented everytime an ID is requested. 
+int _uid = 0;
+
+int getID() => _uid++;
+
 /**
  * Entry function to create YAML documentation from Dart files.
  */
 void main() {
-  // TODO(tmandel): Use args library once flags are clear.
   Options opts = new Options();
   Docgen docgen = new Docgen();
   
-  if (opts.arguments.length > 0) {
-    List<Path> libraries = [new Path(opts.arguments[0])];
+  var parser = createArgParser(docgen);
+  var results = parser.parse(opts.arguments);
+  
+  if (results.rest.length != 1) {
+      print ("Usage: dart docgen.dart [OPTIONS] [FILE/DIR]");
+  } else {
+    Path directory = new Path(opts.arguments.last).directoryPath; 

[Emily Fortuna, 2013/06/17 21:04:54]

usual Dart style is to only type the variable if it's not obvious. So instead,
you can write:

var directory = new Path(....);

[janicejl, 2013/06/18 01:06:22]

Done.

+    List<Path> libraries;

[Emily Fortuna, 2013/06/17 21:04:54]

now about 
var libraries = []; since in both branches you instantiate a list.

[janicejl, 2013/06/18 01:06:22]

Done.

     Path sdkDirectory = new Path("../../../sdk/");
-    var workingMirrors = analyze(libraries, sdkDirectory, 
+    Path packageDir = directory.append("packages/");

[Emily Fortuna, 2013/06/17 21:04:54]

nit: you don't need to append the "/", I don't believe. "append" itself should
deal with that.

[janicejl, 2013/06/18 01:06:22]

Done.

+    var workingMirrors;

[Emily Fortuna, 2013/06/17 21:04:54]

why not just instantiate this down on line 66 where you first use it?

[janicejl, 2013/06/18 01:06:22]

Done.

+
+    if (new Path(opts.arguments.last).extension == "dart") {

[Emily Fortuna, 2013/06/17 21:04:54]

how about FileSystemEntity.isFileSync(opts.arguments.last) ? it's more obvious
what you're testing

[janicejl, 2013/06/18 01:06:22]

Done.

+      libraries = [new Path(opts.arguments.last)];
+    } else {
+      libraries = new List<Path>();
+      new Directory.fromPath(directory).listSync(recursive: true, 
+          followLinks: true).forEach((file) {
+            if (new Path(file.path).extension == "dart") {

[Emily Fortuna, 2013/06/17 21:04:54]

same here. Also, the directory might contain another directory, so you'll want
to do a recursive traversal 

[janicejl, 2013/06/18 01:06:22]

Should I only be checking if it is a file? Since it goes through all the files
in a folder, some of them are not dart files (like pubspec), and if I do not
check if they are dart files it will add pubspec to be analyzed as well. 

It also recursively traverse directories within directories in line 56. 

+              if (!file.path.contains("/packages/")) {
+                libraries.add(new Path(file.path));            
+              }
+            }
+          });
+    }
+    
+    workingMirrors = analyze(libraries, sdkDirectory, 
+        packageRoot: packageDir, 
         options: ['--preserve-comments', '--categories=Client,Server']);
+    
     workingMirrors.then( (MirrorSystem mirrorSystem) {
       var mirrors = mirrorSystem.libraries.values;
       if (mirrors.isEmpty) {
         print("no LibraryMirrors");
       } else {
         docgen.libraries = mirrors;
         docgen.documentLibraries();
       }
     });
   }
 }
 
 /**
+ * Returns a ArgParser with all the flags and options created. 
+ */
+ArgParser createArgParser(Docgen docgen) {
+  var parser = new ArgParser();
+  parser.addFlag("help", abbr: "h", help: "Prints help and usage information", 
+      negatable: false, callback: (help) {
+        if (help) print(parser.getUsage());
+      });
+  parser.addFlag("yaml", abbr: "y", help: "Outputs to YAML", 

[Emily Fortuna, 2013/06/17 21:04:54]

would the user really ever want to output both yaml AND json in a specific run?
That seems unlikely (to me). How about just have the json flag and then if not
specified we default to yaml?

[janicejl, 2013/06/18 01:06:22]

Previously I asked Tate and Andrei if users should have the option to output to
both yaml and json, and they both think they should have the option. Can I leave
this as is until I double check with them, I'll only have to revert back to the
if.. else instead of what I have now. 

[Emily Fortuna, 2013/06/19 17:23:34]

Okay. follow what they said then.

+      defaultsTo: true, negatable: true, callback: (yaml) {
+        docgen.outputToYaml = yaml;
+      });
+  parser.addFlag("json", abbr: "j", help: "Outputs to JSON", 
+      defaultsTo: false, negatable: true, callback: (json) {
+        docgen.outputToJson = json;
+    });
+  parser.addFlag("hide-private", help: "Hides private declarations" , 
+      defaultsTo: false, negatable: false, callback: (hidePrivate) {
+        docgen.hidePrivate = hidePrivate;
+      });
+  parser.addFlag("sdk", help: "Flag to parse SDK Library files", 
+      defaultsTo: true, negatable: true, callback: (sdk) {
+        docgen.sdk = sdk;
+      });
+  
+  return parser;
+}
+
+/**
  * This class documents a list of libraries.
  */
 class Docgen {
 
   /// Libraries to be documented.
   List<LibraryMirror> _libraries;
   
   /// Saves list of libraries for Docgen object.
   void set libraries(value) => _libraries = value;
   
   /// Current library being documented to be used for comment links.
   LibraryMirror _currentLibrary;
   
   /// Current class being documented to be used for comment links.
   ClassMirror _currentClass;
   
   /// Current member being documented to be used for comment links.
   MemberMirror _currentMember;
   
-  /// Should the output file type be JSON?
-  // TODO(tmandel): Add flag to allow for output to JSON.
-  bool outputToJson = false;
-  
   /// Resolves reference links
   markdown.Resolver linkResolver;
   
   /**
    * Docgen constructor initializes the link resolver for markdown parsing.
    */
   Docgen() {
     this.linkResolver = (name) => 
         fixReference(name, _currentLibrary, _currentClass, _currentMember);
   }
   
+  /// Should the output file type be YAML?
+  bool outputToYaml;
+  /// Should the output file type be JSON?
+  bool outputToJson;
+  /// Should the output file hide private declarations?
+  bool hidePrivate;
+  /// Should the output include SDK libraries?
+  bool sdk;
+  
   /**
    * Creates documentation for filtered libraries.
    */
   void documentLibraries() {
-    //TODO(tmandel): Filter libraries and determine output type using flags.
     _libraries.forEach((library) {
-      _currentLibrary = library;
-      var result = new Library(library.qualifiedName, _getComment(library),
-          _getVariables(library.variables), _getMethods(library.functions),
-          _getClasses(library.classes));
-      if (outputToJson) {
-        _writeToFile(stringify(result.toMap()), "${result.name}.json");
-      } else {
-        _writeToFile(getYamlString(result.toMap()), "${result.name}.yaml");
-      }
+      if (sdk || !library.uri.toString().startsWith("dart:")) {

[Emily Fortuna, 2013/06/17 21:04:54]

add a comment here explaining that if it starts with dart:, what that means

[janicejl, 2013/06/18 01:06:22]

Done.

+        _currentLibrary = library;
+        var result = new Library(library.qualifiedName, _getComment(library),
+            _getVariables(library.variables), _getMethods(library.functions),
+            _getClasses(library.classes));
+        if (outputToJson) {
+          _writeToFile(stringify(result.toMap()), "${result.name}.json");
+        } 
+        if (outputToYaml) {
+          _writeToFile(getYamlString(result.toMap()), "${result.name}.yaml");
+        }
+     }
     });
    }
   
   /**
    * Returns any documentation comments associated with a mirror with
    * simple markdown converted to html.
    */
   String _getComment(DeclarationMirror mirror) {
     String commentText;
     mirror.metadata.forEach((metadata) {
       if (metadata is CommentInstanceMirror) {
         CommentInstanceMirror comment = metadata;
         if (comment.isDocComment) {
           if (commentText == null) {
             commentText = comment.trimmedText;
           } else {
             commentText = "$commentText ${comment.trimmedText}";
           }
         } 
       }
     });
-    return commentText == null ? "" : 
+    commentText = commentText == null ? "" : 
       markdown.markdownToHtml(commentText.trim(), linkResolver: linkResolver);
+    return commentText.replaceAll("\n", "<br/>");
   }
 
   /**
    * Converts all [_] references in comments to <code>_</code>.
    */
   // TODO(tmandel): Create proper links for [_] style markdown based
   // on scope once layout of viewer is finished.
   markdown.Node fixReference(String name, LibraryMirror currentLibrary, 
       ClassMirror currentClass, MemberMirror currentMember) {
     return new markdown.Element.text('code', name);
   }
   
   /**
    * Returns a map of [Variable] objects constructed from inputted mirrors.
    */
   Map<String, Variable> _getVariables(Map<String, VariableMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, VariableMirror mirror) {
-      _currentMember = mirror;
-      data[mirrorName] = new Variable(mirrorName, mirror.isFinal,
-          mirror.isStatic, mirror.type.toString(), _getComment(mirror));
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentMember = mirror;
+        data[mirrorName] = new Variable(mirrorName, mirror.isFinal,
+            mirror.isStatic, mirror.type.toString(), _getComment(mirror));
+      }
     });
     return data;
   }
   
   /**
    * Returns a map of [Method] objects constructed from inputted mirrors.
    */
   Map<String, Method> _getMethods(Map<String, MethodMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, MethodMirror mirror) {
-      _currentMember = mirror;
-      data[mirrorName] = new Method(mirrorName, mirror.isSetter,
-          mirror.isGetter, mirror.isConstructor, mirror.isOperator, 
-          mirror.isStatic, mirror.returnType.toString(), _getComment(mirror),
-          _getParameters(mirror.parameters));
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentMember = mirror;
+        data[mirrorName] = new Method(mirrorName, mirror.isSetter,
+            mirror.isGetter, mirror.isConstructor, mirror.isOperator, 
+            mirror.isStatic, mirror.returnType.toString(), _getComment(mirror),
+            _getParameters(mirror.parameters));
+      }
     });
     return data;
   } 
   
   /**
    * Returns a map of [Class] objects constructed from inputted mirrors.
    */
   Map<String, Class> _getClasses(Map<String, ClassMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, ClassMirror mirror) {
-      _currentClass = mirror;
-      var superclass;
-      if (mirror.superclass != null) {
-        superclass = mirror.superclass.qualifiedName;
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentClass = mirror;
+        var superclass;

[Emily Fortuna, 2013/06/17 21:04:54]

superclass is only getting initialized if mirror.superClass != null. is this
what you want?

[janicejl, 2013/06/18 01:06:22]

Usually it will always have a superclass. The only instance where it does not
have a superclass is when it is null, in which on the yaml it outputs "null". 

I have changed it to initialise it to "" if it is null. 

+        if (mirror.superclass != null) {
+          superclass = mirror.superclass.qualifiedName;
+        }
+        var interfaces = 
+            mirror.superinterfaces.map((interface) => interface.qualifiedName);
+        data[mirrorName] = new Class(mirrorName, superclass, mirror.isAbstract,
+            mirror.isTypedef, _getComment(mirror), interfaces,
+            _getVariables(mirror.variables), _getMethods(mirror.methods));
       }
-      var interfaces = 
-          mirror.superinterfaces.map((interface) => interface.qualifiedName);
-      data[mirrorName] = new Class(mirrorName, superclass, mirror.isAbstract,
-          mirror.isTypedef, _getComment(mirror), interfaces,
-          _getVariables(mirror.variables), _getMethods(mirror.methods));
     });
     return data;
   }
   
   /**
    * Returns a map of [Parameter] objects constructed from inputted mirrors.
    */
   Map<String, Parameter> _getParameters(List<ParameterMirror> mirrorList) {
     var data = {};
     mirrorList.forEach((ParameterMirror mirror) {
@@ skipping 15 matching lines @@
     outputMap[key] = value.toMap();
   });
   return outputMap;
 }
 
 /**
  * A class containing contents of a Dart library.
  */
 class Library {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// Top-level variables in the library.
   Map<String, Variable> variables;
   
   /// Top-level functions in the library.
   Map<String, Method> functions;
   
   /// Classes defined within the library
   Map<String, Class> classes;
   
   String name;
   
   Library(this.name, this.comment, this.variables, 
-      this.functions, this.classes);
+      this.functions, this.classes) {
+    this.id = getID();

[Emily Fortuna, 2013/06/17 21:04:54]

how about pass an id number into the constructor?
Library(this.name, this.comment, ...., this.classes, this.id);

[janicejl, 2013/06/18 01:06:22]

Done.

+  }
   
   /// Generates a map describing the [Library] object.
   Map toMap() {
     var libraryMap = {};
+    libraryMap["id"] = id;
     libraryMap["name"] = name;
     libraryMap["comment"] = comment;
     libraryMap["variables"] = recurseMap(variables);
     libraryMap["functions"] = recurseMap(functions);
     libraryMap["classes"] = recurseMap(classes);
     return libraryMap;
   }
 }
 
 /**
  * A class containing contents of a Dart class.
  */
 // TODO(tmandel): Figure out how to do typedefs (what is needed)
 class Class {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// List of the names of interfaces that this class implements.
   List<String> interfaces;
   
   /// Top-level variables in the class.
   Map<String, Variable> variables;
   
   /// Methods in the class.
   Map<String, Method> methods;
   
   String name;
   String superclass;
   bool isAbstract;
   bool isTypedef;
  
   Class(this.name, this.superclass, this.isAbstract, this.isTypedef,
-      this.comment, this.interfaces, this.variables, this.methods); 
+      this.comment, this.interfaces, this.variables, this.methods) {
+    this.id = getID();
+  }
   
   /// Generates a map describing the [Class] object.
   Map toMap() {
     var classMap = {};
+    classMap["id"] = id;
     classMap["name"] = name;
     classMap["comment"] = comment;
     classMap["superclass"] = superclass;
     classMap["abstract"] = isAbstract.toString();
     classMap["typedef"] = isTypedef.toString();
     classMap["implements"] = new List.from(interfaces);
     classMap["variables"] = recurseMap(variables);
     classMap["methods"] = recurseMap(methods);
     return classMap;
   }
 }
 
 /**
  * A class containing properties of a Dart variable.
  */
 class Variable {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   String name;
   bool isFinal;
   bool isStatic;
   String type;
   
-  Variable(this.name, this.isFinal, this.isStatic, this.type, this.comment);
+  Variable(this.name, this.isFinal, this.isStatic, this.type, this.comment) {
+    this.id = getID();
+  }
   
   /// Generates a map describing the [Variable] object.
   Map toMap() {
     var variableMap = {};
+    variableMap["id"] = id;
     variableMap["name"] = name;
     variableMap["comment"] = comment;
     variableMap["final"] = isFinal.toString();
     variableMap["static"] = isStatic.toString();
     variableMap["type"] = type;
     return variableMap;
   }
 }
 
 /**
  * A class containing properties of a Dart method.
  */
 class Method {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// Parameters for this method.
   Map<String, Parameter> parameters;
   
   String name;
   bool isSetter;
   bool isGetter;
   bool isConstructor;
   bool isOperator;
   bool isStatic;
   String returnType;
   
   Method(this.name, this.isSetter, this.isGetter, this.isConstructor,
       this.isOperator, this.isStatic, this.returnType, this.comment,
-      this.parameters);
+      this.parameters) {
+    this.id = getID();
+  }
   
   /// Generates a map describing the [Method] object.
   Map toMap() {
     var methodMap = {};
+    methodMap["id"] = id;
     methodMap["name"] = name;
     methodMap["comment"] = comment;
     methodMap["type"] = isSetter ? "setter" : isGetter ? "getter" :
       isOperator ? "operator" : isConstructor ? "constructor" : "method";
     methodMap["static"] = isStatic.toString();
     methodMap["return"] = returnType;
     methodMap["parameters"] = recurseMap(parameters);
     return methodMap;
   }  
 }
 
 /**
  * A class containing properties of a Dart method/function parameter.
  */
 class Parameter {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   String name;
   bool isOptional;
   bool isNamed;
   bool hasDefaultValue;
   String type;
   String defaultValue;
   
   Parameter(this.name, this.isOptional, this.isNamed, this.hasDefaultValue,
-      this.type, this.defaultValue);
+      this.type, this.defaultValue) {
+    this.id = getID();
+  }
   
   /// Generates a map describing the [Parameter] object.
   Map toMap() {
     var parameterMap = {};
+    parameterMap["id"] = id;
     parameterMap["name"] = name;
     parameterMap["optional"] = isOptional.toString();
     parameterMap["named"] = isNamed.toString();
     parameterMap["default"] = hasDefaultValue.toString();
     parameterMap["type"] = type;
     parameterMap["value"] = defaultValue;
     return parameterMap;
   } 
 }
 
 /**
  * Writes text to a file in the 'docs' directory.
  */
 void _writeToFile(String text, String filename) {
   Directory dir = new Directory('docs');
   if (!dir.existsSync()) {
     dir.createSync();
   }
   File file = new File('docs/$filename');
   if (!file.exists()) {
     file.createSync();
   }
   file.openSync();
   file.writeAsString(text);
 }