docgen working with a temporary link hack.

pkg/docgen/lib/docgen.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * The docgen tool takes in a library as input and produces documentation
  * for the library as well as all libraries it imports and uses. The tool can
  * be run by passing in the path to a .dart file like this:
  *
- *   ./dart docgen.dart path/to/file.dart
+ *   dart docgen.dart [OPTIONS] [FILE/DIR]
  *
  * This outputs information about all classes, variables, functions, and
  * methods defined in the library and its imported libraries.
  */
 library docgen;
 
-// TODO(tmandel): Use 'package:' references for imports with relative paths.
 import 'dart:io';
 import 'dart:json';
 import 'dart:async';
-import '../lib/dart2yaml.dart';
-import '../lib/src/dart2js_mirrors.dart';
 import 'package:markdown/markdown.dart' as markdown;
-import '../../args/lib/args.dart';
-import '../../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors.dart';
-import '../../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors_util.dart';
+import 'package:args/args.dart';
+import 'dart2yaml.dart';
+// TODO (janicejl/amouravski) Remove link import solution when there is a 

[Emily Fortuna, 2013/06/21 21:25:52]

format it like this:
TODO(janicejl/amouravski): Remove ....

Also if you file a bug for us to track at dartbug.com/new and the number in the
comment it will be much appreciated. 

[janicejl, 2013/06/21 21:44:33]

Done.

+// solution for keeping lib/_internal/libraries.dart and lib/_internal/lib 
+// in sync. 
+import 'package:hack/compiler/compiler.dart' as api;
+import 'package:hack/compiler/implementation/filenames.dart';
+import 'package:hack/compiler/implementation/mirrors/dart2js_mirror.dart'
+    as dart2js;
+import 'package:hack/compiler/implementation/mirrors/mirrors.dart';
+import 'package:hack/compiler/implementation/mirrors/mirrors_util.dart';
+import 'package:hack/compiler/implementation/source_file_provider.dart';
+import 'package:logging/logging.dart';
+
+/// Logger for Dart Doc Generator. 
+var logger = new Logger("Docgen");
+
+/// Unique ID, will get incremented everytime an ID is requested. 
+int _uid = 0;
+
+int getID() => _uid++;
+
+const String usage = "Usage: dart docgen.dart [OPTIONS] [FILE/DIR]";
 
 /**
- * Entry function to create YAML documentation from Dart files.
+ * Returns a ArgParser with all the flags and options created. 
  */
-void main() {
-  // TODO(tmandel): Use args library once flags are clear.
-  Options opts = new Options();
-  Docgen docgen = new Docgen();
+ArgParser initArgParser() {
+  var parser = new ArgParser();
+  parser.addFlag("help", abbr: "h", 
+      help: "Prints help and usage information", 
+      negatable: false, 
+      callback: (help) {
+        if (help) print(parser.getUsage());
+      });
+  parser.addFlag("verbose", abbr: "v", 
+      help: "Runs docgen with logging. ", 
+      defaultsTo: false, negatable: false, 
+      callback: (verbose) {
+        if (verbose) logger.onRecord.listen((record) => print(record.message));
+      });
+  parser.addFlag("yaml", abbr: "y", 
+      help: "Outputs to YAML", 
+      defaultsTo: true, negatable: true);
+  parser.addFlag("json", abbr: "j", 
+      help: "Outputs to JSON", 
+      defaultsTo: false, negatable: true);
+  parser.addFlag("hide-private", 
+      help: "Hides private declarations" , 
+      defaultsTo: false, negatable: false);
+  parser.addFlag("sdk", 
+      help: "Flag to parse SDK Library files", 
+      defaultsTo: true, negatable: true);
   
-  if (opts.arguments.length > 0) {
-    List<Path> libraries = [new Path(opts.arguments[0])];
-    Path sdkDirectory = new Path("../../../sdk/");
-    var workingMirrors = analyze(libraries, sdkDirectory, 
-        options: ['--preserve-comments', '--categories=Client,Server']);
-    workingMirrors.then( (MirrorSystem mirrorSystem) {
-      var mirrors = mirrorSystem.libraries.values;
-      if (mirrors.isEmpty) {
-        print("no LibraryMirrors");
-      } else {
-        docgen.libraries = mirrors;
-        docgen.documentLibraries();
-      }
-    });
+  return parser;
+}
+
+List<Path> listLibraries(List<String> args) {
+  if (args.length != 1) {
+    throw new UnsupportedError(usage);
   }
+  var libraries = new List<Path>();
+  var type = FileSystemEntity.typeSync(args[0]);
+  
+  if (type == FileSystemEntityType.NOT_FOUND) { 
+    throw new UnsupportedError("File does not exist. $usage");
+  } else if (type == FileSystemEntityType.LINK) {
+    libraries.addAll(listLibrariesFromDir(new Link(args[0]).targetSync()));
+  } else if (type == FileSystemEntityType.FILE) {
+    libraries.add(new Path(args[0]));
+    logger.info("Added to libraries: ${libraries.last.toString()}");
+  } else if (type == FileSystemEntityType.DIRECTORY) {
+    libraries.addAll(listLibrariesFromDir(args[0]));
+  }
+  return libraries;
+}
+
+List<Path> listLibrariesFromDir(String path) {
+  var libraries = new List<Path>();
+  new Directory(path).listSync(recursive: true, 
+      followLinks: true).forEach((file) {
+        if (new Path(file.path).extension == "dart") {
+          if (!file.path.contains("/packages/")) {
+            libraries.add(new Path(file.path));
+            logger.info("Added to libraries: ${libraries.last.toString()}");
+          }
+        }
+      });
+  return libraries;
 }
 
 /**
  * This class documents a list of libraries.
  */
 class Docgen {
 
   /// Libraries to be documented.
   List<LibraryMirror> _libraries;
   
   /// Saves list of libraries for Docgen object.
-  void set libraries(value) => _libraries = value;
+  void set libraries(value) {
+    _libraries = value;
+  }
   
   /// Current library being documented to be used for comment links.
   LibraryMirror _currentLibrary;
   
   /// Current class being documented to be used for comment links.
   ClassMirror _currentClass;
   
   /// Current member being documented to be used for comment links.
   MemberMirror _currentMember;
   
-  /// Should the output file type be JSON?
-  // TODO(tmandel): Add flag to allow for output to JSON.
-  bool outputToJson = false;
-  
   /// Resolves reference links
   markdown.Resolver linkResolver;
   
+  /// Should the output file type be YAML?
+  bool outputToYaml;
+  /// Should the output file type be JSON?
+  bool outputToJson;
+  /// Should the output file hide private declarations?
+  bool hidePrivate;
+  /// Should the output include SDK libraries?
+  bool sdk;
+  
   /**
    * Docgen constructor initializes the link resolver for markdown parsing.
+   * Also initializes the command line arguments. 
    */
-  Docgen() {
+  Docgen(ArgResults argResults) {  
+    outputToYaml = argResults["yaml"];
+    outputToJson = argResults["json"];
+    hidePrivate = argResults["hide-private"];
+    sdk = argResults["sdk"];
+    
     this.linkResolver = (name) => 
         fixReference(name, _currentLibrary, _currentClass, _currentMember);
   }
   
   /**
+   * Analyzes set of libraries by getting a mirror system and triggers the 
+   * documentation of the libraries. 
+   */
+  void analyze(List<Path> libraries) {
+    // DART_SDK should be set to the root of the SDK library. 

[Emily Fortuna, 2013/06/21 21:25:52]

I'm curious what the other values of DART_SDK were in the other "non-standard"
parts of the project? 

[janicejl, 2013/06/21 21:44:33]

Everyone points it to a dart-sdk, but some point it to the version downloaded,
some point it to the version in out, some point it to the sdk folder in the
repo. 

+    var sdkRoot = Platform.environment["DART_SDK"];
+    if (sdkRoot != null) {
+      logger.info("Using DART_SDK to find SDK at $sdkRoot");
+      sdkRoot = new Path(sdkRoot);
+    } else {
+      // Assuming the dart executable is from the Dart SDK folder inside bin. 

[Emily Fortuna, 2013/06/21 21:25:52]

nit: make it a sentence. Example:

// If DART_SDK is not defined in the environment, assume the dart executable is
from....

[janicejl, 2013/06/21 21:44:33]

Done.

+      sdkRoot = new Path(new Options().executable).directoryPath
+          .directoryPath;
+      logger.info("SDK Root: ${sdkRoot.toString()}");
+    }
+    
+    Path packageDir = libraries.last.directoryPath.append("packages");
+    logger.info("Package Root: ${packageDir.toString()}");
+    getMirrorSystem(libraries, sdkRoot, 
+        packageRoot: packageDir).then((MirrorSystem mirrorSystem) {
+          if (mirrorSystem.libraries.values.isEmpty) {
+            throw new UnsupportedError("No Library Mirrors.");
+          } 
+          this.libraries = mirrorSystem.libraries.values;
+          documentLibraries();
+        });
+  }
+  
+  /**
+   * Analyzes set of libraries and provides a mirror system which can be used 
+   * for static inspection of the source code.
+   */
+  Future<MirrorSystem> getMirrorSystem(List<Path> libraries,
+        Path libraryRoot, {Path packageRoot}) {
+    SourceFileProvider provider = new SourceFileProvider();
+    api.DiagnosticHandler diagnosticHandler =
+          new FormattingDiagnosticHandler(provider).diagnosticHandler;
+    Uri libraryUri = currentDirectory.resolve(appendSlash('$libraryRoot'));
+    Uri packageUri = null;
+    if (packageRoot != null) {
+      packageUri = currentDirectory.resolve(appendSlash('$packageRoot'));
+    }
+    List<Uri> librariesUri = <Uri>[];
+    libraries.forEach((library) {
+      librariesUri.add(currentDirectory.resolve(library.toString()));
+    });
+    return dart2js.analyze(librariesUri, libraryUri, packageUri,
+        provider.readStringFromUri, diagnosticHandler,
+        ['--preserve-comments', '--categories=Client,Server']);
+  }
+  
+  /**
    * Creates documentation for filtered libraries.
    */
   void documentLibraries() {
-    //TODO(tmandel): Filter libraries and determine output type using flags.
     _libraries.forEach((library) {
-      _currentLibrary = library;
-      var result = new Library(library.qualifiedName, _getComment(library),
-          _getVariables(library.variables), _getMethods(library.functions),
-          _getClasses(library.classes));
-      if (outputToJson) {
-        _writeToFile(stringify(result.toMap()), "${result.name}.json");
-      } else {
-        _writeToFile(getYamlString(result.toMap()), "${result.name}.yaml");
-      }
+      // Files belonging to the SDK have a uri that begins with "dart:".
+      if (sdk || !library.uri.toString().startsWith("dart:")) {
+        _currentLibrary = library;
+        var result = new Library(library.qualifiedName, _getComment(library),
+            _getVariables(library.variables), _getMethods(library.functions),
+            _getClasses(library.classes), getID());
+        if (outputToJson) {
+          _writeToFile(stringify(result.toMap()), "${result.name}.json");
+        } 
+        if (outputToYaml) {
+          _writeToFile(getYamlString(result.toMap()), "${result.name}.yaml");
+        }
+     }
     });
    }
   
   /**
    * Returns any documentation comments associated with a mirror with
    * simple markdown converted to html.
    */
   String _getComment(DeclarationMirror mirror) {
     String commentText;
     mirror.metadata.forEach((metadata) {
       if (metadata is CommentInstanceMirror) {
         CommentInstanceMirror comment = metadata;
         if (comment.isDocComment) {
           if (commentText == null) {
             commentText = comment.trimmedText;
           } else {
             commentText = "$commentText ${comment.trimmedText}";
           }
         } 
       }
     });
-    return commentText == null ? "" : 
-      markdown.markdownToHtml(commentText.trim(), linkResolver: linkResolver);
+    commentText = commentText == null ? "" : 
+        markdown.markdownToHtml(commentText.trim(), linkResolver: linkResolver)
+        .replaceAll("\n", "");
+    return commentText;
   }
 
   /**
    * Converts all [_] references in comments to <code>_</code>.
    */
   // TODO(tmandel): Create proper links for [_] style markdown based
   // on scope once layout of viewer is finished.
   markdown.Node fixReference(String name, LibraryMirror currentLibrary, 
       ClassMirror currentClass, MemberMirror currentMember) {
     return new markdown.Element.text('code', name);
   }
   
   /**
    * Returns a map of [Variable] objects constructed from inputted mirrors.
    */
   Map<String, Variable> _getVariables(Map<String, VariableMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, VariableMirror mirror) {
-      _currentMember = mirror;
-      data[mirrorName] = new Variable(mirrorName, mirror.isFinal,
-          mirror.isStatic, mirror.type.toString(), _getComment(mirror));
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentMember = mirror;
+        data[mirrorName] = new Variable(mirrorName, mirror.isFinal,
+            mirror.isStatic, mirror.type.toString(), _getComment(mirror), 
+            getID());
+      }
     });
     return data;
   }
   
   /**
    * Returns a map of [Method] objects constructed from inputted mirrors.
    */
   Map<String, Method> _getMethods(Map<String, MethodMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, MethodMirror mirror) {
-      _currentMember = mirror;
-      data[mirrorName] = new Method(mirrorName, mirror.isSetter,
-          mirror.isGetter, mirror.isConstructor, mirror.isOperator, 
-          mirror.isStatic, mirror.returnType.toString(), _getComment(mirror),
-          _getParameters(mirror.parameters));
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentMember = mirror;
+        data[mirrorName] = new Method(mirrorName, mirror.isSetter,
+            mirror.isGetter, mirror.isConstructor, mirror.isOperator, 
+            mirror.isStatic, mirror.returnType.toString(), _getComment(mirror),
+            _getParameters(mirror.parameters), getID());
+      }
     });
     return data;
   } 
   
   /**
    * Returns a map of [Class] objects constructed from inputted mirrors.
    */
   Map<String, Class> _getClasses(Map<String, ClassMirror> mirrorMap) {
     var data = {};
     mirrorMap.forEach((String mirrorName, ClassMirror mirror) {
-      _currentClass = mirror;
-      var superclass;
-      if (mirror.superclass != null) {
-        superclass = mirror.superclass.qualifiedName;
+      if (!hidePrivate || !mirror.isPrivate) {
+        _currentClass = mirror;
+        var superclass = (mirror.superclass != null) ? 
+            mirror.superclass.qualifiedName : "";
+        var interfaces = 
+            mirror.superinterfaces.map((interface) => interface.qualifiedName);
+        data[mirrorName] = new Class(mirrorName, superclass, mirror.isAbstract,
+            mirror.isTypedef, _getComment(mirror), interfaces.toList(),
+            _getVariables(mirror.variables), _getMethods(mirror.methods), 
+            getID());
       }
-      var interfaces = 
-          mirror.superinterfaces.map((interface) => interface.qualifiedName);
-      data[mirrorName] = new Class(mirrorName, superclass, mirror.isAbstract,
-          mirror.isTypedef, _getComment(mirror), interfaces,
-          _getVariables(mirror.variables), _getMethods(mirror.methods));
     });
     return data;
   }
   
   /**
    * Returns a map of [Parameter] objects constructed from inputted mirrors.
    */
   Map<String, Parameter> _getParameters(List<ParameterMirror> mirrorList) {
     var data = {};
     mirrorList.forEach((ParameterMirror mirror) {
       _currentMember = mirror;
       data[mirror.simpleName] = new Parameter(mirror.simpleName, 
           mirror.isOptional, mirror.isNamed, mirror.hasDefaultValue, 
-          mirror.type.toString(), mirror.defaultValue);
+          mirror.type.toString(), mirror.defaultValue, getID());
     });
     return data;
   }
 }
 
 /**
  * Transforms the map by calling toMap on each value in it.
  */
 Map recurseMap(Map inputMap) {
   var outputMap = {};
   inputMap.forEach((key, value) {
     outputMap[key] = value.toMap();
   });
   return outputMap;
 }
 
 /**
  * A class containing contents of a Dart library.
  */
 class Library {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// Top-level variables in the library.
   Map<String, Variable> variables;
   
   /// Top-level functions in the library.
   Map<String, Method> functions;
   
   /// Classes defined within the library
   Map<String, Class> classes;
   
   String name;
   
   Library(this.name, this.comment, this.variables, 
-      this.functions, this.classes);
+      this.functions, this.classes, this.id);
   
   /// Generates a map describing the [Library] object.
   Map toMap() {
     var libraryMap = {};
+    libraryMap["id"] = id;
     libraryMap["name"] = name;
     libraryMap["comment"] = comment;
     libraryMap["variables"] = recurseMap(variables);
     libraryMap["functions"] = recurseMap(functions);
     libraryMap["classes"] = recurseMap(classes);
     return libraryMap;
   }
 }
 
 /**
  * A class containing contents of a Dart class.
  */
 // TODO(tmandel): Figure out how to do typedefs (what is needed)
 class Class {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// List of the names of interfaces that this class implements.
   List<String> interfaces;
   
   /// Top-level variables in the class.
   Map<String, Variable> variables;
   
   /// Methods in the class.
   Map<String, Method> methods;
   
   String name;
   String superclass;
   bool isAbstract;
   bool isTypedef;
  
   Class(this.name, this.superclass, this.isAbstract, this.isTypedef,
-      this.comment, this.interfaces, this.variables, this.methods); 
+      this.comment, this.interfaces, this.variables, this.methods, this.id);
   
   /// Generates a map describing the [Class] object.
   Map toMap() {
     var classMap = {};
+    classMap["id"] = id;
     classMap["name"] = name;
     classMap["comment"] = comment;
     classMap["superclass"] = superclass;
     classMap["abstract"] = isAbstract.toString();
     classMap["typedef"] = isTypedef.toString();
     classMap["implements"] = new List.from(interfaces);
     classMap["variables"] = recurseMap(variables);
     classMap["methods"] = recurseMap(methods);
     return classMap;
   }
 }
 
 /**
  * A class containing properties of a Dart variable.
  */
 class Variable {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   String name;
   bool isFinal;
   bool isStatic;
   String type;
   
-  Variable(this.name, this.isFinal, this.isStatic, this.type, this.comment);
+  Variable(this.name, this.isFinal, this.isStatic, this.type, 
+      this.comment, this.id);
   
   /// Generates a map describing the [Variable] object.
   Map toMap() {
     var variableMap = {};
+    variableMap["id"] = id;
     variableMap["name"] = name;
     variableMap["comment"] = comment;
     variableMap["final"] = isFinal.toString();
     variableMap["static"] = isStatic.toString();
     variableMap["type"] = type;
     return variableMap;
   }
 }
 
 /**
  * A class containing properties of a Dart method.
  */
 class Method {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   /// Documentation comment with converted markdown.
   String comment;
   
   /// Parameters for this method.
   Map<String, Parameter> parameters;
   
   String name;
   bool isSetter;
   bool isGetter;
   bool isConstructor;
   bool isOperator;
   bool isStatic;
   String returnType;
   
   Method(this.name, this.isSetter, this.isGetter, this.isConstructor,
       this.isOperator, this.isStatic, this.returnType, this.comment,
-      this.parameters);
+      this.parameters, this.id);
   
   /// Generates a map describing the [Method] object.
   Map toMap() {
     var methodMap = {};
+    methodMap["id"] = id;
     methodMap["name"] = name;
     methodMap["comment"] = comment;
     methodMap["type"] = isSetter ? "setter" : isGetter ? "getter" :
       isOperator ? "operator" : isConstructor ? "constructor" : "method";
     methodMap["static"] = isStatic.toString();
     methodMap["return"] = returnType;
     methodMap["parameters"] = recurseMap(parameters);
     return methodMap;
   }  
 }
 
 /**
  * A class containing properties of a Dart method/function parameter.
  */
 class Parameter {
   
+  /// Unique ID number for resolving links. 
+  int id;
+  
   String name;
   bool isOptional;
   bool isNamed;
   bool hasDefaultValue;
   String type;
   String defaultValue;
   
   Parameter(this.name, this.isOptional, this.isNamed, this.hasDefaultValue,
-      this.type, this.defaultValue);
+      this.type, this.defaultValue, this.id);
   
   /// Generates a map describing the [Parameter] object.
   Map toMap() {
     var parameterMap = {};
+    parameterMap["id"] = id;
     parameterMap["name"] = name;
     parameterMap["optional"] = isOptional.toString();
     parameterMap["named"] = isNamed.toString();
     parameterMap["default"] = hasDefaultValue.toString();
     parameterMap["type"] = type;
     parameterMap["value"] = defaultValue;
     return parameterMap;
   } 
 }
 
 /**
  * Writes text to a file in the 'docs' directory.
  */
 void _writeToFile(String text, String filename) {
   Directory dir = new Directory('docs');
   if (!dir.existsSync()) {
     dir.createSync();
   }
   File file = new File('docs/$filename');
-  if (!file.exists()) {
+  if (!file.existsSync()) {
     file.createSync();
   }
   file.openSync();
   file.writeAsString(text);
 }