gn: attach comments to parse tree

tools/gn/parser.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "tools/gn/parser.h"
 
 #include "base/logging.h"
 #include "tools/gn/functions.h"
 #include "tools/gn/operators.h"
 #include "tools/gn/token.h"
@@ skipping 56 matching lines @@
   {&Parser::Group, NULL, -1},                                   // LEFT_PAREN
   {NULL, NULL, -1},                                             // RIGHT_PAREN
   {&Parser::List, &Parser::Subscript, PRECEDENCE_CALL},         // LEFT_BRACKET
   {NULL, NULL, -1},                                             // RIGHT_BRACKET
   {NULL, NULL, -1},                                             // LEFT_BRACE
   {NULL, NULL, -1},                                             // RIGHT_BRACE
   {NULL, NULL, -1},                                             // IF
   {NULL, NULL, -1},                                             // ELSE
   {&Parser::Name, &Parser::IdentifierOrCall, PRECEDENCE_CALL},  // IDENTIFIER
   {NULL, NULL, -1},                                             // COMMA
-  {NULL, NULL, -1},                                             // COMMENT
+  {NULL, NULL, -1},  // UNCLASSIFIED_COMMENT
+  {NULL, NULL, -1},  // LINE_COMMENT
+  {NULL, NULL, -1},  // SUFFIX_COMMENT
 };
 
 Parser::Parser(const std::vector<Token>& tokens, Err* err)
-    : tokens_(tokens), err_(err), cur_(0) {
+    : err_(err), cur_(0) {
+  for (std::vector<Token>::const_iterator i(tokens.begin()); i != tokens.end();
+       ++i) {
+    switch(i->type()) {
+      case Token::LINE_COMMENT:
+        line_comment_tokens_.push_back(*i);
+        break;
+      case Token::SUFFIX_COMMENT:
+        suffix_comment_tokens_.push_back(*i);
+        break;
+      default:
+        tokens_.push_back(*i);
+        break;
+    }
+  }
 }
 
 Parser::~Parser() {
 }
 
 // static
 scoped_ptr<ParseNode> Parser::Parse(const std::vector<Token>& tokens,
                                     Err* err) {
   Parser p(tokens, err);
   return p.ParseFile().PassAs<ParseNode>();
@@ skipping 304 matching lines @@
       break;
     scoped_ptr<ParseNode> statement = ParseStatement();
     if (!statement)
       break;
     file->append_statement(statement.Pass());
   }
   if (!at_end() && !has_error())
     *err_ = Err(cur_token(), "Unexpected here, should be newline.");
   if (has_error())
     return scoped_ptr<ParseNode>();
+
+  // TODO(scottmg): If this is measurably expensive, it could be done only
+  // when necessary (when reformatting, or during tests). Comments are
+  // separate from the parse tree at this point, so downstream code can remain
+  // ignorant of them.
+  AssignComments(file.get());
+
   return file.PassAs<ParseNode>();
 }
 
 scoped_ptr<ParseNode> Parser::ParseStatement() {
   if (LookAhead(Token::LEFT_BRACE)) {
     return ParseBlock().PassAs<ParseNode>();
   } else if (LookAhead(Token::IF)) {
     return ParseCondition();
   } else {
     // TODO(scottmg): Is this too strict? Just drop all the testing if we want
@@ skipping 28 matching lines @@
     scoped_ptr<ParseNode> statement = ParseStatement();
     if (!statement)
       return scoped_ptr<BlockNode>();
     block->append_statement(statement.Pass());
   }
   return block.Pass();
 }
 
 scoped_ptr<ParseNode> Parser::ParseCondition() {
   scoped_ptr<ConditionNode> condition(new ConditionNode);
-  Consume(Token::IF, "Expected 'if'");
+  condition->set_if_token(Consume(Token::IF, "Expected 'if'"));
   Consume(Token::LEFT_PAREN, "Expected '(' after 'if'.");
   condition->set_condition(ParseExpression());
   if (IsAssignment(condition->condition()))
     *err_ = Err(condition->condition(), "Assignment not allowed in 'if'.");
   Consume(Token::RIGHT_PAREN, "Expected ')' after condition of 'if'.");
   condition->set_if_true(ParseBlock().Pass());
   if (Match(Token::ELSE))
     condition->set_if_false(ParseStatement().Pass());
   if (has_error())
     return scoped_ptr<ParseNode>();
   return condition.PassAs<ParseNode>();
 }
+
+void Parser::TraverseOrder(const ParseNode* root,
+                           std::vector<const ParseNode*>* pre,
+                           std::vector<const ParseNode*>* post) {
+  if (root) {
+    pre->push_back(root);
+
+    if (const AccessorNode* accessor = root->AsAccessor()) {
+      TraverseOrder(accessor->index(), pre, post);
+      TraverseOrder(accessor->member(), pre, post);
+    } else if (const BinaryOpNode* binop = root->AsBinaryOp()) {
+      TraverseOrder(binop->left(), pre, post);
+      TraverseOrder(binop->right(), pre, post);
+    } else if (const BlockNode* block = root->AsBlock()) {
+      const std::vector<ParseNode*>& statements = block->statements();
+      for (std::vector<ParseNode*>::const_iterator i(statements.begin());
+          i != statements.end();
+          ++i) {
+        TraverseOrder(*i, pre, post);
+      }
+    } else if (const ConditionNode* condition = root->AsConditionNode()) {
+      TraverseOrder(condition->condition(), pre, post);
+      TraverseOrder(condition->if_true(), pre, post);
+      TraverseOrder(condition->if_false(), pre, post);
+    } else if (const FunctionCallNode* func_call = root->AsFunctionCall()) {
+      TraverseOrder(func_call->args(), pre, post);
+      TraverseOrder(func_call->block(), pre, post);
+    } else if (root->AsIdentifier()) {
+      // Nothing.
+    } else if (const ListNode* list = root->AsList()) {
+      const std::vector<const ParseNode*>& contents = list->contents();
+      for (std::vector<const ParseNode*>::const_iterator i(contents.begin());
+          i != contents.end();
+          ++i) {
+        TraverseOrder(*i, pre, post);
+      }
+    } else if (root->AsLiteral()) {
+      // Nothing.
+    } else if (const UnaryOpNode* unaryop = root->AsUnaryOp()) {
+      TraverseOrder(unaryop->operand(), pre, post);
+    } else {
+      CHECK(false) << "Unhandled case in TraverseOrder.";
+    }
+
+    post->push_back(root);
+  }
+}
+
+void Parser::AssignComments(ParseNode* file) {
+  // Start by generating a pre- and post- order traversal of the tree so we
+  // can determine what's before and after comments.
+  std::vector<const ParseNode*> pre;
+  std::vector<const ParseNode*> post;
+  TraverseOrder(file, &pre, &post);
+
+  // Assign line comments to syntax immediately following.
+  int j = 0;

[brettw, 2014/09/23 21:33:15]

Can you give this a better name?

[scottmg, 2014/09/23 22:15:37]

Done.

+  for (std::vector<const ParseNode*>::const_iterator i = pre.begin();
+       i != pre.end();
+       ++i) {
+    const Location& start = (*i)->GetRange().begin();
+    while (j < static_cast<int>(line_comment_tokens_.size())) {
+      if (start.byte() >= line_comment_tokens_[j].location().byte()) {
+        const_cast<ParseNode*>((*i))->comments_mutable()->append_before(
+            line_comment_tokens_[j]);
+        ++j;
+      } else {
+        break;
+      }
+    }
+  }
+
+  // Remaining line comments go at end of file.
+  for (; j < static_cast<int>(line_comment_tokens_.size()); ++j)
+    file->comments_mutable()->append_after(line_comment_tokens_[j]);
+
+  // Assign suffix to syntax immediately before.
+  j = suffix_comment_tokens_.size() - 1;
+  for (std::vector<const ParseNode*>::const_reverse_iterator i = post.rbegin();
+       i != post.rend();
+       ++i) {
+    // Don't assign suffix comments to the function call or list, but instead
+    // to the last thing inside.
+    if ((*i)->AsFunctionCall() || (*i)->AsList())
+      continue;
+
+    const Location& start = (*i)->GetRange().begin();
+    const Location& end = (*i)->GetRange().end();
+
+    // Don't assign suffic comments to something that starts on an earlier
+    // line, so that in:
+    //
+    // sources = [ "a",
+    //     "b" ] # comment
+    //
+    // it's attached to "b", not sources = [ ... ].
+    if (start.line_number() != end.line_number())
+      continue;
+
+    while (j >= 0) {
+      if (end.byte() <= suffix_comment_tokens_[j].location().byte()) {
+        const_cast<ParseNode*>((*i))->comments_mutable()->append_suffix(
+            suffix_comment_tokens_[j]);
+        --j;
+      } else {
+        break;
+      }
+    }
+  }
+}