gn: Reduce stack memory use while parsing.

tools/gn/parser.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "tools/gn/parser.h"
 
 #include <utility>
 
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
@@ skipping 263 matching lines @@
     {nullptr, nullptr, -1},                                   // ELSE
     {&Parser::Name, &Parser::IdentifierOrCall, PRECEDENCE_CALL},  // IDENTIFIER
     {nullptr, nullptr, -1},                                       // COMMA
     {nullptr, nullptr, -1},                // UNCLASSIFIED_COMMENT
     {nullptr, nullptr, -1},                // LINE_COMMENT
     {nullptr, nullptr, -1},                // SUFFIX_COMMENT
     {&Parser::BlockComment, nullptr, -1},  // BLOCK_COMMENT
 };
 
 Parser::Parser(const std::vector<Token>& tokens, Err* err)
-    : err_(err), cur_(0) {
+    : invalid_token_(Location(), Token::INVALID, base::StringPiece()),
+      err_(err),
+      cur_(0) {
   for (const auto& token : tokens) {
     switch (token.type()) {
       case Token::LINE_COMMENT:
         line_comment_tokens_.push_back(token);
         break;
       case Token::SUFFIX_COMMENT:
         suffix_comment_tokens_.push_back(token);
         break;
       default:
         // Note that BLOCK_COMMENTs (top-level standalone comments) are passed
@@ skipping 75 matching lines @@
   return cur_token().type() == type;
 }
 
 bool Parser::Match(Token::Type type) {
   if (!LookAhead(type))
     return false;
   Consume();
   return true;
 }
 
-Token Parser::Consume(Token::Type type, const char* error_message) {
+const Token& Parser::Consume(Token::Type type, const char* error_message) {
   Token::Type types[1] = { type };
   return Consume(types, 1, error_message);
 }
 
-Token Parser::Consume(Token::Type* types,
-                      size_t num_types,
-                      const char* error_message) {
+const Token& Parser::Consume(Token::Type* types,
+                             size_t num_types,
+                             const char* error_message) {
   if (has_error()) {
     // Don't overwrite current error, but make progress through tokens so that
     // a loop that's expecting a particular token will still terminate.
     if (!at_end())
       cur_++;
-    return Token(Location(), Token::INVALID, base::StringPiece());
+    return invalid_token_;
   }
   if (at_end()) {
     const char kEOFMsg[] = "I hit EOF instead.";
     if (tokens_.empty())
       *err_ = Err(Location(), error_message, kEOFMsg);
     else
       *err_ = Err(tokens_[tokens_.size() - 1], error_message, kEOFMsg);
-    return Token(Location(), Token::INVALID, base::StringPiece());
+    return invalid_token_;
   }
 
   for (size_t i = 0; i < num_types; ++i) {
     if (cur_token().type() == types[i])
       return Consume();
   }
   *err_ = Err(cur_token(), error_message);
-  return Token(Location(), Token::INVALID, base::StringPiece());
+  return invalid_token_;
 }
 
-Token Parser::Consume() {
+const Token& Parser::Consume() {
   return tokens_[cur_++];
 }
 
 std::unique_ptr<ParseNode> Parser::ParseExpression() {
   return ParseExpression(0);
 }
 
 std::unique_ptr<ParseNode> Parser::ParseExpression(int precedence) {
   if (at_end())
     return std::unique_ptr<ParseNode>();
 
-  Token token = Consume();
+  const Token& token = Consume();
   PrefixFunc prefix = expressions_[token.type()].prefix;
 
   if (prefix == nullptr) {
     *err_ = Err(token,
                 std::string("Unexpected token '") + token.value().as_string() +
                     std::string("'"));
     return std::unique_ptr<ParseNode>();
   }
 
   std::unique_ptr<ParseNode> left = (this->*prefix)(token);
   if (has_error())
     return left;
 
   while (!at_end() && !IsStatementBreak(cur_token().type()) &&
          precedence <= expressions_[cur_token().type()].precedence) {
-    token = Consume();
-    InfixFunc infix = expressions_[token.type()].infix;
+    const Token& next_token = Consume();
+    InfixFunc infix = expressions_[next_token.type()].infix;
     if (infix == nullptr) {
-      *err_ = Err(token,
-                  std::string("Unexpected token '") +
-                      token.value().as_string() + std::string("'"));
+      *err_ = Err(next_token, std::string("Unexpected token '") +
+                                  next_token.value().as_string() +
+                                  std::string("'"));
       return std::unique_ptr<ParseNode>();
     }
-    left = (this->*infix)(std::move(left), token);
+    left = (this->*infix)(std::move(left), next_token);
     if (has_error())
       return std::unique_ptr<ParseNode>();
   }
 
   return left;
 }
 
-std::unique_ptr<ParseNode> Parser::Block(Token token) {
+std::unique_ptr<ParseNode> Parser::Block(const Token& token) {
   // This entrypoint into ParseBlock means it's part of an expression and we
   // always want the result.
   return ParseBlock(token, BlockNode::RETURNS_SCOPE);
 }
 
-std::unique_ptr<ParseNode> Parser::Literal(Token token) {
+std::unique_ptr<ParseNode> Parser::Literal(const Token& token) {
   return base::WrapUnique(new LiteralNode(token));
 }
 
-std::unique_ptr<ParseNode> Parser::Name(Token token) {
+std::unique_ptr<ParseNode> Parser::Name(const Token& token) {
   return IdentifierOrCall(std::unique_ptr<ParseNode>(), token);
 }
 
-std::unique_ptr<ParseNode> Parser::BlockComment(Token token) {
+std::unique_ptr<ParseNode> Parser::BlockComment(const Token& token) {
   std::unique_ptr<BlockCommentNode> comment(new BlockCommentNode());
   comment->set_comment(token);
   return std::move(comment);
 }
 
-std::unique_ptr<ParseNode> Parser::Group(Token token) {
+std::unique_ptr<ParseNode> Parser::Group(const Token& token) {
   std::unique_ptr<ParseNode> expr = ParseExpression();
   if (has_error())
     return std::unique_ptr<ParseNode>();
   Consume(Token::RIGHT_PAREN, "Expected ')'");
   return expr;
 }
 
-std::unique_ptr<ParseNode> Parser::Not(Token token) {
+std::unique_ptr<ParseNode> Parser::Not(const Token& token) {
   std::unique_ptr<ParseNode> expr = ParseExpression(PRECEDENCE_PREFIX + 1);
   if (has_error())
     return std::unique_ptr<ParseNode>();
   if (!expr) {
     if (!has_error())
       *err_ = Err(token, "Expected right-hand side for '!'.");
     return std::unique_ptr<ParseNode>();
   }
   std::unique_ptr<UnaryOpNode> unary_op(new UnaryOpNode);
   unary_op->set_op(token);
   unary_op->set_operand(std::move(expr));
   return std::move(unary_op);
 }
 
-std::unique_ptr<ParseNode> Parser::List(Token node) {
+std::unique_ptr<ParseNode> Parser::List(const Token& node) {
   std::unique_ptr<ParseNode> list(ParseList(node, Token::RIGHT_BRACKET, true));
   if (!has_error() && !at_end())
     Consume(Token::RIGHT_BRACKET, "Expected ']'");
   return list;
 }
 
 std::unique_ptr<ParseNode> Parser::BinaryOperator(
     std::unique_ptr<ParseNode> left,
-    Token token) {
+    const Token& token) {
   std::unique_ptr<ParseNode> right =
       ParseExpression(expressions_[token.type()].precedence + 1);
   if (!right) {
     if (!has_error()) {
       *err_ = Err(token, "Expected right-hand side for '" +
                              token.value().as_string() + "'");
     }
     return std::unique_ptr<ParseNode>();
   }
   std::unique_ptr<BinaryOpNode> binary_op(new BinaryOpNode);
   binary_op->set_op(token);
   binary_op->set_left(std::move(left));
   binary_op->set_right(std::move(right));
   return std::move(binary_op);
 }
 
 std::unique_ptr<ParseNode> Parser::IdentifierOrCall(
     std::unique_ptr<ParseNode> left,
-    Token token) {
+    const Token& token) {
   std::unique_ptr<ListNode> list(new ListNode);
   list->set_begin_token(token);
   list->set_end(base::WrapUnique(new EndNode(token)));
   std::unique_ptr<BlockNode> block;
   bool has_arg = false;
   if (LookAhead(Token::LEFT_PAREN)) {
-    Token start_token = Consume();
+    const Token& start_token = Consume();
     // Parsing a function call.
     has_arg = true;
     if (Match(Token::RIGHT_PAREN)) {
       // Nothing, just an empty call.
     } else {
       list = ParseList(start_token, Token::RIGHT_PAREN, false);
       if (has_error())
         return std::unique_ptr<ParseNode>();
       Consume(Token::RIGHT_PAREN, "Expected ')' after call");
     }
@@ skipping 11 matching lines @@
   }
   std::unique_ptr<FunctionCallNode> func_call(new FunctionCallNode);
   func_call->set_function(token);
   func_call->set_args(std::move(list));
   if (block)
     func_call->set_block(std::move(block));
   return std::move(func_call);
 }
 
 std::unique_ptr<ParseNode> Parser::Assignment(std::unique_ptr<ParseNode> left,
-                                              Token token) {
+                                              const Token& token) {
   if (left->AsIdentifier() == nullptr && left->AsAccessor() == nullptr) {
     *err_ = Err(left.get(),
         "The left-hand side of an assignment must be an identifier, "
         "scope access, or array access.");
     return std::unique_ptr<ParseNode>();
   }
   std::unique_ptr<ParseNode> value = ParseExpression(PRECEDENCE_ASSIGNMENT);
   if (!value) {
     if (!has_error())
       *err_ = Err(token, "Expected right-hand side for assignment.");
     return std::unique_ptr<ParseNode>();
   }
   std::unique_ptr<BinaryOpNode> assign(new BinaryOpNode);
   assign->set_op(token);
   assign->set_left(std::move(left));
   assign->set_right(std::move(value));
   return std::move(assign);
 }
 
 std::unique_ptr<ParseNode> Parser::Subscript(std::unique_ptr<ParseNode> left,
-                                             Token token) {
+                                             const Token& token) {
   // TODO: Maybe support more complex expressions like a[0][0]. This would
   // require work on the evaluator too.
   if (left->AsIdentifier() == nullptr) {
     *err_ = Err(left.get(), "May only subscript identifiers.",
         "The thing on the left hand side of the [] must be an identifier\n"
         "and not an expression. If you need this, you'll have to assign the\n"
         "value to a temporary before subscripting. Sorry.");
     return std::unique_ptr<ParseNode>();
   }
   std::unique_ptr<ParseNode> value = ParseExpression();
   Consume(Token::RIGHT_BRACKET, "Expecting ']' after subscript.");
   std::unique_ptr<AccessorNode> accessor(new AccessorNode);
   accessor->set_base(left->AsIdentifier()->value());
   accessor->set_index(std::move(value));
   return std::move(accessor);
 }
 
 std::unique_ptr<ParseNode> Parser::DotOperator(std::unique_ptr<ParseNode> left,
-                                               Token token) {
+                                               const Token& token) {
   if (left->AsIdentifier() == nullptr) {
     *err_ = Err(left.get(), "May only use \".\" for identifiers.",
         "The thing on the left hand side of the dot must be an identifier\n"
         "and not an expression. If you need this, you'll have to assign the\n"
         "value to a temporary first. Sorry.");
     return std::unique_ptr<ParseNode>();
   }
 
   std::unique_ptr<ParseNode> right = ParseExpression(PRECEDENCE_DOT);
   if (!right || !right->AsIdentifier()) {
     *err_ = Err(token, "Expected identifier for right-hand-side of \".\"",
         "Good: a.cookies\nBad: a.42\nLooks good but still bad: a.cookies()");
     return std::unique_ptr<ParseNode>();
   }
 
   std::unique_ptr<AccessorNode> accessor(new AccessorNode);
   accessor->set_base(left->AsIdentifier()->value());
   accessor->set_member(std::unique_ptr<IdentifierNode>(
       static_cast<IdentifierNode*>(right.release())));
   return std::move(accessor);
 }
 
 // Does not Consume the start or end token.
-std::unique_ptr<ListNode> Parser::ParseList(Token start_token,
+std::unique_ptr<ListNode> Parser::ParseList(const Token& start_token,
                                             Token::Type stop_before,
                                             bool allow_trailing_comma) {
   std::unique_ptr<ListNode> list(new ListNode);
   list->set_begin_token(start_token);
   bool just_got_comma = false;
   bool first_time = true;
   while (!LookAhead(stop_before)) {
     if (!first_time) {
       if (!just_got_comma) {
         // Require commas separate things in lists.
@@ skipping 61 matching lines @@
     return BlockComment(Consume());
   } else {
     // TODO(scottmg): Is this too strict? Just drop all the testing if we want
     // to allow "pointless" expressions and return ParseExpression() directly.
     std::unique_ptr<ParseNode> stmt = ParseExpression();
     if (stmt) {
       if (stmt->AsFunctionCall() || IsAssignment(stmt.get()))
         return stmt;
     }
     if (!has_error()) {
-      Token token = cur_or_last_token();
+      const Token& token = cur_or_last_token();
       *err_ = Err(token, "Expecting assignment or function call.");
     }
     return std::unique_ptr<ParseNode>();
   }
 }
 
 std::unique_ptr<BlockNode> Parser::ParseBlock(
-    Token begin_brace,
+    const Token& begin_brace,
     BlockNode::ResultMode result_mode) {
   if (has_error())
     return std::unique_ptr<BlockNode>();
   std::unique_ptr<BlockNode> block(new BlockNode(result_mode));
   block->set_begin_token(begin_brace);
 
   for (;;) {
     if (LookAhead(Token::RIGHT_BRACE)) {
       block->set_end(base::WrapUnique(new EndNode(Consume())));
       break;
@@ skipping 142 matching lines @@
         break;
       }
     }
 
     // Suffix comments were assigned in reverse, so if there were multiple on
     // the same node, they need to be reversed.
     if ((*i)->comments() && !(*i)->comments()->suffix().empty())
       const_cast<ParseNode*>(*i)->comments_mutable()->ReverseSuffix();
   }
 }