tools/gn: add "gn help grammar" with formal grammar

tools/gn/parser.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "tools/gn/parser.h"
 
 #include "base/logging.h"
 #include "tools/gn/functions.h"
 #include "tools/gn/operators.h"
 #include "tools/gn/token.h"
 
-// grammar:
-//
-// file       := (statement)*
-// statement  := block | if | assignment
-// block      := '{' statement* '}'
-// if         := 'if' '(' expr ')' statement [ else ]
-// else       := 'else' (if | statement)*
-// assignment := ident {'=' | '+=' | '-='} expr
+const char kSyntax_Help[] =
+    "GN build language syntax\n"
+    "\n"
+    "  This help topic defines the syntax of the GN build language.\n"
+    "\n"
+    "Tokens\n"
+    "\n"
+    "  GN build files are read as sequences of tokens.  While splitting the\n"
+    "  file into tokens, the next token is the longest sequence of characters\n"
+    "  that form a valid token.\n"
+    "\n"
+    "White space and comments\n"
+    "\n"
+    "  White space is comprised of spaces (U+0020), horizontal tabs (U+0009),\n"
+    "  carriage returns (U+000D), and newlines (U+000A).\n"
+    "\n"
+    "  Comments start at the character \"#\" and stop at the end of the line.\n"

[Dirk Pranke, 2015/03/26 01:29:47]

If you wanted to be picky, how is the "end of the line" defined? 

+    "\n"
+    "  White space and comments are ignored except that they may separate\n"
+    "  tokens that would otherwise combine into a single token.\n"
+    "\n"
+    "Identifiers\n"
+    "\n"
+    "  Identifiers name variables and functions.\n"
+    "\n"
+    "      identifier = letter { letter | digit } .\n"
+    "      letter     = \"A\" ... \"Z\" | \"a\" ... \"z\" | \"_\" .\n"
+    "      digit      = \"0\" ... \"9\" .\n"
+    "\n"
+    "Keywords\n"
+    "\n"
+    "  The following keywords are reserved and may not be used as\n"
+    "  identifiers:\n"
+    "\n"
+    "          else    false   if      true\n"
+    "\n"
+    "Integer literals\n"
+    "\n"
+    "  An integer literal represents a decimal integer value.\n"
+    "\n"
+    "      integer = [ \"-\" ] digit { digit } .\n"
+    "\n"
+    "String literals\n"
+    "\n"
+    "  A string literal represents a string value consisting of the quoted\n"
+    "  characters with possible escape sequences and variable expansions.\n"
+    "\n"
+    "      string    = `\"` { char | escape | expansion } `\"` .\n"
+    "      escape    = `\\` ( \"$\" | `\"` | char ) .\n"
+    "      expansion = \"$\" ( identifier | \"{\" identifier \"}\" ) .\n"
+    "      char      = /* any character except \"$\", `\"`, or newline */ .\n"
+    "\n"
+    "  After a backslash, certain sequences represent special characters:\n"
+    "\n"
+    "          \\\"    U+0022    quotation mark\n"
+    "          \\$    U+0024    dollar sign\n"
+    "          \\\\    U+005c    backslash\n"
+    "\n"
+    "  All other backslashes represent themselves.\n"
+    "\n"
+    "Punctuation\n"
+    "\n"
+    "  The following character sequences represent punctuation:\n"
+    "\n"
+    "          +       +=      ==      !=      (       )\n"
+    "          -       -=      <       <=      [       ]\n"
+    "          !       =       >       >=      {       }\n"
+    "                          &&      ||      .       ,\n"
+    "\n"
+    "Grammar\n"
+    "\n"
+    "  The input tokens form a syntax tree following a context-free grammar:\n"
+    "\n"
+    "      File = StatementList .\n"
+    "\n"
+    "      Statement     = Assignment | Call | Condition .\n"
+    "      Assignment    = identifier AssignOp Expr .\n"
+    "      Call          = identifier \"(\" ExprList \")\" [ Block ] .\n"
+    "      Condition     = \"if\" \"(\" Expr \")\" Block\n"
+    "                      [ \"else\" ( Condition | Block ) ] .\n"
+    "      Block         = \"{\" StatementList \"}\" .\n"
+    "      StatementList = { Statement } .\n"
+    "\n"
+    "      Expr        = UnaryExpr | Expr BinaryOp Expr .\n"
+    "      UnaryExpr   = PrimaryExpr | UnaryOp UnaryExpr .\n"
+    "      PrimaryExpr = identifier | integer | string | Call\n"
+    "                  | identifier \"[\" Expr \"]\"\n"
+    "                  | identifier \".\" identifier\n"
+    "                  | \"(\" Expr \")\" | \"[\" ExprList \"] .\n"
+    "      ExprList    = [ Expr { \",\" Expr } [ \",\" ] ] .\n"
+    "\n"
+    "      AssignOp = \"=\" | \"+=\" | \"-=\" .\n"
+    "      UnaryOp  = \"!\" .\n"
+    "      BinaryOp = \"+\" | \"-\"                  // highest priority\n"
+    "               | \"<\" | \"<=\" | \">\" | \">=\"\n"
+    "               | \"==\" | \"!=\"\n"
+    "               | \"&&\"\n"
+    "               | \"||\" .                     // lowest priority\n"
+    "\n"
+    "  All binary operators are left-associative.\n";
 
 enum Precedence {
   PRECEDENCE_ASSIGNMENT = 1,  // Lowest precedence.
   PRECEDENCE_OR = 2,
   PRECEDENCE_AND = 3,
   PRECEDENCE_EQUALITY = 4,
   PRECEDENCE_RELATION = 5,
   PRECEDENCE_SUM = 6,
   PRECEDENCE_PREFIX = 7,
   PRECEDENCE_CALL = 8,
@@ skipping 220 matching lines @@
   scoped_ptr<ParseNode> expr = ParseExpression(PRECEDENCE_PREFIX + 1);
   if (has_error())
     return scoped_ptr<ParseNode>();
   scoped_ptr<UnaryOpNode> unary_op(new UnaryOpNode);
   unary_op->set_op(token);
   unary_op->set_operand(expr.Pass());
   return unary_op.Pass();
 }
 
 scoped_ptr<ParseNode> Parser::List(Token node) {
-  scoped_ptr<ParseNode> list(ParseList(node, Token::RIGHT_BRACKET, true));
+  scoped_ptr<ParseNode> list(ParseList(node, Token::RIGHT_BRACKET));
   if (!has_error() && !at_end())
     Consume(Token::RIGHT_BRACKET, "Expected ']'");
   return list.Pass();
 }
 
 scoped_ptr<ParseNode> Parser::BinaryOperator(scoped_ptr<ParseNode> left,
                                              Token token) {
   scoped_ptr<ParseNode> right =
       ParseExpression(expressions_[token.type()].precedence + 1);
   if (!right) {
@@ skipping 16 matching lines @@
   list->set_end(make_scoped_ptr(new EndNode(token)));
   scoped_ptr<BlockNode> block;
   bool has_arg = false;
   if (LookAhead(Token::LEFT_PAREN)) {
     Token start_token = Consume();
     // Parsing a function call.
     has_arg = true;
     if (Match(Token::RIGHT_PAREN)) {
       // Nothing, just an empty call.
     } else {
-      list = ParseList(start_token, Token::RIGHT_PAREN, false);
+      list = ParseList(start_token, Token::RIGHT_PAREN);
       if (has_error())
         return scoped_ptr<ParseNode>();
       Consume(Token::RIGHT_PAREN, "Expected ')' after call");
     }
     // Optionally with a scope.
     if (LookAhead(Token::LEFT_BRACE)) {
       block = ParseBlock();
       if (has_error())
         return scoped_ptr<ParseNode>();
     }
@@ skipping 63 matching lines @@
 
   scoped_ptr<AccessorNode> accessor(new AccessorNode);
   accessor->set_base(left->AsIdentifier()->value());
   accessor->set_member(scoped_ptr<IdentifierNode>(
       static_cast<IdentifierNode*>(right.release())));
   return accessor.Pass();
 }
 
 // Does not Consume the start or end token.
 scoped_ptr<ListNode> Parser::ParseList(Token start_token,
-                                       Token::Type stop_before,
-                                       bool allow_trailing_comma) {
+                                       Token::Type stop_before) {
   scoped_ptr<ListNode> list(new ListNode);
   list->set_begin_token(start_token);
   bool just_got_comma = false;
   bool first_time = true;
   while (!LookAhead(stop_before)) {
     if (!first_time) {
       if (!just_got_comma) {
         // Require commas separate things in lists.
         *err_ = Err(cur_token(), "Expected comma between items.");
         return scoped_ptr<ListNode>();
       }
     }
     first_time = false;
 
     // Why _OR? We're parsing things that are higher precedence than the ,
     // that separates the items of the list. , should appear lower than
     // boolean expressions (the lowest of which is OR), but above assignments.
     list->append_item(ParseExpression(PRECEDENCE_OR));
     if (has_error())
       return scoped_ptr<ListNode>();
     if (at_end()) {
       *err_ =
           Err(tokens_[tokens_.size() - 1], "Unexpected end of file in list.");
       return scoped_ptr<ListNode>();
     }
     if (list->contents().back()->AsBlockComment()) {
       // If there was a comment inside the list, we don't need a comma to the
       // next item, so pretend we got one, if we're expecting one.
-      just_got_comma = allow_trailing_comma;
+      just_got_comma = true;
     } else {
       just_got_comma = Match(Token::COMMA);
     }
   }
-  if (just_got_comma && !allow_trailing_comma) {
-    *err_ = Err(cur_token(), "Trailing comma");
-    return scoped_ptr<ListNode>();
-  }
   list->set_end(make_scoped_ptr(new EndNode(cur_token())));
   return list.Pass();
 }
 
 scoped_ptr<ParseNode> Parser::ParseFile() {
   scoped_ptr<BlockNode> file(new BlockNode);
   for (;;) {
     if (at_end())
       break;
     scoped_ptr<ParseNode> statement = ParseStatement();
@@ skipping 186 matching lines @@
         break;
       }
     }
 
     // Suffix comments were assigned in reverse, so if there were multiple on
     // the same node, they need to be reversed.
     if ((*i)->comments() && !(*i)->comments()->suffix().empty())
       const_cast<ParseNode*>(*i)->comments_mutable()->ReverseSuffix();
   }
 }