Experimental parser: add docs.

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
 #       with the distribution.
 #     * Neither the name of Google Inc. nor the names of its
 #       contributors may be used to endorse or promote products derived
 #       from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+# Character classes and auxiliary regexps.
+
 line_terminator = [:line_terminator:];
 identifier_start = [$_:letter:];
 identifier_char = [:identifier_start::identifier_part_not_letter:];
 digit = [0-9];
 hex_digit = [0-9a-fA-F];
 single_escape_char = ['"\\bfnrtv];
 maybe_exponent = /([eE][\-+]?[:digit:]+)?/;
 octal_number = /0[0-7]+/;
 
 # Octal numbers are pretty complicated. For example, 01.0 is invalid, since it's
 # parsed as 2 numbers (01 and .0), since 01 is a valid octal number. However,
 # 09.0 is a valid, since 09 cannot be octal. In addition, 0 and 0.1 are valid
 # numbers starting with 0.
 
 non_octal_whole_part = "0" | (
   /0[:digit:]*[8-9][:digit:]*/ |
   /[1-9][:digit:]*/ );
 
 number =
   /0[xX][:hex_digit:]+/ | (
   /\.[:digit:]+/ maybe_exponent |
   non_octal_whole_part /(\.[:digit:]*)?/ maybe_exponent );
 harmony_number = "0"[bBoO][:digit:]+;
 line_terminator_sequence = /[:line_terminator:]|(\r\n|\n\r)/;
 eos = [:eos:];
 
-# grammar is
-#   regex <action_on_state_entry|action_on_match|transition>
+# Rules.
+
+# Grammar is
+#   regex <entry_action|match_action|transition>
 #
-# actions are identifiers to be passed to codegen
-# transition must be 'continue' or the name of a subgraph
+# Actions are identifiers to be passed to codegen.
+#
+# Entry action is executed when we enter the corresponding automaton state, that
+# is, right after seeing something that matches the regex. Match action is
+# executed when we have matched the regex but cannot continue to match something
+# bigger (there is no legal transition out with the next character we're
+# lexing).
+#
+# Transition must be 'continue' or the name of a subgraph.
 
 <<default>>
 
 "|="          <|token(ASSIGN_BIT_OR)|>
 "^="          <|token(ASSIGN_BIT_XOR)|>
 "&="          <|token(ASSIGN_BIT_AND)|>
 "+="          <|token(ASSIGN_ADD)|>
 "-="          <|token(ASSIGN_SUB)|>
 "*="          <|token(ASSIGN_MUL)|>
 "/="          <|token(ASSIGN_DIV)|>
@@ skipping 158 matching lines @@
 eos              <|skip_and_terminate|>
 catch_all        <||continue>
 
 <<MultiLineComment>>
 /\*+\//          <|skip|>
 # TODO find a way to generate the below rule
 /\*+[^\/*]/      <||continue>
 line_terminator  <line_terminator_in_multiline_comment||continue>
 eos              <|terminate_illegal|>
 catch_all        <||continue>