Experimental parser: add catch all rule

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 145 matching lines @@
 /\\u[0-9a-fA-F]{4}/ {
   if (V8_UNLIKELY(!ValidIdentifierStart())) {
     PUSH_TOKEN(ILLEGAL);
   }
 } <<Identifier>>
 
 eof             <<terminate>>
 default_action  { PUSH_TOKEN(ILLEGAL); }
 
 <DoubleQuoteString>
-"\\"      <<continue>>
-"\\\""    <<continue>>
-"\""      { PUSH_TOKEN(STRING); }
 /\\\n\r?/ <<continue>>
 /\\\r\n?/ <<continue>>
-/\n|\r/    { PUSH_TOKEN(ILLEGAL); }
+/\\./     <<continue>>
+/\n|\r/   { PUSH_TOKEN(ILLEGAL); }
+"\""      { PUSH_TOKEN(STRING); }
 eof       <<terminate_illegal>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <SingleQuoteString>
-"\\"      <<continue>>
-"\\'"     <<continue>>
-"'"       { PUSH_TOKEN(STRING); }
 /\\\n\r?/ <<continue>>
 /\\\r\n?/ <<continue>>
-/\n|\r/    { PUSH_TOKEN(ILLEGAL); }
+/\\./     <<continue>>
+/\n|\r/   { PUSH_TOKEN(ILLEGAL); }
+"'"       { PUSH_TOKEN(STRING); }
 eof       <<terminate_illegal>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <Identifier>
 identifier_char    <<continue>>
 /\\u[0-9a-fA-F]{4}/ {
   if (V8_UNLIKELY(!ValidIdentifierStart())) {
     PUSH_TOKEN(ILLEGAL);
   }
 } <<continue>>
 default_action { PUSH_TOKEN(IDENTIFIER); }
 
 <SingleLineComment>
 line_terminator  { PUSH_LINE_TERMINATOR(); }
 eof              <<terminate>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <MultiLineComment>
 "*/"             { SKIP(); }
+/\*./             <<continue>>
 # need to force action
 line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
 eof              <<terminate>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <HtmlComment>
 "-->"            { SKIP(); }
+/--./            <<continue>>
+/-./             <<continue>>

[marja, 2013/11/12 08:45:02]

Why are these two needed?

[dcarney, 2013/11/12 09:03:59]

otherwise the string "<!-- --k" would produce an illegal token when it got to
the "--k"  it's kind of ugly.  if there's a lot of need for it, we can produce
some kind of syntax to handle better

 # need to force action
 line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
 eof              <<terminate>>
-/./ <<continue>>
+catch_all <<continue>>