Experimental parser: add catch all rule

src/lexer/lexer_py.re

Index: src/lexer/lexer_py.re
diff --git a/src/lexer/lexer_py.re b/src/lexer/lexer_py.re
index 37c9462c24076106929c882c10982935f5f9cbc3..902993bcd114c544baaf9a0a953bc87e16622c1f 100644
--- a/src/lexer/lexer_py.re
+++ b/src/lexer/lexer_py.re
@@ -163,24 +163,22 @@ eof             <<terminate>>
 default_action  { PUSH_TOKEN(ILLEGAL); }
 
 <DoubleQuoteString>
-"\\"      <<continue>>
-"\\\""    <<continue>>
-"\""      { PUSH_TOKEN(STRING); }
 /\\\n\r?/ <<continue>>
 /\\\r\n?/ <<continue>>
-/\n|\r/    { PUSH_TOKEN(ILLEGAL); }
+/\\./     <<continue>>
+/\n|\r/   { PUSH_TOKEN(ILLEGAL); }
+"\""      { PUSH_TOKEN(STRING); }
 eof       <<terminate_illegal>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <SingleQuoteString>
-"\\"      <<continue>>
-"\\'"     <<continue>>
-"'"       { PUSH_TOKEN(STRING); }
 /\\\n\r?/ <<continue>>
 /\\\r\n?/ <<continue>>
-/\n|\r/    { PUSH_TOKEN(ILLEGAL); }
+/\\./     <<continue>>
+/\n|\r/   { PUSH_TOKEN(ILLEGAL); }
+"'"       { PUSH_TOKEN(STRING); }
 eof       <<terminate_illegal>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <Identifier>
 identifier_char    <<continue>>
@@ -194,18 +192,21 @@ default_action { PUSH_TOKEN(IDENTIFIER); }
 <SingleLineComment>
 line_terminator  { PUSH_LINE_TERMINATOR(); }
 eof              <<terminate>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <MultiLineComment>
 "*/"             { SKIP(); }
+/\*./             <<continue>>
 # need to force action
 line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
 eof              <<terminate>>
-/./ <<continue>>
+catch_all <<continue>>
 
 <HtmlComment>
 "-->"            { SKIP(); }
+/--./            <<continue>>
+/-./             <<continue>>

[marja, 2013/11/12 08:45:02]

Why are these two needed?

[dcarney, 2013/11/12 09:03:59]

otherwise the string "<!-- --k" would produce an illegal token when it got to
the "--k"  it's kind of ugly.  if there's a lot of need for it, we can produce
some kind of syntax to handle better

 # need to force action
 line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
 eof              <<terminate>>
-/./ <<continue>>
+catch_all <<continue>>