Experimental lexer generator: fixing the rule parser + adding tests.

tools/lexer_generator/rule_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 14 matching lines @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import ply.yacc as yacc
 from rule_lexer import RuleLexer
 
 class RuleParser:
 
   tokens = RuleLexer.tokens
 
-  aliases = dict()
-  transitions = dict()
+  def __init__(self):
+    self.aliases = dict()
+    self.transitions = dict()
 
   def p_statement_alias(self, p):
     'statement : ALIAS EQUALS REGEX'
     regex = self.lexer.lexer.lexmatch.group('regex')
     self.aliases[p[1]] = regex
 
   def p_statement_condition_transition(self, p):
-    'statement : CONDITION_BEGIN CONDITION CONDITION_END REGEX_AND_TRANSITION'
+    'statement : CONDITION_BEGIN CONDITION CONDITION_END REGEX_TRANSITION'
     old_condition = p[2]
-    regex = self.lexer.lexer.lexmatch.group('regex')
+    regex = self.lexer.lexer.lexmatch.group('regex').strip()
     new_condition = self.lexer.lexer.lexmatch.group('new')
     if old_condition not in self.transitions:
-      self.transitions[old_condition] = []
-    self.transitions[old_condition].append((regex, new_condition))
+      self.transitions[old_condition] = dict()
+    self.transitions[old_condition][regex] = ('condition', new_condition)
 
   def p_statement_condition_body(self, p):
-    'statement : CONDITION_BEGIN CONDITION CONDITION_END REGEX_AND_BODY'
+    'statement : CONDITION_BEGIN CONDITION CONDITION_END REGEX_BODY'
     old_condition = p[2]
-    regex = self.lexer.lexer.lexmatch.group('regex')
-    body = self.lexer.lexer.lexmatch.group('body')
+    regex = self.lexer.lexer.lexmatch.group('regex').strip()
+    body = self.lexer.lexer.lexmatch.group('body').strip()
     if old_condition not in self.transitions:
-      self.transitions[old_condition] = []
-    self.transitions[old_condition].append((regex, body))
+      self.transitions[old_condition] = dict()
+    self.transitions[old_condition][regex] = ('body', body)
 
-  def p_empty(self, p):
-    'empty :'
+  def p_statement_condition_transition_body(self, p):
+    'statement : CONDITION_BEGIN CONDITION CONDITION_END REGEX_TRANSITION_BODY'
+    old_condition = p[2]
+    regex = self.lexer.lexer.lexmatch.group('regex').strip()
+    new_condition = self.lexer.lexer.lexmatch.group('new').strip()
+    body = self.lexer.lexer.lexmatch.group('body').strip()
+    if old_condition not in self.transitions:
+      self.transitions[old_condition] = dict()
+    self.transitions[old_condition][regex] = (
+        'condition_and_body', new_condition, body)
 
   def p_error(self, p):
     raise Exception("Syntax error in input '%s'" % p)
 
   def build(self, **kwargs):
     self.parser = yacc.yacc(module=self, **kwargs)
     self.lexer = RuleLexer()
     self.lexer.build(**kwargs)
 
   def parse(self, data):
     return self.parser.parse(data, lexer=self.lexer.lexer)