Experimental lexer generator: First draft of a Python lexer (based on the automata).

tools/lexer_generator/lexer_test.py

Index: tools/lexer_generator/lexer_test.py
diff --git a/tools/testrunner/local/junit_output.py b/tools/lexer_generator/lexer_test.py
similarity index 61%
copy from tools/testrunner/local/junit_output.py
copy to tools/lexer_generator/lexer_test.py
index 437adb178931f82364aa87e66315231b7b57a56d..5e3e97025cdfa2472cdc9f342bb81ee5e7dc2be5 100644
--- a/tools/testrunner/local/junit_output.py
+++ b/tools/lexer_generator/lexer_test.py
@@ -25,25 +25,43 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import unittest
+from lexer import Lexer
 
-import xml.etree.ElementTree as xml
+class LexerTestCase(unittest.TestCase):
 
+  def __verify_action_stream(self, stream, expected_stream):
+    for (ix, item) in enumerate(expected_stream):
+      self.assertEquals(stream[ix][0], item[0])
+      self.assertEquals(stream[ix][4], item[1])
 
-class JUnitTestOutput:
-  def __init__(self, test_suite_name):
-    self.root = xml.Element("testsuite")
-    self.root.attrib["name"] = test_suite_name
+  def test_simple(self):
+    rules = '''
+    <default>
+    "("           { LBRACE }
+    ")"           { RBRACE }
 
-  def HasRunTest(self, test_name, test_duration, test_failure):
-    testCaseElement = xml.Element("testcase")
-    testCaseElement.attrib["name"] = " ".join(test_name)
-    testCaseElement.attrib["time"] = str(round(test_duration, 3))
-    if len(test_failure):
-      failureElement = xml.Element("failure")
-      failureElement.text = test_failure
-      testCaseElement.append(failureElement)
-    self.root.append(testCaseElement)
+    "foo"         { FOO }
+    eof           <<terminate>>'''
 
-  def FinishAndWrite(self, file):
-    xml.ElementTree(self.root).write(file, "UTF-8")
+    lexer = Lexer(rules)
 
+    string = 'foo()\0'
+    self.__verify_action_stream(
+        lexer.lex(string),
+        [('FOO', 'foo'), ('LBRACE', '('), ('RBRACE', ')')])
+
+  def test_maximal_matching(self):
+    rules = '''
+    <default>
+    "<"           { LT }
+    "<<"          { SHL }
+    " "           { SPACE }
+    eof           <<terminate>>'''
+
+    lexer = Lexer(rules)
+
+    string = '<< <\0'
+    self.__verify_action_stream(
+        lexer.lex(string),
+        [('SHL', '<<'), ('SPACE', ' '), ('LT', '<')])