Experimental Parser: add lexer generator

tools/lexer_generator/regex_lexer.py

Index: tools/lexer_generator/regex_lexer.py
diff --git a/tools/testrunner/local/junit_output.py b/tools/lexer_generator/regex_lexer.py
similarity index 56%
copy from tools/testrunner/local/junit_output.py
copy to tools/lexer_generator/regex_lexer.py
index 437adb178931f82364aa87e66315231b7b57a56d..deabd84c5d6ab42850ec384e8179ae4ceb9e0419 100644
--- a/tools/testrunner/local/junit_output.py
+++ b/tools/lexer_generator/regex_lexer.py
@@ -25,25 +25,78 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import ply.lex as lex
 
-import xml.etree.ElementTree as xml
+class RegexLexer:
 
+  tokens = (
 
-class JUnitTestOutput:
-  def __init__(self, test_suite_name):
-    self.root = xml.Element("testsuite")
-    self.root.attrib["name"] = test_suite_name
+    'GROUP_BEGIN',
+    'GROUP_END',
 
-  def HasRunTest(self, test_name, test_duration, test_failure):
-    testCaseElement = xml.Element("testcase")
-    testCaseElement.attrib["name"] = " ".join(test_name)
-    testCaseElement.attrib["time"] = str(round(test_duration, 3))
-    if len(test_failure):
-      failureElement = xml.Element("failure")
-      failureElement.text = test_failure
-      testCaseElement.append(failureElement)
-    self.root.append(testCaseElement)
+    'CLASS_BEGIN',
+    'CLASS_END',
 
-  def FinishAndWrite(self, file):
-    xml.ElementTree(self.root).write(file, "UTF-8")
+    'OR',
+    'ONE_OR_MORE',
+    'ZERO_OR_MORE',
+    'ZERO_OR_ONE',
+    'ANY',
 
+    'LITERAL',
+
+    'RANGE',
+    'NOT',
+    'CLASS_LITERAL',
+  )
+
+  states = (
+    ('class','exclusive'),
+  )
+
+  def t_ESCAPED_LITERAL(self, t):
+    r'\\\(|\\\)|\\\[|\\\]|\\\||\\\+|\\\*|\\\?|\\\.|\\\\'
+    t.type = 'LITERAL'
+    t.value = t.value[1:]
+    return t
+
+  t_GROUP_BEGIN = r'\('
+  t_GROUP_END = r'\)'
+
+  t_OR = r'\|'
+  t_ONE_OR_MORE = r'\+'
+  t_ZERO_OR_MORE = r'\*'
+  t_ZERO_OR_ONE = r'\?'
+
+  t_ANY = r'\.'
+
+  t_LITERAL = r'.'
+
+  def t_CLASS_BEGIN(self, t):
+    r'\['
+    self.lexer.push_state('class')
+    return t
+
+  def t_class_CLASS_END(self, t):
+    r'\]'
+    self.lexer.pop_state()
+    return t
+
+  t_class_RANGE = '-'
+  t_class_NOT = '\^'
+
+  def t_class_ESCAPED_CLASS_LITERAL(self, t):
+    r'\\\^|\\-'
+    t.type = 'CLASS_LITERAL'
+    t.value = t.value[1:]
+    return t
+
+  t_class_CLASS_LITERAL = r'[a-zA-Z]' # fix this
+
+  t_ANY_ignore  = '\n'
+
+  def t_ANY_error(self, t):
+    raise Exception("Illegal character '%s'" % t.value[0])
+
+  def build(self, **kwargs):
+    self.lexer = lex.lex(module=self, **kwargs)