Update Lexer/Parser to support '#inline' and 'label'

ppapi/generators/idl_lexer.py

Index: ppapi/generators/idl_lexer.py
===================================================================
--- ppapi/generators/idl_lexer.py	(revision 90352)
+++ ppapi/generators/idl_lexer.py	(working copy)
@@ -49,7 +49,9 @@
       'COMMENT',
       'DESCRIBE',
       'ENUM',
+      'LABEL',
       'SYMBOL',
+      'INLINE',
       'INTERFACE',
       'STRUCT',
       'TYPEDEF',
@@ -68,8 +70,10 @@
   # 'keywords' is a map of string to token type.  All SYMBOL tokens are
   # matched against keywords, to determine if the token is actually a keyword.
   keywords = {
+    'attribute' : 'ATTRIBUTE',
     'describe' : 'DESCRIBE',
     'enum'  : 'ENUM',
+    'label' : 'LABEL',
     'interface' : 'INTERFACE',
     'readonly' : 'READONLY',
     'struct' : 'STRUCT',
@@ -114,9 +118,12 @@
   def t_COMMENT(self, t):
     r'(/\*(.|\n)*?\*/)|(//.*)'
     self.AddLines(t.value.count('\n'))
+    return t
 
-    # C++ comments should keep the newline
-    if t.value[:2] == '//': t.value += '\n'
+  # Return a "preprocessor" inline block
+  def t_INLINE(self, t):
+    r'\#inline (.|\n)*\#endinl.*'
+    self.AddLines(t.value.count('\n'))
     return t
 
   # A symbol or keyword.
@@ -128,10 +135,22 @@
     return t
 
   def t_ANY_error(self, t):
+    msg = "Unrecognized input"
     line = self.lexobj.lineno
+
+    # If that line has not been accounted for, then we must have hit
+    # EoF, so compute the beginning of the line that caused the problem.
+    if line >= len(self.index):
+      # Find the offset in the line of the first word causing the issue
+      word = t.value.split()[0]
+      offs = self.lines[line - 1].find(word)
+      # Add the computed line's starting position
+      self.index.append(self.lexobj.lexpos - offs)
+      msg = "Unexpected EoF reached after"
+
     pos = self.lexobj.lexpos - self.index[line]
     file = self.lexobj.filename
-    out = self.ErrorMessage(file, line, pos, "Unrecognized input")
+    out = self.ErrorMessage(file, line, pos, msg)
     sys.stderr.write(out + '\n')
 
   def AddLines(self, count):
@@ -189,6 +208,17 @@
       outlist.append(t)
   return outlist
 
+
+def TokensFromText(text):
+  lexer = IDLLexer()
+  lexer.SetData('unknown', text)
+  outlist = []
+  while 1:
+    t = lexer.lexobj.token()
+    if t is None: break
+    outlist.append(t.value)
+  return outlist
+
 #
 # TextToTokens
 #
@@ -212,19 +242,32 @@
 # single space.  The new source is then tokenized and compared against the
 # old set.
 #
-def TestSame(values):
-  src1 = ' '.join(values)
-  src2 = ' '.join(TextToTokens(src1))
+def TestSame(values1):
+  # Recreate the source from the tokens.  We use newline instead of whitespace
+  # since the '//' and #inline regex are line sensitive.
+  text = '\n'.join(values1)
+  values2 = TextToTokens(text)
 
+  count1 = len(values1)
+  count2 = len(values2)
+  if count1 != count2:
+    print "Size mismatch original %d vs %d\n" % (count1, count2)
+    if count1 > count2: count1 = count2
+
+  for i in range(count1):
+    if values1[i] != values2[i]:
+      print "%d >>%s<< >>%s<<" % (i, values1[i], values2[i])
+
   if GetOption('output'):
     sys.stdout.write('Generating original.txt and tokenized.txt\n')
     open('original.txt', 'w').write(src1)
     open('tokenized.txt', 'w').write(src2)
 
-  if src1 == src2:
+  if values1 == values2:
     sys.stdout.write('Same: Pass\n')
     return 0
 
+  print "****************\n%s\n%s***************\n" % (src1, src2)
   sys.stdout.write('Same: Failed\n')
   return -1