Update Lexer/Parser to support '#inline' and 'label'

ppapi/generators/idl_lexer.py

 #!/usr/bin/python
 #
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """ Lexer for PPAPI IDL """
 
 #
 # IDL Lexer
@@ skipping 28 matching lines @@
 Option('output', 'Generate output.')
 
 #
 # IDL Lexer
 #
 class IDLLexer(object):
   # 'tokens' is a value required by lex which specifies the complete list
   # of valid token types.
   tokens = [
     # Symbol and keywords types
       'COMMENT',

[sehr (please use chromium), 2011/06/29 16:43:06]

should 'ATTRIBUTE' be here also?

[noelallen1, 2011/06/29 18:59:01]

An extended attribute is a grammar component consisting of three or more lexer
tokens in a form like:
  '[' SYMBOL ']'

'[' and ']' are literal tokens listed a few lines bellow in the member variable
"literals", so they do not have a regex t_XXX function like other tokens.

       'DESCRIBE',
       'ENUM',
+      'LABEL',
       'SYMBOL',
+      'INLINE',
       'INTERFACE',
       'STRUCT',
       'TYPEDEF',
 
     # Data types
       'FLOAT',
       'OCT',
       'INT',
       'HEX',
       'STRING',
 
     # Operators
       'LSHIFT'
   ]
 
   # 'keywords' is a map of string to token type.  All SYMBOL tokens are
   # matched against keywords, to determine if the token is actually a keyword.
   keywords = {
+    'attribute' : 'ATTRIBUTE',
     'describe' : 'DESCRIBE',
     'enum'  : 'ENUM',
+    'label' : 'LABEL',
     'interface' : 'INTERFACE',
     'readonly' : 'READONLY',
     'struct' : 'STRUCT',
     'typedef' : 'TYPEDEF',
   }
 
   # 'literals' is a value expected by lex which specifies a list of valid
   # literal tokens, meaning the token type and token value are identical.
   literals = '"*.(){}[],;:=+-'
 
@@ skipping 25 matching lines @@
     r'"[^"]*"'
     t.value = t.value[1:-1]
     self.AddLines(t.value.count('\n'))
     return t
 
   # A C or C++ style comment:  /* xxx */ or //
   def t_COMMENT(self, t):
     r'(/\*(.|\n)*?\*/)|(//.*)'
     self.AddLines(t.value.count('\n'))
 
-    # C++ comments should keep the newline
-    if t.value[:2] == '//': t.value += '\n'
+    # Remove comment markers
+    if t.value[:2] == '//':
+      # For C++ style, remove the preceeding '//'

[sehr (please use chromium), 2011/06/29 16:43:06]

s/preceeding/preceding/

[noelallen1, 2011/06/29 18:59:01]

Done.

+      t.value = t.value[2:].rstrip()
+    else:
+      # For C style, remove ending '*/''
+      lines = []
+      for line in t.value[:-2].split('\n'):
+        # Remove characters until start marker for this line '*' if found
+        # otherwise it should be blank.
+        offs = line.find('*')
+        if offs >= 0:
+          line = line[offs + 1:].rstrip()
+        else:
+          line = ''
+        lines.append(line)
+      t.value = '\n'.join(lines)
+    return t
+
+  # Return a "preprocessor" inline block
+  def t_INLINE(self, t):
+    r'\#inline (.|\n)*\#endinl.*'
+    self.AddLines(t.value.count('\n'))
     return t
 
   # A symbol or keyword.
   def t_KEYWORD_SYMBOL(self, t):
     r'[A-Za-z][A-Za-z_0-9]*'
 
     #All non-keywords are assumed to be symbols
     t.type = self.keywords.get(t.value, 'SYMBOL')
     return t
 
   def t_ANY_error(self, t):
+    msg = "Unrecognized intput"

[sehr (please use chromium), 2011/06/29 16:43:06]

s/intput/input/

[noelallen1, 2011/06/29 18:59:01]

Done.

     line = self.lexobj.lineno
+
+    # If that line has not been accounted for, then we must have hit
+    # EoF, so compute the beginning of the that caused the problem.

[sehr (please use chromium), 2011/06/29 16:43:06]

the what that?

[noelallen1, 2011/06/29 18:59:01]

Done.

+    if line >= len(self.index):
+      # Find the offset in the line of the first word causing the issue
+      word = t.value.split()[0]
+      offs = self.lines[line - 1].find(word)
+      # Add the computed line's starting possition

[sehr (please use chromium), 2011/06/29 16:43:06]

s/possition/position/

[noelallen1, 2011/06/29 18:59:01]

Done.

+      self.index.append(self.lexobj.lexpos - offs)
+      msg = "Unexpected EoF reached after"
+
     pos = self.lexobj.lexpos - self.index[line]
     file = self.lexobj.filename
-    out = self.ErrorMessage(file, line, pos, "Unrecognized input")
+    out = self.ErrorMessage(file, line, pos, msg)
     sys.stderr.write(out + '\n')
 
   def AddLines(self, count):
     # Set the lexer position for the beginning of the next line.  In the case
     # of multiple lines, tokens can not exist on any of the lines except the
     # last one, so the recorded value for previous lines are unused.  We still
     # fill the array however, to make sure the line count is correct.
     self.lexobj.lineno += count
     for i in range(count):
       self.index.append(self.lexobj.lexpos)
@@ skipping 62 matching lines @@
 
 #
 # TestSame
 #
 # From a set of token values, generate a new source text by joining with a
 # single space.  The new source is then tokenized and compared against the
 # old set.
 #
 def TestSame(values):
   src1 = ' '.join(values)
-  src2 = ' '.join(TextToTokens(src1))
+  values2 = TextToTokens(src1)
+  src2 = ' '.join(values2)
+
+  for i in range(len(values)):
+    if values[i] != values2[i]:
+      print "%d >>%s<< >>%s<<" % (i, values[i], values2[i])
 
   if GetOption('output'):
     sys.stdout.write('Generating original.txt and tokenized.txt\n')
     open('original.txt', 'w').write(src1)
     open('tokenized.txt', 'w').write(src2)
 
   if src1 == src2:
     sys.stdout.write('Same: Pass\n')
     return 0
 
+  print "****************\n%s\n%s***************\n" % (src1, src2)
   sys.stdout.write('Same: Failed\n')
   return -1
 
 
 #
 # TestExpect
 #
 # From a set of tokens pairs, verify the type field of the second matches
 # the value of the first, so that:
 # INT 123 FLOAT 1.1
@@ skipping 38 matching lines @@
         return -1
     return 0
 
   except lex.LexError as le:
     sys.stderr.write('%s\n' % str(le))
   return -1
 
 if __name__ == '__main__':
   sys.exit(Main(sys.argv[1:]))