IDL parser rewrite in Python

Source/bindings/scripts/blink_idl_parser.py

+# Copyright (C) 2013 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Parser for Blink IDL
+
+The parser uses the PLY (Python Lex-Yacc) library to build a set of parsing
+rules which understand the Blink dialect of Web IDL.
+It derives from a standard Web IDL parser, overriding the rules where Blink
+IDL differs.
+
+Web IDL:
+    http://www.w3.org/TR/WebIDL/
+Web IDL Grammar:
+    http://www.w3.org/TR/WebIDL/#idl-grammar
+PLY:
+    http://www.dabeaz.com/ply/
+"""
+
+# Disable check for line length and Member as Function due to how grammar rules
+# are defined with PLY
+#
+# pylint: disable=R0201
+# pylint: disable=C0301
+#
+# Disable attribute validation, as lint can't import parent class to check
+# pylint: disable=E1101
+
+import os.path
+import sys
+
+# PLY is in Chromium src/third_party/ply
+module_path, module_name = os.path.split(__file__)
+third_party = os.path.join(module_path, os.pardir, os.pardir, os.pardir, os.pardir)
+sys.path.append(third_party)
+from ply import yacc
+
+# Base parser is in Chromium src/tools/idl_parser
+tools_dir = os.path.join(module_path, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'tools')
+sys.path.append(tools_dir)
+from idl_parser.idl_parser import IDLParser, ListFromConcat
+
+from blink_idl_lexer import BlinkIDLLexer
+
+
+REMOVED_RULES = ['Comments',  # [0.1]
+                 'CommentsRest',  # [0.2]
+                 'AttributeOrOperation',  # [30]
+                 'StringifierAttributeOrOperation',   # [31]
+                 'Operation',   # [35]
+                 'Qualifiers',  # [36]
+                ]
+
+
+class BlinkIDLParser(IDLParser):
+    # Numbering scheme is:
+    # [1] for Web IDL spec (or base Pepper IDL)
+    # [b1] for Blink IDL (overrides Web IDL or inserts rule)
+    # [s1] is for numbering Sakamoto
+
+    # [0] Override Pepper definition, since we strip comments
+    def p_Top(self, p):
+        """Top : Definitions"""
+        p[0] = p[1]
+
+    # [3] Override Pepper action, since we distinguish callbacks
+    def p_CallbackOrInterface(self, p):
+        """CallbackOrInterface : CALLBACK CallbackRestOrInterface
+                               | Interface"""
+        if len(p) > 2:
+            p[2].AddChildren(self.BuildTrue('Callback'))
+            p[0] = p[2]
+        else:
+            p[0] = p[1]
+
+    # [b10]
+    def p_InterfaceMember(self, p):
+        """InterfaceMember : Const
+                           | AttributeOrOperationOrIterator"""
+        # Standard is (no 'OrIterator'):
+        # InterfaceMember : Const
+        #                 | AttributeOrOperation
+        p[0] = p[1]
+
+    # [b19]
+    # List needed for multiple inheritance in SVG
+    # FIXME: remove when multiple inheritance (which is deprecated) is removed
+    def p_Inheritance(self, p):
+        """Inheritance : ':' ScopedNameList
+                       |"""
+        # Standard is (single identifier, not list):
+        # """Inheritance : ':' identifier
+        #                |"""
+        if len(p) > 1:
+            p[0] = self.BuildProduction('MultipleInherit', p, 2, p[2])
+
+    # [b27]
+    def p_ConstValue(self, p):
+        """ConstValue : BooleanLiteral
+                      | FloatLiteral
+                      | IntegerLiteral
+                      | StringLiteral
+                      | null"""
+        # Standard is (no 'string', fewer 'Literals'):
+        # ConstValue : BooleanLiteral
+        #            | FloatLiteral
+        #            | integer
+        #            | NULL
+        p[0] = p[1]
+
+    # [b27.1]
+    def p_IntegerLiteral(self, p):
+        """IntegerLiteral : integer"""
+        p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'integer'),
+                              self.BuildAttribute('NAME', p[1]))
+
+    # [b27.2]
+    def p_StringLiteral(self, p):
+        """StringLiteral : string"""
+        p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'DOMString'),
+                              self.BuildAttribute('NAME', p[1]))
+
+    # [b30]
+    # Replaces [30] AttributeOrOperation
+    def p_AttributeOrOperationOrIterator(self, p):
+        """AttributeOrOperationOrIterator : Serializer
+                                          | Qualifier AttributeOrOperationRest
+                                          | Attribute
+                                          | OperationOrIterator"""
+        # Standard is:
+        # AttributeOrOperation : "stringifier" StringifierAttributeOrOperation
+        #                      | Attribute
+        #                      | Operation
+        if len(p) > 2:
+            p[2].AddChildren(p[1])
+            p[0] = p[2]
+        else:
+            p[0] = p[1]
+
+    # Serialization
+    # FIXME: can we remove??
+    # [s31]
+    def p_Serializer(self, p):
+        """Serializer : SERIALIZER SerializerRest"""
+        p[0] = p[1]
+
+    # [s32]
+    def p_SerializerRest(self, p):
+        """SerializerRest : OperationRest
+                          | '=' SerializationPattern
+                          |"""
+        if len(p) > 2:
+            p[0] = p[2]
+        elif len(p) == 2:
+            p[0] = p[1]
+
+    # [s33]
+    def p_SerializationPattern(self, p):
+        """SerializationPattern : '{' SerializationPatternMap '}'
+                                | '[' SerializationPatternList ']'
+                                | identifier"""
+        if len(p) == 4:
+            p[0] = p[2]
+        else:
+            p[0] = p[1]
+
+    # [s34]
+    def p_SerializationPatternMap(self, p):
+        """SerializationPatternMap : GETTER
+                                   | INHERIT Identifiers
+                                   | identifier Identifiers
+                                   |"""
+        if len(p) > 1:
+            p[0] = p[1]
+
+    # [s35]
+    def p_SerializationPatternList(self, p):
+        """SerializationPatternList : GETTER
+                                    | identifier Identifiers
+                                    |"""
+        if len(p) > 1:
+            p[0] = p[1]
+
+    # [s36]
+    def p_Identifiers(self, p):
+        """Identifiers : ',' identifier Identifiers
+                       |"""
+        if len(p) > 1:
+            p[0] = p[2]
+
+    # Iterator and SpecialOperation
+    # Follows [35] Operation
+    # [b35.1]
+    def p_OperationOrIterator(self, p):
+        """OperationOrIterator : ReturnType OperationOrIteratorRest
+                               | SpecialOperation"""
+        if len(p) > 2:
+            p[0] = self.BuildProduction('OperationOrIterator', p, 2, ListFromConcat(p[1], p[2]))
+        else:
+            p[0] = p[1]
+
+    # [b35.2]
+    def p_OperationOrIteratorRest(self, p):
+        """OperationOrIteratorRest : IteratorRest
+                                   | OperationRest"""
+        p[0] = p[1]
+
+    # [b35.3]
+    def p_IteratorRest(self, p):
+        """IteratorRest : ITERATOR OptionalIteratorInterfaceOrObject ';'"""
+        p[0] = p[2]
+
+    # [b35.4]
+    def p_OptionalIteratorInterfaceOrObject(self, p):
+        """OptionalIteratorInterfaceOrObject : OptionalIteratorInterface
+                                             | OBJECT"""
+        p[0] = p[1]
+
+    # [b35.5]
+    def p_OptionalIteratorInterface(self, p):
+        """OptionalIteratorInterface : '=' identifier
+                                     |"""
+        if len(p) > 2:
+            p[0] = p[2]
+
+    # [b35.6]
+    def p_SpecialOperation(self, p):
+        """SpecialOperation : Special Specials ReturnType OperationRest"""
+        specials = ListFromConcat(p[1], p[2])
+        p[4].AddChildren(p[3])
+        p[0] = p[4]
+        p[0].AddChildren(specials)
+
+    # Follows [36] Qualifiers
+    # [b36.1] [s37]
+    def p_Qualifier(self, p):
+        """Qualifier : STATIC
+                     | STRINGIFIER"""
+        if p[1] == 'static':
+            p[0] = self.BuildTrue('STATIC')
+        elif p[1] == 'stringifier':
+            p[0] = self.BuildTrue('STRINGIFIER')
+
+    # [b39]
+    def p_OperationRest(self, p):
+        """OperationRest : OptionalIdentifier '(' ArgumentList ')' Raises ';'"""
+        # Standard is:
+        # OperationRest : ReturnType OptionalIdentifier '(' ArgumentList ')' ';'
+        arguments = self.BuildProduction('Arguments', p, 2, p[3])
+        p[0] = self.BuildNamed('Operation', p, 1, arguments)
+
+    # [b39.1]
+    def p_Raises(self, p):
+        """Raises : RAISES ExceptionList
+                  |"""
+        if len(p) > 2:
+            p[0] = p[2]
+
+    # [s39.2]
+    def p_ExceptionList(self, p):
+        """ExceptionList : '(' ScopedNameList ')'"""
+        p[0] = p[2]
+
+    # [b39.3]
+    def p_AttributeOrOperationRest(self, p):
+        """AttributeOrOperationRest : AttributeRest
+                                    | ReturnType OperationRest
+                                    | ';'"""
+        if len(p) > 2:
+            p[2].AddChildren(p[1])
+            p[0] = p[2]
+        elif len(p) == 2 and p[1] != ';':
+            p[0] = p[1]
+
+    # [b39.2]
+    def p_AttributeRest(self, p):
+        """AttributeRest : ReadOnly ATTRIBUTE Type identifier Get ';'"""
+        p[0] = self.BuildNamed('Attribute', p, 4,                                                             ListFromConcat(p[1], p[3], p[5]))
+
+    # [b39.3]
+    def p_Get(self, p):
+        """Get : InheritsGetter SetRaises
+               | SetGetRaises
+               |"""
+        if len(p) > 2:
+            p[0] = p[2]
+        elif len(p) == 2:
+            p[0] = p[1]
+
+    # [s39.4]
+    def p_InheritsGetter(self, p):
+        """InheritsGetter : INHERITS GETTER"""
+        pass
+
+    # [b47]
+    def p_ExceptionMember(self, p):
+        """ExceptionMember : Const
+                           | ExceptionField
+                           | Attribute
+                           | ExceptionFieldToString"""
+        # Standard is:
+        # ExceptionMember : Const
+        #                 | ExceptionField
+        p[0] = p[1]
+
+    # [b47.1]
+    def p_ExceptionFieldToString(self, p):
+        """ExceptionFieldToString : Type identifier '(' ')' ';'"""
+        # Needed to handle:
+        # // Override in a Mozilla compatible format
+        # [NotEnumerable] DOMString toString();
+        p[0] = self.BuildNamed('ExceptionFieldToString', p, 2, p[1])
+
+    # Extended attributes
+    # [b49] Override Pepper: remove comment field, since comments stripped
+    # FIXME: Upstream
+    def p_ExtendedAttributeList(self, p):
+        """ExtendedAttributeList : '[' ExtendedAttribute ExtendedAttributes ']'
+                                 | '[' ']'
+                                 | """
+        if len(p) > 3:
+            items = ListFromConcat(p[2], p[3])
+            attribs = self.BuildProduction('ExtAttributes', p, 1, items)
+            p[0] = ListFromConcat(p[0], attribs)
+
+    # [b50] Allow optional trailing comma
+    def p_ExtendedAttributes(self, p):
+        """ExtendedAttributes : ',' ExtendedAttribute ExtendedAttributes
+                              | ','
+                              |"""
+        if len(p) > 2:
+            p[0] = ListFromConcat(p[2], p[3])
+
+    # [b51] Add ExtendedAttributeIdentOrIdent and ExtendedAttributeIdentAndIdent
+    def p_ExtendedAttribute(self, p):
+        """ExtendedAttribute : ExtendedAttributeNoArgs
+                             | ExtendedAttributeArgList
+                             | ExtendedAttributeIdent
+                             | ExtendedAttributeIdentOrIdent
+                             | ExtendedAttributeIdentAndIdent
+                             | ExtendedAttributeNamedArgList"""
+        p[0] = p[1]
+
+    # FIXME: Upstream UnionType
+    # [59]
+    def p_UnionType(self, p):
+        """UnionType : '(' UnionMemberType OR UnionMemberType UnionMemberTypes ')'"""
+        members = ListFromConcat(p[2], p[4], p[5])
+        p[0] = self.BuildProduction('UnionType', p, 1, members)
+
+    # [60]
+    def p_UnionMemberType(self, p):
+        """UnionMemberType : NonAnyType
+                           | UnionType TypeSuffix
+                           | ANY '[' ']' TypeSuffix"""
+        if len(p) == 2:
+            p[0] = p[1]
+        elif len(p) == 3:
+            p[0] = ListFromConcat(p[1], p[2])
+        else:
+            p[0] = ListFromConcat(self.BuildProduction('Any', p, 1), p[4])
+
+    # [61]
+    def p_UnionMemberTypes(self, p):
+        """UnionMemberTypes : OR UnionMemberType UnionMemberTypes
+                            |"""
+        if len(p) > 1:
+            p[0] = ListFromConcat(p[2], p[3])
+
+    # [70] Override Pepper to remove non-standard sized array
+    # FIXME: upstream should remove sized array from base
+    def p_TypeSuffix(self, p):
+        """TypeSuffix : '[' ']' TypeSuffix
+                      | '?' TypeSuffixStartingWithArray
+                      |"""
+        if len(p) == 4:
+            p[0] = self.BuildProduction('Array', p, 1, p[3])
+
+        if len(p) == 3:
+            p[0] = ListFromConcat(self.BuildTrue('NULLABLE'), p[2])
+
+    # [b76.1]
+    def p_ExtendedAttributeIdentOrIdent(self, p):
+        """ExtendedAttributeIdentOrIdent : identifier '=' identifier '|' identifier"""
+        # Used in a handful of files
+        value = self.BuildAttribute('VALUE', p[3] + '|' + p[5])
+        p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
+
+    # [b76.2]
+    def p_ExtendedAttributeIdentAndIdent(self, p):
+        """ExtendedAttributeIdentAndIdent : identifier '=' identifier '&' identifier"""
+        # FIXME: Used only in NavigatorContentUtils.idl,
+        # which doesn't appear to be parsed - remove?
+        value = self.BuildAttribute('VALUE', p[3] + '&' + p[5])
+        p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
+
+    # Exceptions
+    # [s25]
+    def p_SetGetRaises(self, p):
+        """SetGetRaises : SetRaises GetRaises2
+                        | GetRaises
+                        | SetGetRaises2"""
+        p[0] = p[1]
+
+    # [s27]
+    def p_GetRaises(self, p):
+        """GetRaises : GETTER RAISES ExceptionList"""
+        p[0] = p[3]
+
+    # [s27.5]
+    def p_GetRaises2(self, p):
+        """GetRaises2 : ',' GETTER RAISES ExceptionList
+                      |"""
+        p[0] = p[4]
+
+    # [s28]
+    def p_SetRaises(self, p):
+        """SetRaises : SETTER RAISES ExceptionList"""
+        p[0] = p[3]
+
+    # [s29]
+    def p_SetGetRaises2(self, p):
+        """SetGetRaises2 : GetRaises3 SetRaises3"""
+        # FIXME
+        p[0] = p[1]
+
+    # [s29a]
+    def p_GetRaises3(self, p):
+        """GetRaises3 : GETRAISES ExceptionList
+                      |"""
+        p[0] = p[2]
+
+    # [s29b]
+    def p_SetRaises3(self, p):
+        """SetRaises3 : SETRAISES ExceptionList
+                      |"""
+        p[0] = p[2]
+
+    # Scoped names
+    # FIXME: remove??
+    # [s63]
+    def p_ScopedNameList(self, p):
+        """ScopedNameList : ScopedName ScopedNames"""
+        scoped_name = self.BuildNamed('ScopedName', p, 1)
+        p[0] = ListFromConcat(scoped_name, p[2])
+
+    # [s64]
+    def p_ScopedNames(self, p):
+        """ScopedNames : ',' ScopedName ScopedNames
+                       |"""
+        if len(p) > 1:
+            scoped_name = self.BuildNamed('ScopedName', p, 2)
+            p[0] = ListFromConcat(scoped_name, p[3])
+
+    # [s65]
+    def p_ScopedName(self, p):
+        """ScopedName : AbsoluteScopedName
+                      | RelativeScopedName"""
+        p[0] = p[1]
+
+    # [s66]
+    def p_AbsoluteScopedName(self, p):
+        """AbsoluteScopedName : SCOPE_RESOLUTION identifier ScopedNameParts"""
+        p[0] = p[2]
+
+    # [s67]
+    def p_RelativeScopedName(self, p):
+        """RelativeScopedName : identifier ScopedNameParts"""
+        p[0] = p[1]
+
+    # [s68]
+    def p_ScopedNameParts(self, p):
+        """ScopedNameParts : SCOPE_RESOLUTION identifier ScopedNameParts
+                           |"""
+        # if len(p) > 1:
+        #     p[0] = p[2]
+        pass
+
+    def __dir__(self):
+        # Remove rules from listing so yacc doesn't parse them
+        keys = set(self.__dict__.keys() + dir(self.__class__))
+        for rule in REMOVED_RULES:
+            keys.remove('p_' + rule)
+        return list(keys)
+
+    def __init__(self, lexer, verbose=False, debug=False, mute_error=False):
+        self.lexer = lexer
+        self.tokens = lexer.KnownTokens()
+        # SLR yields faster table generation (same output), ok b/c LL(1) grammar
+        self.yaccobj = yacc.yacc(module=self, debug=debug, method='SLR')
+        # Optimized mode, improves startup time of yacc compiler
+        # optimize=1
+        # Turn on table generation with:
+        # write_tables=1
+        # Turn off table generation with:
+        # tabmodule=None, write_tables=0
+        self.parse_debug = debug
+        self.verbose = verbose
+        self.mute_error = mute_error
+        self._parse_errors = 0
+        self._parse_warnings = 0
+        self._last_error_msg = None
+        self._last_error_lineno = 0
+        self._last_error_pos = 0
+
+
+# If run by itself, attempt to build the parser
+if __name__ == '__main__':
+    parser = BlinkIDLParser(BlinkIDLLexer())