Pre-cache PLY lex table and use optimized mode (5% build time improvement)

Source/bindings/scripts/blink_idl_lexer.py

 # Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 36 matching lines @@
 PLY:
     http://www.dabeaz.com/ply/
 """
 
 # Disable attribute validation, as lint can't import parent class to check
 # pylint: disable=E1101
 
 import os.path
 import sys
 
+# PLY is in Chromium src/third_party/ply
+module_path, module_name = os.path.split(__file__)
+third_party = os.path.join(module_path, os.pardir, os.pardir, os.pardir, os.pardir)
+# Insert at front to override system libraries, and after path[0] == script dir
+sys.path.insert(1, third_party)
+from ply import lex
+
 # Base lexer is in Chromium src/tools/idl_parser
-module_path, module_name = os.path.split(__file__)
-tools_dir = os.path.join(module_path, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, 'tools')
+tools_dir = os.path.join(third_party, os.pardir, 'tools')
 sys.path.append(tools_dir)
-
 from idl_parser.idl_lexer import IDLLexer
 
 REMOVE_TOKENS = ['COMMENT']
 
 
 class BlinkIDLLexer(IDLLexer):
     # ignore comments
     def t_COMMENT(self, t):
         r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)'
         self.AddLines(t.value.count('\n'))
 
     # Analogs to _AddToken/_AddTokens in base lexer
     # Needed to remove COMMENT token, since comments ignored
     def _RemoveToken(self, token):
         if token in self.tokens:
             self.tokens.remove(token)
 
     def _RemoveTokens(self, tokens):
         for token in tokens:
             self._RemoveToken(token)
 
-    def __init__(self):
+    def __init__(self, debug=False, outputdir=None, optimize=True):

[haraken, 2014/03/06 07:58:41]

Do we need to distinguish |debug| and |optimize| ?

[Nils Barth (inactive), 2014/03/06 08:09:47]

Yes, in general: these 3 are possible: FT, FF, TF
optimize reads/writes the lex table, and skips error checking;
debug turns on logging.
You may want to turn off optimize but have no debug if you don't want
any extra files created (no lextab.py file or debug file).
Also, this makes it consistent with the underlying lex library
(see the lex.lex line below).

+        if debug:
+            # Turn off optimization and caching to help debugging
+            optimize = False
+            outputdir = None
+        if outputdir:
+            # Need outputdir in path because lex imports the cached lex table
+            # as a Python module
+            sys.path.append(outputdir)
+
         IDLLexer.__init__(self)
+        # Overrides to parent class
         self._RemoveTokens(REMOVE_TOKENS)
+        # Optimized mode substantially decreases startup time (by disabling
+        # error checking), and also allows use of Python's optimized mode.
+        # See: Optimized Mode
+        # http://www.dabeaz.com/ply/ply.html#ply_nn15
+        self._lexobj = lex.lex(object=self,
+                               debug=debug,
+                               optimize=optimize,
+                               outputdir=outputdir)
 
 
-# If run by itself, attempt to build the lexer
+################################################################################
+
+def main(argv):
+    # If file itself executed, build and cache lex table
+    try:
+        outputdir = argv[1]
+    except IndexError as err:
+        print 'Usage: %s OUTPUT_DIR' % argv[0]
+        return 1
+    lexer = BlinkIDLLexer(outputdir=outputdir)
+
+
 if __name__ == '__main__':
-    lexer = BlinkIDLLexer()
+    sys.exit(main(sys.argv))