| OLD | NEW |
| 1 # Copyright 2013 the V8 project authors. All rights reserved. | 1 # Copyright 2013 the V8 project authors. All rights reserved. |
| 2 # Redistribution and use in source and binary forms, with or without | 2 # Redistribution and use in source and binary forms, with or without |
| 3 # modification, are permitted provided that the following conditions are | 3 # modification, are permitted provided that the following conditions are |
| 4 # met: | 4 # met: |
| 5 # | 5 # |
| 6 # * Redistributions of source code must retain the above copyright | 6 # * Redistributions of source code must retain the above copyright |
| 7 # notice, this list of conditions and the following disclaimer. | 7 # notice, this list of conditions and the following disclaimer. |
| 8 # * Redistributions in binary form must reproduce the above | 8 # * Redistributions in binary form must reproduce the above |
| 9 # copyright notice, this list of conditions and the following | 9 # copyright notice, this list of conditions and the following |
| 10 # disclaimer in the documentation and/or other materials provided | 10 # disclaimer in the documentation and/or other materials provided |
| 11 # with the distribution. | 11 # with the distribution. |
| 12 # * Neither the name of Google Inc. nor the names of its | 12 # * Neither the name of Google Inc. nor the names of its |
| 13 # contributors may be used to endorse or promote products derived | 13 # contributors may be used to endorse or promote products derived |
| 14 # from this software without specific prior written permission. | 14 # from this software without specific prior written permission. |
| 15 # | 15 # |
| 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | 27 |
| 28 import os | 28 import os |
| 29 import sys | 29 import sys |
| 30 import logging |
| 30 import jinja2 | 31 import jinja2 |
| 31 from copy import deepcopy | 32 from copy import deepcopy |
| 32 from dfa import Dfa | 33 from dfa import Dfa |
| 33 from term import Term | 34 from term import Term |
| 34 from transition_key import TransitionKey | 35 from transition_key import TransitionKey |
| 35 | 36 |
| 36 class CodeGenerator: | 37 class CodeGenerator: |
| 37 | 38 |
| 38 def __init__(self, | 39 def __init__(self, |
| 39 rule_processor, | 40 rule_processor, |
| 40 minimize_default = True, | 41 minimize_default = True, |
| 41 inline = True, | 42 inline = True, |
| 42 switching = True, | 43 switching = True, |
| 43 debug_print = False, | 44 debug_print = False): |
| 44 log = False): | |
| 45 if minimize_default: | 45 if minimize_default: |
| 46 dfa = rule_processor.default_automata().minimal_dfa() | 46 dfa = rule_processor.default_automata().minimal_dfa() |
| 47 else: | 47 else: |
| 48 dfa = rule_processor.default_automata().dfa() | 48 dfa = rule_processor.default_automata().dfa() |
| 49 self.__dfa = dfa | 49 self.__dfa = dfa |
| 50 self.__default_action = rule_processor.default_action() | 50 self.__default_action = rule_processor.default_action() |
| 51 self.__debug_print = debug_print | 51 self.__debug_print = debug_print |
| 52 self.__log = log | |
| 53 self.__inline = inline | 52 self.__inline = inline |
| 54 self.__switching = switching | 53 self.__switching = switching |
| 55 self.__jump_table = [] | 54 self.__jump_table = [] |
| 56 | 55 |
| 57 __jump_labels = ['state_entry', 'after_entry_code'] | 56 __jump_labels = ['state_entry', 'after_entry_code'] |
| 58 | 57 |
| 59 @staticmethod | 58 @staticmethod |
| 60 def __transform_state(encoding, state): | 59 def __transform_state(encoding, state): |
| 61 # action data | 60 # action data |
| 62 # generate ordered transitions | 61 # generate ordered transitions |
| (...skipping 310 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 373 end_offset + total_nodes_created, nodes_created, inline_mapping) | 372 end_offset + total_nodes_created, nodes_created, inline_mapping) |
| 374 total_nodes_created += nodes_created + created | 373 total_nodes_created += nodes_created + created |
| 375 return total_nodes_created | 374 return total_nodes_created |
| 376 | 375 |
| 377 def process(self): | 376 def process(self): |
| 378 | 377 |
| 379 self.__build_dfa_states() | 378 self.__build_dfa_states() |
| 380 dfa_states = self.__dfa_states | 379 dfa_states = self.__dfa_states |
| 381 # split transitions | 380 # split transitions |
| 382 switched = reduce(self.__split_transitions, dfa_states, 0) | 381 switched = reduce(self.__split_transitions, dfa_states, 0) |
| 383 if self.__log: | 382 logging.info("%s states use switch (instead of if)" % switched) |
| 384 print "%s states use switch (instead of if)" % switched | |
| 385 # rewrite deferred transitions | 383 # rewrite deferred transitions |
| 386 for state in dfa_states: | 384 for state in dfa_states: |
| 387 self.__rewrite_deferred_transitions(state) | 385 self.__rewrite_deferred_transitions(state) |
| 388 # set nodes to inline | 386 # set nodes to inline |
| 389 if self.__inline: | 387 if self.__inline: |
| 390 inlined = reduce(self.__set_inline, dfa_states, 0) | 388 inlined = reduce(self.__set_inline, dfa_states, 0) |
| 391 if self.__log: | 389 logging.info("%s states inlined" % inlined) |
| 392 print "%s states inlined" % inlined | |
| 393 # rewrite transitions to use jumps | 390 # rewrite transitions to use jumps |
| 394 inlined_nodes = self.__rewrite_transitions_to_jumps(0, len(dfa_states), {}) | 391 inlined_nodes = self.__rewrite_transitions_to_jumps(0, len(dfa_states), {}) |
| 395 if self.__log: | 392 logging.info("%s inlined nodes created" % inlined_nodes) |
| 396 print "%s inlined nodes created" % inlined_nodes | |
| 397 # mark the entry point in case there are implicit jumps to it | 393 # mark the entry point in case there are implicit jumps to it |
| 398 self.__dfa_states[0]['entry_points']['state_entry'] = True | 394 self.__dfa_states[0]['entry_points']['state_entry'] = True |
| 399 | 395 |
| 400 default_action = self.__default_action | 396 default_action = self.__default_action |
| 401 assert default_action | 397 assert default_action |
| 402 | 398 |
| 403 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | 399 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| 404 template_env = jinja2.Environment( | 400 template_env = jinja2.Environment( |
| 405 loader = jinja2.PackageLoader('lexer_generator', '.'), | 401 loader = jinja2.PackageLoader('lexer_generator', '.'), |
| 406 undefined = jinja2.StrictUndefined) | 402 undefined = jinja2.StrictUndefined) |
| 407 template = template_env.get_template('code_generator.jinja') | 403 template = template_env.get_template('code_generator.jinja') |
| 408 | 404 |
| 409 encoding = self.__dfa.encoding() | 405 encoding = self.__dfa.encoding() |
| 410 char_types = {'latin1': 'uint8_t', 'utf16': 'uint16_t', 'utf8': 'int8_t'} | 406 char_types = {'latin1': 'uint8_t', 'utf16': 'uint16_t', 'utf8': 'int8_t'} |
| 411 char_type = char_types[encoding.name()] | 407 char_type = char_types[encoding.name()] |
| 412 | 408 |
| 413 return template.render( | 409 return template.render( |
| 414 start_node_number = 0, | 410 start_node_number = 0, |
| 415 debug_print = self.__debug_print, | 411 debug_print = self.__debug_print, |
| 416 default_action = default_action, | 412 default_action = default_action, |
| 417 dfa_states = dfa_states, | 413 dfa_states = dfa_states, |
| 418 jump_table = self.__jump_table, | 414 jump_table = self.__jump_table, |
| 419 encoding = encoding.name(), | 415 encoding = encoding.name(), |
| 420 char_type = char_type, | 416 char_type = char_type, |
| 421 upper_bound = encoding.primary_range()[1]) | 417 upper_bound = encoding.primary_range()[1]) |
| OLD | NEW |