Convert MediaQueryTokenizer's codepoints array to be static data

Source/build/scripts/make_mediaquery_tokenizer_codepoints.py

Index: Source/build/scripts/make_mediaquery_tokenizer_codepoints.py
diff --git a/Source/build/scripts/make_mediaquery_tokenizer_codepoints.py b/Source/build/scripts/make_mediaquery_tokenizer_codepoints.py
new file mode 100755
index 0000000000000000000000000000000000000000..060a950670ffd94d40562345ce9b68acb3745153
--- /dev/null
+++ b/Source/build/scripts/make_mediaquery_tokenizer_codepoints.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import in_generator
+import sys
+
+CPP_START_TEMPLATE = """
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Auto-generated by make_mediaquery_tokenizer_codepoints.py
+
+const MediaQueryTokenizer::CodePoint MediaQueryTokenizer::codePoints[%d] = {
+"""
+
+CPP_FOOTER_TEMPLATE = """    };
+const unsigned codePointsNumber = %d;
+"""
+
+
+class MakeMediaQueryTokenizerCodePointsWriter(in_generator.Writer):
+    defaults = {
+        'Conditional': None,
+        'RuntimeEnabled': None,
+        'ImplementedAs': None,
+    }
+    filters = {
+    }
+    default_parameters = {
+        'namespace': '',
+        'export': '',
+    }
+
+    def __init__(self, in_file_path):
+        super(MakeMediaQueryTokenizerCodePointsWriter, self).__init__(in_file_path)
+
+        self._outputs = {
+            ('MediaQueryTokenizerCodepoints.cpp'): self.generate,
+        }
+        self._template_context = {
+            'namespace': '',
+            'export': '',
+        }
+
+    def generate(self):
+        array_size = 128  # SCHAR_MAX + 1
+        output_string = CPP_START_TEMPLATE % array_size
+        for i in range(array_size):
+            c = chr(i)
+            if c in ['\n', '\r', '\t', '\f', ' ']:

[Nils Barth (inactive), 2014/03/24 01:34:23]

This would be rather clearer and less redundant if the content was factored out
into a function:
def token_type(i):
    c = chr(i)
    if c in ...:
        return 'whiteSpace'
    ...
    return None

...and then for the main loop:
token_type = token(i)
value = '&MediaQueryTokenizer::whiteSpace%s' % token_type if token_type else '0'
output_string += '        %s,\n' % value

Also, slightly slicker than header/footer templates would be a single format,
call it CPP_TEMPLATE (with {array_size} and {token_lines}),
and then fill it in with
return CPP_TEMPLATE.format(array_size=array_size,
token_lines='\n'.join(token_lines))
...and indeed make token_lines a list comprehension.

Indeed, the overall output file would be clearer if this were a Jinja template,
though perhaps that's overkill.

[Timothy Loh, 2014/03/24 03:22:19]

Also I'll add that I'd write the function as:

if c.isspace():
    return "whiteSpace"
if c.isdigit():
    return "asciiDigit"
if c.isalpha() or c == '_':
    return "nameStart"

return {
    '(': leftParenthesis',
    ')': 'rightParenthesis',
    '+': 'plusOrFullStop',
    '.': 'plusOrFullStop',
    '-': 'hyphenMinus',
    ',': 'comma',
    '/': 'solidus',
    ':': 'colon',
    ';': 'semiColon',
    '\\': 'reverseSolidus',
}.get(c)

[Nils Barth (inactive), 2014/03/24 03:24:27]

<snip>

Good point - even better!

+                output_string += "        &MediaQueryTokenizer::whiteSpace,\n"
+            elif c == '(':
+                output_string += "        &MediaQueryTokenizer::leftParenthesis,\n"
+            elif c == ')':
+                output_string += "        &MediaQueryTokenizer::rightParenthesis,\n"
+            elif c in ['+', '.']:
+                output_string += "        &MediaQueryTokenizer::plusOrFullStop,\n"
+            elif c == '-':
+                output_string += "        &MediaQueryTokenizer::hyphenMinus,\n"
+            elif c == ',':
+                output_string += "        &MediaQueryTokenizer::comma,\n"
+            elif c == '/':
+                output_string += "        &MediaQueryTokenizer::solidus,\n"
+            elif c == ':':
+                output_string += "        &MediaQueryTokenizer::colon,\n"
+            elif c == ';':
+                output_string += "        &MediaQueryTokenizer::semiColon,\n"
+            elif c == '\\':
+                output_string += "        &MediaQueryTokenizer::reverseSolidus,\n"
+            elif c >= '0' and c <= '9':
+                output_string += "        &MediaQueryTokenizer::asciiDigit,\n"
+            elif (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or c == '_':
+                output_string += "        &MediaQueryTokenizer::nameStart,\n"
+            elif i == 0:
+                output_string += "        &MediaQueryTokenizer::endOfFile,\n"
+            else:
+                output_string += "        0,\n"
+
+        output_string += CPP_FOOTER_TEMPLATE % array_size
+
+        return output_string
+
+if __name__ == "__main__":
+    in_generator.Maker(MakeMediaQueryTokenizerCodePointsWriter).main(sys.argv)