Convert MediaQueryTokenizer's codepoints array to be static data

Source/build/scripts/make_mediaquery_tokenizer_codepoints.py

+#!/usr/bin/env python
+
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import in_generator
+import sys
+
+CPP_TEMPLATE = """
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Auto-generated by make_mediaquery_tokenizer_codepoints.py
+
+const MediaQueryTokenizer::CodePoint MediaQueryTokenizer::codePoints[{array_size}] = {{
+{token_lines}
+}};
+const unsigned codePointsNumber = {array_size};
+"""
+
+
+def token_type(i):
+    codepoints = {'(': "leftParenthesis",

[Nils Barth (inactive), 2014/03/25 02:51:46]

nit: single quote for strings throughout
(at least be consistent, single quote preferred; no char/string distinction)

+                  ')': "rightParenthesis",
+                  '+': "plusOrFullStop",
+                  '.': "plusOrFullStop",
+                  '-': "hyphenMinus",
+                  ',': "comma",
+                  '/': "solidus",
+                  '\\': "reverseSolidus",
+                  ':': "colon",
+                  ';': "semiColon",
+                  }
+    whitespace = ['\n', '\r', '\t', '\f', ' ']

[Nils Barth (inactive), 2014/03/25 02:51:46]

You can actually make this a single string:
whitespace = '\n\r\t\f '
...as Python's |in| functions as a substring test:
http://docs.python.org/2/library/stdtypes.html#index-22

+    c = chr(i)
+    if c in whitespace:
+        return "whiteSpace"
+    if c.isdigit():
+        return "asciiDigit"
+    if c.isalpha() or c == '_':
+        return "nameStart"
+    if i == 0:
+        return "endOfFile"
+    return codepoints.get(c)
+
+
+class MakeMediaQueryTokenizerCodePointsWriter(in_generator.Writer):
+    defaults = {
+        'Conditional': None,
+        'RuntimeEnabled': None,
+        'ImplementedAs': None,
+    }
+    filters = {
+    }
+    default_parameters = {
+        'namespace': '',
+        'export': '',
+    }
+
+    def __init__(self, in_file_path):
+        super(MakeMediaQueryTokenizerCodePointsWriter, self).__init__(in_file_path)
+
+        self._outputs = {
+            ('MediaQueryTokenizerCodepoints.cpp'): self.generate,
+        }
+        self._template_context = {
+            'namespace': '',
+            'export': '',
+        }
+
+    def generate(self):
+        array_size = 128  # SCHAR_MAX + 1
+        token_lines = ["    &MediaQueryTokenizer::%s," % token_type(i) if token_type(i) else "    0," for i in range(array_size)]

[Nils Barth (inactive), 2014/03/25 02:51:46]

This would be more legible with line breaks:
token_lines = ["    &MediaQueryTokenizer::%s," % token_type(i)
               if token_type(i) else "    0,"
               for i in range(array_size)]

+        return CPP_TEMPLATE.format(array_size=array_size, token_lines="\n".join(token_lines))
+
+if __name__ == "__main__":
+    in_generator.Maker(MakeMediaQueryTokenizerCodePointsWriter).main(sys.argv)