Add very basic CSS3 Syntax compatible tokenizer

Source/core/css/parser/CSSToken.h

Index: Source/core/css/parser/CSSToken.h
diff --git a/Source/core/css/parser/CSSToken.h b/Source/core/css/parser/CSSToken.h
new file mode 100644
index 0000000000000000000000000000000000000000..34800123f29390c7f4a1a236bf9d4ea48ad49569
--- /dev/null
+++ b/Source/core/css/parser/CSSToken.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2013 Google Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef CSSToken_h
+#define CSSToken_h
+
+#include "wtf/text/WTFString.h"
+
+namespace WebCore {
+
+enum CSSTokenType {
+    IdentToken,
+    FunctionToken,
+    AtKeywordToken,
+    HashToken,
+    StringToken,
+    BadStringToken,
+    URLToken,
+    BadURLToken,
+    DelimToken,
+    NumberToken,
+    PercentageToken,
+    DimensionToken,
+    UnicodeRangeToken,
+    IncludeMatchToken,
+    DashMatchToken,
+    PrefixMatchToken,
+    SuffixMatchToken,
+    SubstringMatchToken,
+    ColumnToken,
+    WhitespaceToken,
+    CDOToken,
+    CDCToken,
+    ColonToken,
+    SemicolonToken,
+    CommaToken,
+    LeftBracketToken,
+    RightBracketToken,
+    LeftParenToken,
+    RightParenToken,
+    LeftBraceToken,
+    RightBraceToken,
+    EOFToken,
+};
+
+// Probably should just be rolled into TokenType.
+enum HashTokenType {
+    IdHashToken,
+    UnrestrictedHashToken,
+};
+
+// Probably should just be rolled into TokenType.
+enum NumericValueType {
+    IntegerValueType,
+    NumberValueType,
+};
+
+class CSSToken {
+public:
+    CSSToken(CSSTokenType);

[abarth-chromium, 2014/01/01 18:47:51]

Please mark one-argument constructors explicit

+    CSSToken(CSSTokenType, String);
+
+    CSSToken(CSSTokenType, UChar); // for DelimToken
+    CSSToken(CSSTokenType, String, HashTokenType); // for HashToken
+    CSSToken(CSSTokenType, String, double, NumericValueType); // for NumberToken
+
+    // Converts NumberToken to DimensionToken.
+    void convertToDimensionWithUnit(String);
+
+    // Converts NumberToken to PercentageToken.
+    void convertToPercentage();
+
+    CSSTokenType type() const { return m_type; }
+
+    // FIXME: Most of these may not be needed, only to make things compile atm.
+    HashTokenType hashType() const { return m_hashTokenType; }
+    UChar delimeter() const { return m_delimeter; }
+    NumericValueType numericValueType() const { return m_numericValueType; }
+    double numericValue() const { return m_numericValue; }
+    String unitType() const { return m_unit; }
+
+private:
+    CSSTokenType m_type;
+    String m_value;
+    HashTokenType m_hashTokenType;
+
+    UChar m_delimeter; // Could be rolled into m_value?
+
+    NumericValueType m_numericValueType;
+    double m_numericValue;
+    String m_unit; // FIXME: Is this needed?
+
+    // UnicodeRangeToken:
+    // unsigned m_start;
+    // unsigned m_end;
+};
+
+}
+
+#endif // CSSToken_h