Add very basic CSS3 Syntax compatible tokenizer

Source/core/css/parser/NewCSSTokenizer.h

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 10 matching lines @@
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef URLRegistry_h
-#define URLRegistry_h
+#ifndef NewCSSTokenizer_h
+#define NewCSSTokenizer_h
 
+#include "core/css/parser/CSSToken.h"
+#include "core/html/parser/InputStreamPreprocessor.h"
 #include "wtf/text/WTFString.h"
 
 namespace WebCore {
 
-class KURL;
-class SecurityOrigin;
-class URLRegistry;
+// The CSS Syntax spec is written as a (fixed size) look-ahead
+// tokenizer instead of a state-machine like HTML
+// so we have our own input stream class. :(
+class CSSInputStream {
+    WTF_MAKE_NONCOPYABLE(CSSInputStream);
+    WTF_MAKE_FAST_ALLOCATED;
+public:
+    CSSInputStream(String input);
 
-class URLRegistrable {
-public:
-    virtual ~URLRegistrable() { }
-    virtual URLRegistry& registry() const = 0;
+    UChar currentInputChar();
+    UChar nextInputChar(); // peek1();
+    UChar peek2();
+    UChar peek3(); // Needed?
+
+    void advance();
+    void pushBack(UChar);
+
+private:
+    size_t m_offset;
+    String m_string;

[abarth-chromium, 2014/01/01 18:47:51]

I would have expected you'd want to templatize this class based on the CharType
and have two copies of the CSS parser in the binary: one for UChar and one for
LChar.  Taking the is8Bit branch for every peek is going to be expensive.

 };
 
-class URLRegistry {
+
+class NewCSSTokenizer {
+    WTF_MAKE_NONCOPYABLE(NewCSSTokenizer);
     WTF_MAKE_FAST_ALLOCATED;
 public:
-    virtual ~URLRegistry() { }
-    virtual void registerURL(SecurityOrigin*, const KURL&, URLRegistrable*) = 0;
-    virtual void unregisterURL(const KURL&) = 0;
+    NewCSSTokenizer();
 
-    // This is an optional API
-    virtual URLRegistrable* lookup(const String&) { ASSERT_NOT_REACHED(); return 0; }
+    CSSToken nextToken(CSSInputStream&);
+
+private:
+    UChar consume();
+    void reconsume(UChar);
+
+    CSSToken consumeStringTokenUntil(UChar);
+    CSSToken consumeNumericToken();
+    CSSToken consumeIdentLikeToken();
+    CSSToken consumeNumber();
+    CSSToken consumeURLToken();
+
+    void consumeUntilNotWhitespace();
+    void consumeThroughCommentEndOrUntilEOF();
+
+    bool consumeIfNext(UChar);
+    bool consumeIfNext(String);
+    String consumeName();
+    UChar consumeEscape();
+
+    bool nextIsValidEscape();
+    bool nextCharIsName();
+    bool nextTwoCharsAreValidEscape();
+    bool nextCharsAreNumber();
+    bool nextCharsAreIdentifier();
+
+    CSSInputStream* m_input;
 };
 
 } // namespace WebCore
 
-#endif // URLRegistry_h
+#endif // NewCSSTokenizer_h