Add very basic CSS3 Syntax compatible tokenizer

Source/core/css/parser/NewCSSTokenizer.h

Index: Source/core/css/parser/NewCSSTokenizer.h
diff --git a/Source/core/html/URLRegistry.h b/Source/core/css/parser/NewCSSTokenizer.h
similarity index 54%
copy from Source/core/html/URLRegistry.h
copy to Source/core/css/parser/NewCSSTokenizer.h
index 660acb92852636ea136d2c49dfa68c31e268946b..74cbf9c3468fb9862ff3721e5795ed80a8d12d44 100644
--- a/Source/core/html/URLRegistry.h
+++ b/Source/core/css/parser/NewCSSTokenizer.h
@@ -28,34 +28,73 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef URLRegistry_h
-#define URLRegistry_h
+#ifndef NewCSSTokenizer_h
+#define NewCSSTokenizer_h
 
+#include "core/css/parser/CSSToken.h"
+#include "core/html/parser/InputStreamPreprocessor.h"
 #include "wtf/text/WTFString.h"
 
 namespace WebCore {
 
-class KURL;
-class SecurityOrigin;
-class URLRegistry;
-
-class URLRegistrable {
+// The CSS Syntax spec is written as a (fixed size) look-ahead
+// tokenizer instead of a state-machine like HTML
+// so we have our own input stream class. :(
+class CSSInputStream {
+    WTF_MAKE_NONCOPYABLE(CSSInputStream);
+    WTF_MAKE_FAST_ALLOCATED;
 public:
-    virtual ~URLRegistrable() { }
-    virtual URLRegistry& registry() const = 0;
+    CSSInputStream(String input);
+
+    UChar currentInputChar();
+    UChar nextInputChar(); // peek1();
+    UChar peek2();
+    UChar peek3(); // Needed?
+
+    void advance();
+    void pushBack(UChar);
+
+private:
+    size_t m_offset;
+    String m_string;

[abarth-chromium, 2014/01/01 18:47:51]

I would have expected you'd want to templatize this class based on the CharType
and have two copies of the CSS parser in the binary: one for UChar and one for
LChar.  Taking the is8Bit branch for every peek is going to be expensive.

 };
 
-class URLRegistry {
+
+class NewCSSTokenizer {
+    WTF_MAKE_NONCOPYABLE(NewCSSTokenizer);
     WTF_MAKE_FAST_ALLOCATED;
 public:
-    virtual ~URLRegistry() { }
-    virtual void registerURL(SecurityOrigin*, const KURL&, URLRegistrable*) = 0;
-    virtual void unregisterURL(const KURL&) = 0;
+    NewCSSTokenizer();
+
+    CSSToken nextToken(CSSInputStream&);
+
+private:
+    UChar consume();
+    void reconsume(UChar);
+
+    CSSToken consumeStringTokenUntil(UChar);
+    CSSToken consumeNumericToken();
+    CSSToken consumeIdentLikeToken();
+    CSSToken consumeNumber();
+    CSSToken consumeURLToken();
+
+    void consumeUntilNotWhitespace();
+    void consumeThroughCommentEndOrUntilEOF();
+
+    bool consumeIfNext(UChar);
+    bool consumeIfNext(String);
+    String consumeName();
+    UChar consumeEscape();
+
+    bool nextIsValidEscape();
+    bool nextCharIsName();
+    bool nextTwoCharsAreValidEscape();
+    bool nextCharsAreNumber();
+    bool nextCharsAreIdentifier();
 
-    // This is an optional API
-    virtual URLRegistrable* lookup(const String&) { ASSERT_NOT_REACHED(); return 0; }
+    CSSInputStream* m_input;
 };
 
 } // namespace WebCore
 
-#endif // URLRegistry_h
+#endif // NewCSSTokenizer_h