Add very basic CSS3 Syntax compatible tokenizer

Source/core/css/parser/NewCSSTokenizerTest.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 12 matching lines @@
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "config.h"
 
-#include "heap/Heap.h"
+#include "core/css/parser/NewCSSTokenizer.h"
+#include "wtf/PassOwnPtr.h"
 
 #include <gtest/gtest.h>
 
 using namespace WebCore;
 
 namespace {
 
-TEST(HeapTest, Init)
+void tokenize(String string, Vector<CSSToken>& outTokens)
 {
-    Heap::init(0);
-    Heap::shutdown();
+    NewCSSTokenizer tokenizer;
+    CSSInputStream input(string);
+    while (true) {
+        outTokens.append(tokenizer.nextToken(input));
+        if (outTokens.last().type() == EOFToken)
+            return;
+    }
+}
+
+TEST(CSSTokenizerTest, Basic)
+{
+    Vector<CSSToken> tokens;
+    tokenize("foo", tokens);
+    printf("%zu\n", tokens.size());
+    for (size_t i = 0; i < tokens.size(); i++)
+        printf("%zu: %i\n", i, tokens[i].type());
 }
 
 } // namespace