CSS Tokenizer: Fix handling of escaped newlines

Source/core/css/parser/CSSTokenizerTest.cpp

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "config.h"
 #include "core/css/parser/CSSTokenizer.h"
 
 #include "core/css/parser/MediaQueryBlockWatcher.h"
 #include <gtest/gtest.h>
 
@@ skipping 139 matching lines @@
     TEST_TOKENS("spac\\65\n\rs", ident("space"), whitespace, ident("s"));
     TEST_TOKENS("sp\\61\tc\\65\fs", ident("spaces"));
     TEST_TOKENS("hel\\6c  o", ident("hell"), whitespace, ident("o"));
     TEST_TOKENS("test\\\n", ident("test"), delim('\\'), whitespace);
     TEST_TOKENS("eof\\", ident("eof"), delim('\\'));
     TEST_TOKENS("test\\D799", ident("test" + fromUChar32(0xD799)));
     TEST_TOKENS("\\E000", ident(fromUChar32(0xE000)));
     TEST_TOKENS("te\\s\\t", ident("test"));
     TEST_TOKENS("spaces\\ in\\\tident", ident("spaces in\tident"));
     TEST_TOKENS("\\.\\,\\:\\!", ident(".,:!"));
+    TEST_TOKENS("\\\r", delim('\\'), whitespace);
+    TEST_TOKENS("\\\f", delim('\\'), whitespace);
+    TEST_TOKENS("\\\r\n", delim('\\'), whitespace);
     // FIXME: We don't correctly return replacement characters
     // String replacement = fromUChar32(0xFFFD);
     // TEST_TOKENS("null\\0", ident("null" + replacement));
     // TEST_TOKENS("null\\0000", ident("null" + replacement));
     // TEST_TOKENS("large\\110000", ident("large" + replacement));
     // TEST_TOKENS("surrogate\\D800", ident("surrogate" + replacement));
     // TEST_TOKENS("surrogate\\0DABC", ident("surrogate" + replacement));
     // TEST_TOKENS("\\00DFFFsurrogate", ident(replacement + "surrogate"));
     // FIXME: We don't correctly return supplementary plane characters
     // TEST_TOKENS("\\10fFfF", ident(fromUChar32(0x10ffff) + "0"));
     // TEST_TOKENS("\\10000000", ident(fromUChar32(0x100000) + "000"));
-    // FIXME: We don't correctly match newlines (normally handled in preprocessing)
-    // TEST_TOKENS("\\\r", delim('\\'), whitespace);
-    // TEST_TOKENS("\\\f", delim('\\'), whitespace);
-    // TEST_TOKENS("\\\r\n", delim('\\'), whitespace);
 }
 
 TEST(CSSTokenizerTest, IdentToken)
 {
     TEST_TOKENS("simple-ident", ident("simple-ident"));
     TEST_TOKENS("testing123", ident("testing123"));
     TEST_TOKENS("hello!", ident("hello"), delim('!'));
     TEST_TOKENS("world\5", ident("world"), delim('\5'));
     TEST_TOKENS("_under score", ident("_under"), whitespace, ident("score"));
     TEST_TOKENS("-_underscore", ident("-_underscore"));
@@ skipping 26 matching lines @@
     TEST_TOKENS("'testing, 123!'", string("testing, 123!"));
     TEST_TOKENS("'es\\'ca\\\"pe'", string("es'ca\"pe"));
     TEST_TOKENS("'\"quotes\"'", string("\"quotes\""));
     TEST_TOKENS("\"'quotes'\"", string("'quotes'"));
     TEST_TOKENS("\"mismatch'", string("mismatch'"));
     TEST_TOKENS("'text\5\t\13'", string("text\5\t\13"));
     TEST_TOKENS("\"end on eof", string("end on eof"));
     TEST_TOKENS("'esca\\\nped'", string("escaped"));
     TEST_TOKENS("\"esc\\\faped\"", string("escaped"));
     TEST_TOKENS("'new\\\rline'", string("newline"));
+    TEST_TOKENS("\"new\\\r\nline\"", string("newline"));
     TEST_TOKENS("'bad\nstring", badString, whitespace, ident("string"));
     TEST_TOKENS("'bad\rstring", badString, whitespace, ident("string"));
     TEST_TOKENS("'bad\r\nstring", badString, whitespace, ident("string"));
     TEST_TOKENS("'bad\fstring", badString, whitespace, ident("string"));
     // FIXME: Preprocessing is supposed to replace U+0000 with U+FFFD
     // TEST_TOKENS("'\0'", string(fromUChar32(0xFFFD)));
-    // FIXME: We don't correctly match newlines (normally handled in preprocessing)
-    // TEST_TOKENS("\"new\\\r\nline\"", string("newline"));
 }
 
 TEST(CSSTokenizerTest, NumberToken)
 {
     TEST_TOKENS("10", number(IntegerValueType, 10));
     TEST_TOKENS("12.0", number(NumberValueType, 12));
     TEST_TOKENS("+45.6", number(NumberValueType, 45.6));
     TEST_TOKENS("-7", number(IntegerValueType, -7));
     TEST_TOKENS("010", number(IntegerValueType, 10));
     TEST_TOKENS("10e0", number(NumberValueType, 10));
@@ skipping 105 matching lines @@
             blockWatcher.handleToken(tokens[j]);
             level = blockWatcher.blockLevel();
             maxLevel = std::max(level, maxLevel);
         }
         ASSERT_EQ(testCases[i].maxLevel, maxLevel);
         ASSERT_EQ(testCases[i].finalLevel, level);
     }
 }
 
 } // namespace