CSS Custom Properties (Variables)

Source/core/css/parser/CSSTokenizer.cpp

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "config.h"
 #include "core/css/parser/CSSTokenizer.h"
 
 namespace blink {
 #include "core/CSSTokenizerCodepoints.cpp"
 }
 
 #include "core/css/parser/CSSParserObserverWrapper.h"
 #include "core/css/parser/CSSParserTokenRange.h"
 #include "core/css/parser/CSSTokenizerInputStream.h"
 #include "core/html/parser/HTMLParserIdioms.h"
 #include "wtf/text/CharacterNames.h"
 
 namespace blink {
 
 CSSTokenizer::Scope::Scope(const String& string)
-: m_string(string)
+    : m_string(string)
 {
     // According to the spec, we should perform preprocessing here.
     // See: http://dev.w3.org/csswg/css-syntax/#input-preprocessing
     //
     // However, we can skip this step since:
     // * We're using HTML spaces (which accept \r and \f as a valid white space)
     // * Do not count white spaces
     // * CSSTokenizerInputStream::peek replaces NULLs for replacement characters
 
     if (string.isEmpty())
         return;
 
     // To avoid resizing we err on the side of reserving too much space.
     // Most strings we tokenize have about 3.5 to 5 characters per token.
     m_tokens.reserveInitialCapacity(string.length() / 3);
 
     CSSTokenizerInputStream input(string);
     CSSTokenizer tokenizer(input, *this);
     while (true) {
         CSSParserToken token = tokenizer.nextToken();
         if (token.type() == CommentToken)
             continue;
         if (token.type() == EOFToken)
             return;
         m_tokens.append(token);
     }
 }
 
 CSSTokenizer::Scope::Scope(const String& string, CSSParserObserverWrapper& wrapper)
-: m_string(string)
+    : m_string(string)
 {
     if (string.isEmpty())
         return;
 
     CSSTokenizerInputStream input(string);
     CSSTokenizer tokenizer(input, *this);
 
     unsigned offset = 0;
     while (true) {
         CSSParserToken token = tokenizer.nextToken();
@@ skipping 197 matching lines @@
     return CSSParserToken(DelimiterToken, cc);
 }
 
 CSSParserToken CSSTokenizer::colon(UChar cc)
 {
     return CSSParserToken(ColonToken);
 }
 
 CSSParserToken CSSTokenizer::semiColon(UChar cc)
 {
-    return CSSParserToken(SemicolonToken);
+    return CSSParserToken(SemicolonToken, lastConsumedCharacter());

[alancutter (OOO until 2018), 2015/08/05 08:01:43]

It's not clear to me why we need to store the semicolon character.

 }
 
 CSSParserToken CSSTokenizer::hash(UChar cc)
 {
     UChar nextChar = m_input.nextInputChar();
     if (isNameChar(nextChar) || twoCharsAreValidEscape(nextChar, m_input.peek(1))) {
         HashTokenType type = nextCharsAreIdentifier() ? HashTokenId : HashTokenUnrestricted;
         return CSSParserToken(type, consumeName());
     }
 
@@ skipping 348 matching lines @@
 {
     while (true) {
         UChar cc = consume();
         if (cc == ')' || cc == kEndOfFileMarker)
             return;
         if (twoCharsAreValidEscape(cc, m_input.nextInputChar()))
             consumeEscape();
     }
 }
 
-void CSSTokenizer::consumeUntilNonWhitespace()
+unsigned CSSTokenizer::consumeUntilNonWhitespace()
 {
+    unsigned count = 0;
     // Using HTML space here rather than CSS space since we don't do preprocessing
-    while (isHTMLSpace<UChar>(m_input.nextInputChar()))
+    while (isHTMLSpace<UChar>(m_input.nextInputChar())) {
+        ++count;
         consume();
+    }
+    return count;
 }
 
 void CSSTokenizer::consumeSingleWhitespaceIfNext()
 {
     // We check for \r\n and HTML spaces since we don't do preprocessing
     UChar c = m_input.nextInputChar();
     if (c == '\r' && m_input.peek(1) == '\n')
         consume(2);
     else if (isHTMLSpace(c))
         consume();
@@ skipping 48 matching lines @@
         }
         if (twoCharsAreValidEscape(cc, m_input.nextInputChar())) {
             result.append(consumeEscape());
             continue;
         }
         reconsume(cc);
         return registerString(result.toString());
     }
 }
 
+CSSParserString CSSTokenizer::lastConsumedCharacter()
+{
+    return m_input.rangeAsCSSParserString(m_input.offset() - 1, 1);
+}
+
 // http://dev.w3.org/csswg/css-syntax/#consume-an-escaped-code-point
 UChar32 CSSTokenizer::consumeEscape()
 {
     UChar cc = consume();
     ASSERT(!isNewLine(cc));
     if (isASCIIHexDigit(cc)) {
         unsigned consumedHexDigits = 1;
         StringBuilder hexChars;
         hexChars.append(cc);
         while (consumedHexDigits < 6 && isASCIIHexDigit(m_input.nextInputChar())) {
@@ skipping 64 matching lines @@
 
 CSSParserString CSSTokenizer::registerString(const String& string)
 {
     m_scope.storeString(string);
     CSSParserString result;
     result.init(string);
     return result;
 }
 
 } // namespace blink