Parse Sky entities according to the spec

sky/engine/core/html/parser/HTMLTokenizer.cpp

 /*
  * Copyright (C) 2008 Apple Inc. All Rights Reserved.
  * Copyright (C) 2009 Torch Mobile, Inc. http://www.torchmobile.com/
  * Copyright (C) 2010 Google, Inc. All Rights Reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
@@ skipping 93 matching lines @@
 }
 
 HTMLTokenizer::~HTMLTokenizer()
 {
 }
 
 void HTMLTokenizer::reset()
 {
     m_state = HTMLTokenizer::DataState;
     m_token = 0;
-    m_additionalAllowedCharacter = '\0';
-}
-
-inline bool HTMLTokenizer::processEntity(SegmentedString& source)
-{
-    bool notEnoughCharacters = false;
-    DecodedHTMLEntity decodedEntity;
-    bool success = consumeHTMLEntity(source, decodedEntity, notEnoughCharacters);
-    if (notEnoughCharacters)
-        return false;
-    if (!success) {
-        ASSERT(decodedEntity.isEmpty());
-        bufferCharacter('&');
-    } else {
-        for (unsigned i = 0; i < decodedEntity.length; ++i)
-            bufferCharacter(decodedEntity.data[i]);
-    }
-    return true;
 }
 
 bool HTMLTokenizer::flushBufferedEndTag(SegmentedString& source)
 {
     ASSERT(m_token->type() == HTMLToken::Character || m_token->type() == HTMLToken::Uninitialized);
     source.advanceAndUpdateLineNumber();
     if (m_token->type() == HTMLToken::Character)
         return true;
     m_token->beginEndTag(m_bufferedEndTagName);
     m_bufferedEndTagName.clear();
     m_appropriateEndTagName.clear();
     m_temporaryBuffer.clear();
     return false;
 }
 
 #define FLUSH_AND_ADVANCE_TO(stateName)                                    \
     do {                                                                   \
-        m_state = HTMLTokenizer::stateName;                           \
+        m_state = HTMLTokenizer::stateName;                                \
         if (flushBufferedEndTag(source))                                   \
             return true;                                                   \
         if (source.isEmpty()                                               \
             || !m_inputStreamPreprocessor.peek(source))                    \
             return haveBufferedCharacterToken();                           \
         cc = m_inputStreamPreprocessor.nextInputCharacter();               \
         goto stateName;                                                    \
     } while (false)
 
 bool HTMLTokenizer::flushEmitAndResumeIn(SegmentedString& source, HTMLTokenizer::State state)
@@ skipping 23 matching lines @@
         }
     }
 
     if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source))
         return haveBufferedCharacterToken();
     UChar cc = m_inputStreamPreprocessor.nextInputCharacter();
 
     // Source: http://www.whatwg.org/specs/web-apps/current-work/#tokenisation0
     switch (m_state) {
     HTML_BEGIN_STATE(DataState) {
-        if (cc == '&')
+        if (cc == '&') {
+            m_returnState = DataState;
+            m_entityParser.reset();
             HTML_ADVANCE_TO(CharacterReferenceInDataState);
-        else if (cc == '<') {
+        } else if (cc == '<') {
             if (m_token->type() == HTMLToken::Character) {
                 // We have a bunch of character tokens queued up that we
                 // are emitting lazily here.
                 return true;
             }
             HTML_ADVANCE_TO(TagOpenState);
         } else if (cc == kEndOfFileMarker)
             return emitEndOfFile(source);
         else {
             bufferCharacter(cc);
             HTML_ADVANCE_TO(DataState);
         }
     }
     END_STATE()
 
     HTML_BEGIN_STATE(CharacterReferenceInDataState) {
-        if (!processEntity(source))
+        if (!m_entityParser.parse(source))
             return haveBufferedCharacterToken();
+        for (const UChar& entityCharacter : m_entityParser.result())
+            bufferCharacter(entityCharacter);
+        cc = m_inputStreamPreprocessor.nextInputCharacter();
+        ASSERT(m_returnState == m_returnState);
         HTML_SWITCH_TO(DataState);
     }
     END_STATE()
 
+    HTML_BEGIN_STATE(CharacterReferenceInAttributeValueState) {
+        if (!m_entityParser.parse(source))
+            return haveBufferedCharacterToken();
+        for (const UChar& entityCharacter : m_entityParser.result())
+            m_token->appendToAttributeValue(entityCharacter);
+        cc = m_inputStreamPreprocessor.nextInputCharacter();
+
+        if (m_returnState == AttributeValueDoubleQuotedState)
+            HTML_SWITCH_TO(AttributeValueDoubleQuotedState);
+        else if (m_returnState == AttributeValueSingleQuotedState)
+            HTML_SWITCH_TO(AttributeValueSingleQuotedState);
+        else if (m_returnState == AttributeValueUnquotedState)
+            HTML_SWITCH_TO(AttributeValueUnquotedState);
+        else
+            ASSERT_NOT_REACHED();
+    }
+    END_STATE()
+
     HTML_BEGIN_STATE(RAWTEXTState) {
         if (cc == '<')
             HTML_ADVANCE_TO(RAWTEXTLessThanSignState);
         else if (cc == kEndOfFileMarker)
             return emitEndOfFile(source);
         else {
             bufferCharacter(cc);
             HTML_ADVANCE_TO(RAWTEXTState);
         }
     }
@@ skipping 242 matching lines @@
             HTML_ADVANCE_TO(AttributeValueUnquotedState);
         }
     }
     END_STATE()
 
     HTML_BEGIN_STATE(AttributeValueDoubleQuotedState) {
         if (cc == '"') {
             m_token->endAttributeValue(source.numberOfCharactersConsumed());
             HTML_ADVANCE_TO(AfterAttributeValueQuotedState);
         } else if (cc == '&') {
-            m_additionalAllowedCharacter = '"';
+            m_returnState = AttributeValueDoubleQuotedState;
+            m_entityParser.reset();
             HTML_ADVANCE_TO(CharacterReferenceInAttributeValueState);
         } else if (cc == kEndOfFileMarker) {
             parseError();
             m_token->endAttributeValue(source.numberOfCharactersConsumed());
             HTML_RECONSUME_IN(DataState);
         } else {
             m_token->appendToAttributeValue(cc);
             HTML_ADVANCE_TO(AttributeValueDoubleQuotedState);
         }
     }
     END_STATE()
 
     HTML_BEGIN_STATE(AttributeValueSingleQuotedState) {
         if (cc == '\'') {
             m_token->endAttributeValue(source.numberOfCharactersConsumed());
             HTML_ADVANCE_TO(AfterAttributeValueQuotedState);
         } else if (cc == '&') {
-            m_additionalAllowedCharacter = '\'';
+            m_returnState = AttributeValueSingleQuotedState;
+            m_entityParser.reset();
             HTML_ADVANCE_TO(CharacterReferenceInAttributeValueState);
         } else if (cc == kEndOfFileMarker) {
             parseError();
             m_token->endAttributeValue(source.numberOfCharactersConsumed());
             HTML_RECONSUME_IN(DataState);
         } else {
             m_token->appendToAttributeValue(cc);
             HTML_ADVANCE_TO(AttributeValueSingleQuotedState);
         }
     }
     END_STATE()
 
     HTML_BEGIN_STATE(AttributeValueUnquotedState) {
         if (isTokenizerWhitespace(cc)) {
             m_token->endAttributeValue(source.numberOfCharactersConsumed());
             HTML_ADVANCE_TO(BeforeAttributeNameState);
         } else if (cc == '&') {
-            m_additionalAllowedCharacter = '>';
+            m_returnState = AttributeValueUnquotedState;
+            m_entityParser.reset();
             HTML_ADVANCE_TO(CharacterReferenceInAttributeValueState);
         } else if (cc == '>') {
             m_token->endAttributeValue(source.numberOfCharactersConsumed());
             return emitAndResumeIn(source, HTMLTokenizer::DataState);
         } else if (cc == kEndOfFileMarker) {
             parseError();
             m_token->endAttributeValue(source.numberOfCharactersConsumed());
             HTML_RECONSUME_IN(DataState);
         } else {
             if (cc == '"' || cc == '\'' || cc == '<' || cc == '=' || cc == '`')
                 parseError();
             m_token->appendToAttributeValue(cc);
             HTML_ADVANCE_TO(AttributeValueUnquotedState);
         }
     }
     END_STATE()
 
-    HTML_BEGIN_STATE(CharacterReferenceInAttributeValueState) {
-        bool notEnoughCharacters = false;
-        DecodedHTMLEntity decodedEntity;
-        bool success = consumeHTMLEntity(source, decodedEntity, notEnoughCharacters, m_additionalAllowedCharacter);
-        if (notEnoughCharacters)
-            return haveBufferedCharacterToken();
-        if (!success) {
-            ASSERT(decodedEntity.isEmpty());
-            m_token->appendToAttributeValue('&');
-        } else {
-            for (unsigned i = 0; i < decodedEntity.length; ++i)
-                m_token->appendToAttributeValue(decodedEntity.data[i]);
-        }
-        // We're supposed to switch back to the attribute value state that
-        // we were in when we were switched into this state. Rather than
-        // keeping track of this explictly, we observe that the previous
-        // state can be determined by m_additionalAllowedCharacter.
-        if (m_additionalAllowedCharacter == '"')
-            HTML_SWITCH_TO(AttributeValueDoubleQuotedState);
-        else if (m_additionalAllowedCharacter == '\'')
-            HTML_SWITCH_TO(AttributeValueSingleQuotedState);
-        else if (m_additionalAllowedCharacter == '>')
-            HTML_SWITCH_TO(AttributeValueUnquotedState);
-        else
-            ASSERT_NOT_REACHED();
-    }
-    END_STATE()
-
     HTML_BEGIN_STATE(AfterAttributeValueQuotedState) {
         if (isTokenizerWhitespace(cc))
             HTML_ADVANCE_TO(BeforeAttributeNameState);
         else if (cc == '/')
             HTML_ADVANCE_TO(SelfClosingStartTagState);
         else if (cc == '>')
             return emitAndResumeIn(source, HTMLTokenizer::DataState);
         else if (cc == kEndOfFileMarker) {
             parseError();
             HTML_RECONSUME_IN(DataState);
@@ skipping 187 matching lines @@
 
     return true;
 }
 
 inline void HTMLTokenizer::parseError()
 {
     notImplemented();
 }
 
 }