Defer setting type in the WebVTT tokenizer until emitting the token

Source/core/html/track/vtt/VTTTokenizer.cpp

 /*
  * Copyright (C) 2011 Google Inc.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 28 matching lines @@
 
 #define WEBVTT_BEGIN_STATE(stateName) BEGIN_STATE(VTTTokenizerState, stateName)
 #define WEBVTT_ADVANCE_TO(stateName) ADVANCE_TO(VTTTokenizerState, stateName)
 
 template<unsigned charactersCount>
 ALWAYS_INLINE bool equalLiteral(const StringBuilder& s, const char (&characters)[charactersCount])
 {
     return WTF::equal(s, reinterpret_cast<const LChar*>(characters), charactersCount - 1);
 }
 
-VTTTokenizer::VTTTokenizer()
-    : m_inputStreamPreprocessor(this)
+VTTTokenizer::VTTTokenizer(const String& input)
+    : m_input(input)
+    , m_inputStreamPreprocessor(this)
 {
     reset();
+
+    // Append a EOF marker and close the input "stream".
+    ASSERT(!m_input.isClosed());
+    m_input.append(SegmentedString(String(&kEndOfFileMarker, 1)));
+    m_input.close();
 }
 
 void VTTTokenizer::reset()
 {
     m_token = 0;
     m_buffer.clear();
 }
 
-bool VTTTokenizer::nextToken(SegmentedString& source, VTTToken& token)
+bool VTTTokenizer::nextToken(VTTToken& token)
 {
     // If we have a token in progress, then we're supposed to be called back
     // with the same token so we can finish it.
-    ASSERT(!m_token || m_token == &token || token.type() == VTTTokenTypes::Uninitialized);
+    ASSERT(!m_token || m_token == &token);
     m_token = &token;
 
-    if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source))
+    if (m_input.isEmpty() || !m_inputStreamPreprocessor.peek(m_input))
         return haveBufferedCharacterToken();
 
     UChar cc = m_inputStreamPreprocessor.nextInputCharacter();
+    if (cc == kEndOfFileMarker) {
+        m_inputStreamPreprocessor.advance(m_input);
+        return false;
+    }
 
     m_state = VTTTokenizerState::DataState;
 
+    // The ADVANCE_TO helper macros expect this name ('source') on the input variable.
+    SegmentedString& source = m_input;
+
     // 4.8.10.13.4 WebVTT cue text tokenizer
     switch (m_state) {
         WEBVTT_BEGIN_STATE(DataState) {
             if (cc == '&') {
                 m_buffer.append(static_cast<LChar>(cc));
                 WEBVTT_ADVANCE_TO(EscapeState);
             } else if (cc == '<') {
-                if (m_token->type() == VTTTokenTypes::Uninitialized || m_token->characters().isEmpty()) {
+                if (m_token->characters().isEmpty()) {
                     WEBVTT_ADVANCE_TO(TagState);
                 } else {
                     // We don't want to advance input or perform a state transition - just return a (new) token.
                     // (On the next call to nextToken we will see '<' again, but take the other branch in this if instead.)
                     return emitToken(VTTTokenTypes::Character);
                 }
             } else if (cc == kEndOfFileMarker) {
-                return emitToken(VTTTokenTypes::Character);
+                return advanceAndEmitToken(source, VTTTokenTypes::Character);
             } else {
                 bufferCharacter(cc);
                 WEBVTT_ADVANCE_TO(DataState);
             }
         }
         END_STATE()
 
         WEBVTT_BEGIN_STATE(EscapeState) {
             if (cc == ';') {
                 if (equalLiteral(m_buffer, "&amp")) {
@@ skipping 12 matching lines @@
                     m_buffer.append(static_cast<LChar>(cc));
                     m_token->appendToCharacter(m_buffer);
                 }
                 m_buffer.clear();
                 WEBVTT_ADVANCE_TO(DataState);
             } else if (isASCIIAlphanumeric(cc)) {
                 m_buffer.append(static_cast<LChar>(cc));
                 WEBVTT_ADVANCE_TO(EscapeState);
             } else if (cc == kEndOfFileMarker) {
                 m_token->appendToCharacter(m_buffer);
-                return emitToken(VTTTokenTypes::Character);
+                return advanceAndEmitToken(source, VTTTokenTypes::Character);
             } else {
                 if (!equalLiteral(m_buffer, "&"))
                     m_token->appendToCharacter(m_buffer);
                 m_buffer.clear();
                 WEBVTT_ADVANCE_TO(DataState);
             }
         }
         END_STATE()
 
         WEBVTT_BEGIN_STATE(TagState) {
             if (isTokenizerWhitespace(cc)) {
                 m_token->beginEmptyStartTag();
                 WEBVTT_ADVANCE_TO(StartTagAnnotationState);
             } else if (cc == '.') {
                 m_token->beginEmptyStartTag();
                 WEBVTT_ADVANCE_TO(StartTagClassState);
             } else if (cc == '/') {
-                WEBVTT_ADVANCE_TO(EndTagOpenState);
+                WEBVTT_ADVANCE_TO(EndTagState);
             } else if (WTF::isASCIIDigit(cc)) {
                 m_token->beginTimestampTag(cc);
                 WEBVTT_ADVANCE_TO(TimestampTagState);
             } else if (cc == '>' || cc == kEndOfFileMarker) {
                 m_token->beginEmptyStartTag();
                 return advanceAndEmitToken(source, VTTTokenTypes::StartTag);
             } else {
                 m_token->beginStartTag(cc);
                 WEBVTT_ADVANCE_TO(StartTagState);
             }
@@ skipping 35 matching lines @@
         WEBVTT_BEGIN_STATE(StartTagAnnotationState) {
             if (cc == '>' || cc == kEndOfFileMarker) {
                 m_token->addNewAnnotation();
                 return advanceAndEmitToken(source, VTTTokenTypes::StartTag);
             }
             m_token->appendToAnnotation(cc);
             WEBVTT_ADVANCE_TO(StartTagAnnotationState);
         }
         END_STATE()
 
-        WEBVTT_BEGIN_STATE(EndTagOpenState) {
-            if (cc == '>' || cc == kEndOfFileMarker) {
-                m_token->beginEndTag('\0');
-                return advanceAndEmitToken(source, VTTTokenTypes::EndTag);
-            }
-            m_token->beginEndTag(cc);
-            WEBVTT_ADVANCE_TO(EndTagState);
-        }
-        END_STATE()
-
         WEBVTT_BEGIN_STATE(EndTagState) {
             if (cc == '>' || cc == kEndOfFileMarker)
                 return advanceAndEmitToken(source, VTTTokenTypes::EndTag);
             m_token->appendToName(cc);
             WEBVTT_ADVANCE_TO(EndTagState);
         }
         END_STATE()
 
         WEBVTT_BEGIN_STATE(TimestampTagState) {
             if (cc == '>' || cc == kEndOfFileMarker)
                 return advanceAndEmitToken(source, VTTTokenTypes::TimestampTag);
             m_token->appendToTimestamp(cc);
             WEBVTT_ADVANCE_TO(TimestampTagState);
         }
         END_STATE()
 
     }
 
     ASSERT_NOT_REACHED();
     return false;
 }
 
 }