Defer setting type in the WebVTT tokenizer until emitting the token

Source/core/html/track/vtt/VTTTokenizer.h

Index: Source/core/html/track/vtt/VTTTokenizer.h
diff --git a/Source/core/html/track/vtt/VTTTokenizer.h b/Source/core/html/track/vtt/VTTTokenizer.h
index d2ac81edc17df563b687fc8bf240f2680b0b0a1e..52b43b538806d49f9e21f432e8770747c6451c54 100644
--- a/Source/core/html/track/vtt/VTTTokenizer.h
+++ b/Source/core/html/track/vtt/VTTTokenizer.h
@@ -48,32 +48,26 @@ public:
         StartTagClassState,
         StartTagAnnotationState,
         EndTagState,
-        EndTagOpenState,
         TimestampTagState,
     };
 };
 
 class VTTTokenizer {
     WTF_MAKE_NONCOPYABLE(VTTTokenizer);
-    WTF_MAKE_FAST_ALLOCATED;
 public:
-    static PassOwnPtr<VTTTokenizer> create() { return adoptPtr(new VTTTokenizer); }
+    explicit VTTTokenizer(const String& input);
 
     typedef VTTTokenizerState State;
 
     void reset();
 
-    bool nextToken(SegmentedString&, VTTToken&);
+    bool nextToken(VTTToken&);
 
-    inline bool haveBufferedCharacterToken()
-    {
-        return m_token->type() == VTTToken::Type::Character;
-    }
+    inline bool haveBufferedCharacterToken() { return false; }

[jochen (gone - plz use gerrit), 2013/11/20 14:09:37]

why not just remove this method?

 
     inline void bufferCharacter(UChar character)
     {
         ASSERT(character != kEndOfFileMarker);
-        m_token->ensureIsCharacterToken();
         m_token->appendToCharacter(character);
     }
 
@@ -85,15 +79,13 @@ public:
 
     inline bool emitToken(VTTTokenTypes::Type type)
     {
-        ASSERT(m_token->type() == type);
+        m_token->setType(type);
         return true;
     }
 
     bool shouldSkipNullCharacters() const { return true; }
 
 private:
-    VTTTokenizer();
-
     // m_token is owned by the caller. If nextToken is not on the stack,
     // this member might be pointing to unallocated memory.
     VTTToken* m_token;
@@ -103,6 +95,7 @@ private:
     VTTTokenizerState::State m_state;
 
     StringBuilder m_buffer;
+    SegmentedString m_input;
 
     // ://www.whatwg.org/specs/web-apps/current-work/#preprocessing-the-input-stream
     InputStreamPreprocessor<VTTTokenizer> m_inputStreamPreprocessor;