Defer setting type in the WebVTT tokenizer until emitting the token

Source/core/html/track/vtt/VTTTokenizer.h

 /*
  * Copyright (C) 2011 Google Inc.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 30 matching lines @@
 class VTTTokenizerState {
 public:
     enum State {
         DataState,
         EscapeState,
         TagState,
         StartTagState,
         StartTagClassState,
         StartTagAnnotationState,
         EndTagState,
-        EndTagOpenState,
         TimestampTagState,
     };
 };
 
 class VTTTokenizer {
     WTF_MAKE_NONCOPYABLE(VTTTokenizer);
-    WTF_MAKE_FAST_ALLOCATED;
 public:
-    static PassOwnPtr<VTTTokenizer> create() { return adoptPtr(new VTTTokenizer); }
+    explicit VTTTokenizer(const String& input);
 
     typedef VTTTokenizerState State;
 
     void reset();
 
-    bool nextToken(SegmentedString&, VTTToken&);
+    bool nextToken(VTTToken&);
 
-    inline bool haveBufferedCharacterToken()
-    {
-        return m_token->type() == VTTToken::Type::Character;
-    }
+    inline bool haveBufferedCharacterToken() { return false; }

[jochen (gone - plz use gerrit), 2013/11/20 14:09:37]

why not just remove this method?

 
     inline void bufferCharacter(UChar character)
     {
         ASSERT(character != kEndOfFileMarker);
-        m_token->ensureIsCharacterToken();
         m_token->appendToCharacter(character);
     }
 
     inline bool advanceAndEmitToken(SegmentedString& source, VTTTokenTypes::Type type)
     {
         source.advanceAndUpdateLineNumber();
         return emitToken(type);
     }
 
     inline bool emitToken(VTTTokenTypes::Type type)
     {
-        ASSERT(m_token->type() == type);
+        m_token->setType(type);
         return true;
     }
 
     bool shouldSkipNullCharacters() const { return true; }
 
 private:
-    VTTTokenizer();
-
     // m_token is owned by the caller. If nextToken is not on the stack,
     // this member might be pointing to unallocated memory.
     VTTToken* m_token;
 
     // This member does not really need to be an instance member - it's only used in nextToken.
     // The reason it's stored here is because of the use of the ADVANCE_TO state helpers.
     VTTTokenizerState::State m_state;
 
     StringBuilder m_buffer;
+    SegmentedString m_input;
 
     // ://www.whatwg.org/specs/web-apps/current-work/#preprocessing-the-input-stream
     InputStreamPreprocessor<VTTTokenizer> m_inputStreamPreprocessor;
 };
 
 }
 
 #endif