[WIP] Streaming CSS parser

third_party/WebKit/Source/core/css/parser/CSSParserTokenStream.h

Index: third_party/WebKit/Source/core/css/parser/CSSParserTokenStream.h
diff --git a/third_party/WebKit/Source/core/css/parser/CSSParserTokenStream.h b/third_party/WebKit/Source/core/css/parser/CSSParserTokenStream.h
new file mode 100644
index 0000000000000000000000000000000000000000..b78fd0a0dc04cd3b850024a10cf525f598daf600
--- /dev/null
+++ b/third_party/WebKit/Source/core/css/parser/CSSParserTokenStream.h
@@ -0,0 +1,184 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CSSParserTokenStream_h
+#define CSSParserTokenStream_h
+
+#include "core/css/parser/CSSParserToken.h"
+#include "core/css/parser/CSSParserTokenRange.h"
+#include "core/css/parser/CSSTokenizer.h"
+
+namespace blink {
+
+class CSSParserObserver;
+
+// This class is a lazily tokenized stream of CSSParserTokens. It provides only
+// a single token of lookahead. When this is not sufficient, the functions
+// index(), consumeUntilAtEndOrPeekedTypeIs() and makeSubRangeFrom() are used
+// to make a CSSParserTokenRange.
+class CSSParserTokenStream {
+ public:
+  enum MakeSubStreamTag { MakeSubStream };

[Charlie Harrison, 2017/01/09 21:35:07]

optional: Put this right above the constructor that needs this, so as to make it
obvious this enum's purpose.

+
+  explicit CSSParserTokenStream(CSSTokenizer& tokenizer)
+      : m_tokenizer(tokenizer),
+        m_startIndex(0),
+        m_currentIndex(0),
+        m_isSubStream(false) {
+    DCHECK(tokenizer.m_tokens.isEmpty());
+  }
+
+  // This constructor can be called when the next token is a LeftBraceToken.
+  // The new stream will end at the end of the block and clean up by deleting
+  // all the tokens in the block. The original stream cannot be used until the
+  // sub-stream is destroyed.
+  CSSParserTokenStream(CSSParserTokenStream& stream, MakeSubStreamTag)
+      : m_tokenizer(stream.m_tokenizer),
+        m_startIndex(stream.m_currentIndex),
+        m_currentIndex(stream.m_currentIndex + 1),
+        m_isSubStream(true) {
+    DCHECK_EQ(m_tokenizer.m_tokens.size(), m_currentIndex);
+    DCHECK_EQ(m_tokenizer.m_tokens.back().type(), LeftBraceToken);
+  }
+
+  ~CSSParserTokenStream();
+
+  // Delete all the tokens this stream has consumed. This is used to ensure
+  // the tokenizer doesn't take up much memory.
+  void clean() {
+    m_tokenizer.m_tokens.remove(m_startIndex, m_currentIndex - m_startIndex);

[Charlie Harrison, 2017/01/09 21:35:07]

I'll need to look at all usage, but it seems like if it is possible to be
removing in the middle of a Vector this could have bad perf consequences.

Disclaimer: I'm not seeing any of this in profiles.

+    m_currentIndex = m_startIndex;
+  }
+
+  const CSSParserToken& peek() {
+    const CSSParserToken& result = peekInternal();
+    if (result.getBlockType() == CSSParserToken::BlockEnd)
+      return staticEOFToken;
+    return result;
+  }
+
+  bool atEnd() { return peek().type() == EOFToken; }
+
+  // Consumes component values until the next token is a listed type or EOF.
+  template <CSSParserTokenType... types>
+  void consumeUntilAtEndOrPeekedTypeIs();
+
+  // Only for single-length component values.
+  void consume() {
+    DCHECK(hasLookedAhead());
+    DCHECK(peekInternal().getBlockType() == CSSParserToken::NotBlock);
+    DCHECK(!atEnd());
+    m_currentIndex++;
+  }
+
+  // These do not create tokens
+  void skipWhitespaceAndComments() {
+    DCHECK_EQ(m_currentIndex, m_tokenizer.m_tokens.size());
+    m_tokenizer.skipWhitespaceAndComments();
+  }
+  void skipBlock() {
+    DCHECK(hasLookedAhead());
+    DCHECK(peek().type() == LeftBraceToken);
+    m_tokenizer.skipToBlockEnd();
+  }
+
+  // This is the current character index in the original string.
+  // This can only be called if we have not looked ahead. After calling peek(),
+  // atEnd(), or consumeUntilAtEndOrPeekedTypeIs(), we can only call offset()
+  // if we're at the end of the stream.
+  // After calling consume() or making a substream we can call offset().
+  size_t offset() {
+    if (hasLookedAhead()) {
+      DCHECK(atEnd());
+      return previousOffset();
+    }
+    return m_tokenizer.m_input.offset();
+  }
+  // This is the character index in the original string of where we have
+  // consumed up to, when we have looked ahead. This is only correct when
+  // the look-ahead token is a single character or EOF.
+  size_t previousOffset() {
+    if (m_tokenizer.m_finishedTokenizing) {
+      DCHECK_EQ(m_currentIndex, m_tokenizer.m_tokens.size());
+      return m_tokenizer.m_input.offset();
+    }
+    DCHECK_EQ(m_currentIndex, m_tokenizer.m_tokens.size() - 1);
+    return m_tokenizer.m_input.offset() - 1;
+  }
+
+  size_t offsetAfterComments() {
+    if (!hasLookedAhead())
+      m_tokenizer.skipComments();
+    return offset();
+  }
+
+  // CSSParserTokenRange objects are created by calling index(), consuming from
+  // the stream, then calling makeSubRangeFrom.
+  size_t index() { return m_currentIndex; }
+  CSSParserTokenRange makeSubRangeFrom(size_t startIndex) const {
+    return CSSParserTokenRange(m_tokenizer.m_tokens)
+        .makeSubRange(m_tokenizer.m_tokens.begin() + startIndex,
+                      m_tokenizer.m_tokens.begin() + m_currentIndex);
+  }
+
+  void yieldComments(CSSParserObserver& observer) {
+    if (hasLookedAhead())
+      DCHECK(atEnd());
+    else
+      m_tokenizer.yieldComments(observer);
+  }
+
+ private:
+  bool hasLookedAhead() {

[Charlie Harrison, 2017/01/09 21:35:07]

Maybe replace this method with:

DCHECK(m_currentIndex == m_tokenizer.size() || m_currentIndex ==
m_tokenizer.m_tokens.size() - 1);
return m_currentIndex != m_tokenizer.m_tokens.size();

Consider this optional if the current code is successfully optimized away
though.

+    if (m_currentIndex == m_tokenizer.m_tokens.size())
+      return false;
+    DCHECK_EQ(m_currentIndex, m_tokenizer.m_tokens.size() - 1);
+    return true;
+  }
+
+  const CSSParserToken& peekInternal();
+
+  CSSTokenizer& m_tokenizer;
+  size_t m_startIndex;
+  size_t m_currentIndex;
+  bool m_isSubStream;
+};
+
+template <typename... emptyBaseCase>
+inline bool typeIs(CSSParserTokenType type) {
+  return false;
+}
+
+template <CSSParserTokenType head, CSSParserTokenType... tail>
+inline bool typeIs(CSSParserTokenType type) {
+  return type == head || typeIs<tail...>(type);
+}
+
+template <CSSParserTokenType... types>
+void CSSParserTokenStream::consumeUntilAtEndOrPeekedTypeIs() {
+  unsigned nestingLevel = 0;
+  const CSSParserToken* token = &peekInternal();
+  if (token->type() == EOFToken)
+    return;
+
+  while (nestingLevel || !typeIs<types...>(token->type())) {
+    if (token->getBlockType() == CSSParserToken::BlockStart) {
+      nestingLevel++;
+    } else if (token->getBlockType() == CSSParserToken::BlockEnd) {
+      if (nestingLevel == 0)
+        return;
+      nestingLevel--;
+    }
+
+    m_currentIndex++;
+    m_tokenizer.tokenizeSingle();
+    if (m_tokenizer.m_finishedTokenizing)
+      return;
+    token = &m_tokenizer.m_tokens[m_currentIndex];
+  }
+}
+
+}  // namespace blink
+
+#endif  // CSSParserTokenStream_h