Make it possible to modify background HTML parser token limits.

Source/core/html/parser/BackgroundHTMLParser.cpp

 /*
  * Copyright (C) 2013 Google, Inc. All Rights Reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 18 matching lines @@
 #include "core/HTMLNames.h"
 #include "core/html/parser/HTMLDocumentParser.h"
 #include "core/html/parser/TextResourceDecoder.h"
 #include "core/html/parser/XSSAuditor.h"
 #include "platform/Task.h"
 #include "platform/ThreadSafeFunctional.h"
 #include "public/platform/Platform.h"
 #include "public/platform/WebScheduler.h"
 #include "wtf/text/TextPosition.h"
 
+#include <algorithm>
+
 namespace blink {
 
 // On a network with high latency and high bandwidth, using a device
 // with a fast CPU, we could end up speculatively tokenizing
 // the whole document, well ahead of when the main-thread actually needs it.
 // This is a waste of memory (and potentially time if the speculation fails).
 // So we limit our outstanding tokens arbitrarily to 10,000.
 // Our maximal memory spent speculating will be approximately:
-// (outstandingTokenLimit + pendingTokenLimit) * sizeof(CompactToken)
+// (defaultOutstandingTokenLimit + defaultPendingTokenLimit) *
+//     sizeof(CompactToken)
 // We use a separate low and high water mark to avoid constantly topping
 // off the main thread's token buffer.
 // At time of writing, this is (10000 + 1000) * 28 bytes = ~308kb of memory.
 // These numbers have not been tuned.
-static const size_t outstandingTokenLimit = 10000;
+static const size_t defaultOutstandingTokenLimit = 10000;
 
 // We limit our chucks to 1000 tokens, to make sure the main
 // thread is never waiting on the parser thread for tokens.
 // This was tuned in https://bugs.webkit.org/show_bug.cgi?id=110408.
-static const size_t pendingTokenLimit = 1000;
+static const size_t defaultPendingTokenLimit = 1000;
 
 using namespace HTMLNames;
 
 #if ENABLE(ASSERT)
 
 static void checkThatTokensAreSafeToSendToAnotherThread(const CompactHTMLTokenStream* tokens)
 {
     for (size_t i = 0; i < tokens->size(); ++i)
         ASSERT(tokens->at(i).isSafeToSendToAnotherThread());
 }
@@ skipping 11 matching lines @@
 }
 
 #endif
 
 void BackgroundHTMLParser::start(PassRefPtr<WeakReference<BackgroundHTMLParser>> reference, PassOwnPtr<Configuration> config, WebScheduler* scheduler)
 {
     new BackgroundHTMLParser(reference, config, scheduler);
     // Caller must free by calling stop().
 }
 
+BackgroundHTMLParser::Configuration::Configuration()
+    : outstandingTokenLimit(defaultOutstandingTokenLimit)
+    , pendingTokenLimit(defaultPendingTokenLimit)
+{
+}
+
 BackgroundHTMLParser::BackgroundHTMLParser(PassRefPtr<WeakReference<BackgroundHTMLParser>> reference, PassOwnPtr<Configuration> config, WebScheduler* scheduler)
     : m_weakFactory(reference, this)
     , m_token(adoptPtr(new HTMLToken))
     , m_tokenizer(HTMLTokenizer::create(config->options))
     , m_treeBuilderSimulator(config->options)
     , m_options(config->options)
+    , m_outstandingTokenLimit(std::max(config->outstandingTokenLimit, config->pendingTokenLimit))

[Nate Chapin, 2015/06/11 18:48:23]

Why take the max of the two here? Should we assert that outstanding is always
larger than pending in the Configuration?

[Bryan McQuade, 2015/06/12 14:12:59]

Yes, that's clearer. Agree. I've made this change.

     , m_parser(config->parser)
     , m_pendingTokens(adoptPtr(new CompactHTMLTokenStream))
+    , m_pendingTokenLimit(config->pendingTokenLimit)
     , m_xssAuditor(config->xssAuditor.release())
     , m_preloadScanner(config->preloadScanner.release())
     , m_decoder(config->decoder.release())
     , m_scheduler(scheduler)
     , m_startingScript(false)
 {
+    ASSERT(m_outstandingTokenLimit > 0);
+    ASSERT(m_pendingTokenLimit > 0);
 }
 
 BackgroundHTMLParser::~BackgroundHTMLParser()
 {
 }
 
 void BackgroundHTMLParser::appendRawBytesFromParserThread(const char* data, int dataLength)
 {
     ASSERT(m_decoder);
     updateDocument(m_decoder->decode(data, dataLength));
@@ skipping 85 matching lines @@
     ASSERT(!m_input.current().isClosed());
     m_input.append(String(&kEndOfFileMarker, 1));
     m_input.close();
 }
 
 void BackgroundHTMLParser::pumpTokenizer()
 {
     HTMLTreeBuilderSimulator::SimulatedToken simulatedToken = HTMLTreeBuilderSimulator::OtherToken;
 
     // No need to start speculating until the main thread has almost caught up.
-    if (m_input.totalCheckpointTokenCount() > outstandingTokenLimit)
+    if (m_input.totalCheckpointTokenCount() > m_outstandingTokenLimit)
         return;
 
     while (true) {
         m_sourceTracker.start(m_input.current(), m_tokenizer.get(), *m_token);
         if (!m_tokenizer->nextToken(m_input.current(), *m_token)) {
             // We've reached the end of our current input.
             sendTokensToMainThread();
             break;
         }
         m_sourceTracker.end(m_input.current(), m_tokenizer.get(), *m_token);
@@ skipping 16 matching lines @@
             if (simulatedToken == HTMLTreeBuilderSimulator::ScriptStart) {
                 sendTokensToMainThread();
                 m_startingScript = true;
             }
 
             m_pendingTokens->append(token);
         }
 
         m_token->clear();
 
-        if (simulatedToken == HTMLTreeBuilderSimulator::ScriptEnd || m_pendingTokens->size() >= pendingTokenLimit) {
+        if (simulatedToken == HTMLTreeBuilderSimulator::ScriptEnd || m_pendingTokens->size() >= m_pendingTokenLimit) {
             sendTokensToMainThread();
             // If we're far ahead of the main thread, yield for a bit to avoid consuming too much memory.
-            if (m_input.totalCheckpointTokenCount() > outstandingTokenLimit)
+            if (m_input.totalCheckpointTokenCount() > m_outstandingTokenLimit)
                 break;
         }
     }
 }
 
 void BackgroundHTMLParser::sendTokensToMainThread()
 {
     if (m_pendingTokens->isEmpty())
         return;
 
@@ skipping 15 matching lines @@
     m_startingScript = false;
 
     m_scheduler->postLoadingTask(
         FROM_HERE,
         new Task(threadSafeBind(&HTMLDocumentParser::didReceiveParsedChunkFromBackgroundParser, AllowCrossThreadAccess(m_parser), chunk.release())));
 
     m_pendingTokens = adoptPtr(new CompactHTMLTokenStream);
 }
 
 }