Make the HTMLDocumentParser yield more aggressively

Source/core/html/parser/BackgroundHTMLParser.cpp

 /*
  * Copyright (C) 2013 Google, Inc. All Rights Reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 26 matching lines @@
 // On a network with high latency and high bandwidth, using a device
 // with a fast CPU, we could end up speculatively tokenizing
 // the whole document, well ahead of when the main-thread actually needs it.
 // This is a waste of memory (and potentially time if the speculation fails).
 // So we limit our outstanding tokens arbitrarily to 10,000.
 // Our maximal memory spent speculating will be approximately:
 // (outstandingTokenLimit + pendingTokenLimit) * sizeof(CompactToken)
 // We use a separate low and high water mark to avoid constantly topping
 // off the main thread's token buffer.
 // At time of writing, this is (10000 + 1000) * 28 bytes = ~308kb of memory.
 // These numbers have not been tuned.

[abarth-chromium, 2014/04/29 18:29:00]

Can you update the math in this comment?

-static const size_t outstandingTokenLimit = 10000;

[eseidel, 2014/04/29 00:23:13]

I didn't mean to leave these in.

[eseidel, 2014/04/29 00:24:12]

These aren't bad.  I think they can just be deferred to a later change with more
mobile testing.

+static const size_t outstandingTokenLimit = 1000;
 
 // We limit our chucks to 1000 tokens, to make sure the main

[abarth-chromium, 2014/04/29 18:29:00]

This comment is now out of date.

 // thread is never waiting on the parser thread for tokens.
 // This was tuned in https://bugs.webkit.org/show_bug.cgi?id=110408.
-static const size_t pendingTokenLimit = 1000;
+static const size_t pendingTokenLimit = 100;
 
 using namespace HTMLNames;
 
 #ifndef NDEBUG
 
 static void checkThatTokensAreSafeToSendToAnotherThread(const CompactHTMLTokenStream* tokens)
 {
     for (size_t i = 0; i < tokens->size(); ++i)
         ASSERT(tokens->at(i).isSafeToSendToAnotherThread());
 }
@@ skipping 188 matching lines @@
     chunk->treeBuilderState = m_treeBuilderSimulator.state();
     chunk->inputCheckpoint = m_input.createCheckpoint(m_pendingTokens->size());
     chunk->preloadScannerCheckpoint = m_preloadScanner->createCheckpoint();
     chunk->tokens = m_pendingTokens.release();
     callOnMainThread(bind(&HTMLDocumentParser::didReceiveParsedChunkFromBackgroundParser, m_parser, chunk.release()));
 
     m_pendingTokens = adoptPtr(new CompactHTMLTokenStream);
 }
 
 }