Renderer-side changes for NoState Prefetch

third_party/WebKit/Source/core/html/parser/HTMLDocumentParser.cpp

 /*
  * Copyright (C) 2010 Google, Inc. All Rights Reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 71 matching lines @@
         || (contextTag.matches(noscriptTag) && options.scriptEnabled)
         || contextTag.matches(noframesTag))
         return reportErrors ? HTMLTokenizer::RAWTEXTState : HTMLTokenizer::PLAINTEXTState;
     if (contextTag.matches(scriptTag))
         return reportErrors ? HTMLTokenizer::ScriptDataState : HTMLTokenizer::PLAINTEXTState;
     if (contextTag.matches(plaintextTag))
         return HTMLTokenizer::PLAINTEXTState;
     return HTMLTokenizer::DataState;
 }
 
+// static
+HTMLDocumentParser* HTMLDocumentParser::create(

[Charlie Harrison, 2016/08/03 14:40:03]

I have a slight preference for passing the sync policy from the callsite,
instead of reaching into the document in threadingAllowedForDocument.

[droger, 2016/08/03 16:45:50]

Done.

+    HTMLDocument& document,
+    ParserSynchronizationPolicy backgroundParsingPolicy)
+{
+    return new HTMLDocumentParser(
+        document,
+        threadingAllowedForDocument(document)
+            ? backgroundParsingPolicy : ForceSynchronousParsing);
+}
+
 HTMLDocumentParser::HTMLDocumentParser(HTMLDocument& document, ParserSynchronizationPolicy syncPolicy)
     : HTMLDocumentParser(document, AllowScriptingContent, syncPolicy)
 {
     m_scriptRunner = HTMLScriptRunner::create(&document, this);
     m_treeBuilder = HTMLTreeBuilder::create(this, document, AllowScriptingContent, m_options);
 }
 
 HTMLDocumentParser::HTMLDocumentParser(DocumentFragment* fragment, Element* contextElement, ParserContentPolicy parserContentPolicy)
     : HTMLDocumentParser(fragment->document(), parserContentPolicy, ForceSynchronousParsing)
 {
@@ skipping 41 matching lines @@
 {
     visitor->trace(m_treeBuilder);
     visitor->trace(m_parserScheduler);
     visitor->trace(m_xssAuditorDelegate);
     visitor->trace(m_scriptRunner);
     visitor->trace(m_preloader);
     ScriptableDocumentParser::trace(visitor);
     HTMLScriptRunnerHost::trace(visitor);
 }
 
+// static
+bool HTMLDocumentParser::threadingAllowedForDocument(HTMLDocument& document)
+{
+    return !document.isPrefetchOnly();
+}
+
 void HTMLDocumentParser::detach()
 {
     if (!isParsingFragment() && m_tokenizedChunkQueue.get() && m_tokenizedChunkQueue->peakPendingChunkCount()) {
         DEFINE_STATIC_LOCAL(CustomCountHistogram, peakPendingChunkHistogram, ("Parser.PeakPendingChunkCount", 1, 1000, 50));
         peakPendingChunkHistogram.count(m_tokenizedChunkQueue->peakPendingChunkCount());
         DEFINE_STATIC_LOCAL(CustomCountHistogram, peakPendingTokenHistogram, ("Parser.PeakPendingTokenCount", 1, 100000, 50));
         peakPendingTokenHistogram.count(m_tokenizedChunkQueue->peakPendingTokenCount());
     }
 
     if (m_haveBackgroundParser)
@@ skipping 537 matching lines @@
     excludedLineNumberSource.setExcludeLineNumbers();
     m_input.insertAtCurrentInsertionPoint(excludedLineNumberSource);
     pumpTokenizerIfPossible();
 
     if (isWaitingForScripts()) {
         // Check the document.write() output with a separate preload scanner as
         // the main scanner can't deal with insertions.
         if (!m_insertionPreloadScanner)
             m_insertionPreloadScanner = createPreloadScanner();
         m_insertionPreloadScanner->appendToEnd(source);
-        m_insertionPreloadScanner->scanAndPreload(m_preloader.get(), document()->validBaseElementURL(), nullptr);
+        m_insertionPreloadScanner->scanAndPreload(

[Charlie Harrison, 2016/08/03 14:40:03]

nit: Unnecessary line break.

[droger, 2016/08/03 16:45:50]

Done.

+            m_preloader.get(), document()->validBaseElementURL(), nullptr);
     }
 
     endIfDelayed();
 }
 
 void HTMLDocumentParser::startBackgroundParser()
 {
     ASSERT(!isStopped());
     ASSERT(shouldUseThreading());
     ASSERT(!m_haveBackgroundParser);
@@ skipping 60 matching lines @@
     if (isStopped())
         return;
 
     // We should never reach this point if we're using a parser thread,
     // as appendBytes() will directly ship the data to the thread.
     ASSERT(!shouldUseThreading());
 
     TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("blink.debug"), "HTMLDocumentParser::append", "size", inputSource.length());
     const SegmentedString source(inputSource);
 
+    if (document()->isPrefetchOnly()) {
+        if (!m_preloadScanner)
+            m_preloadScanner = createPreloadScanner();
+
+        m_preloadScanner->appendToEnd(source);
+        m_preloadScanner->scanAndPreload(m_preloader.get(), document()->validBaseElementURL(), nullptr);

[Charlie Harrison, 2016/08/03 14:40:03]

This just preloads the tokens normally. Do we want to add load flags like
LOAD_PREFETCH so all resources are forced into the HTTP cache?

Also, if we aren't using this render process for the final page load, this will
bloat the MemoryCache with in-memory resources that will never be used.

+shivanisha@ is thinking of implementing a "fire-and-forget" system in Blink
that will send requests but ignore results (make sure they don't land in the
memory cache). It might make sense to collaborate on this.

(note: I understand this is just a prototype, I just want to understand what the
final design will look like.)

[droger, 2016/08/03 16:45:50]

This uses the same code as the standard preload scanner. I would think that it
uses the LOAD_PREFETCH flags accordingly, and (at first at least) we want to
have the same behavior for prefetch.
Yes, this is a good point. Disabling the memory cache would probably be nice.

[Charlie Harrison, 2016/08/03 17:15:14]

Nope, LOAD_PREFETCH afaik is only used for the link rel="prefetch" feature.
These loads will look like normal requests from the net stack, and will not
follow special caching behavior like LOAD_PREFETCH.

I did a brief trace and you can see that the load flag is added when the request
has RequestContext::RequestContextPrefetch which only happens for link rel
prefetches. One thing that you could do in this patch to solve this problem is
to inspect the preloads that are being scanned, and set their type to
LinkRelPrefetch. This would require:
1. Adding "setForPrefetch" to PreloadRequest.h which sets the resource type to
LinkRelPrefetch.
2. one of:
 a. Modifying HTMLPreloadScanner in some way to do this for you.
 b. Using a TokenPreloadScanner instead of HTMLPreloadScanner (look at
HTMLPreloadScanner for how to do this)

If you'd like LOAD_PREFETCH on your requests, you'll need to do this otherwise
you will double download no-store resources, etc.

This technique sounds reasonable to me, but again, I am not an owner so you'll
need to get sign-off from some blink owner :)

[pasko, 2016/08/04 12:23:08]

csharrison: thank you for investigation and the info. Let's handle load flags
properly in a separate patch. I indeed see that LOAD_PREFETCH is not set in the
net stack, but strangely the following page load still fetches from cache
ignoring validators. Hmm.. I thought only LOAD_PREFETCH can do that..

+
+        // Return after the preload scanner, do not actually parse the document.
+        return;
+    }
+
     if (m_preloadScanner) {
         if (m_input.current().isEmpty() && !isWaitingForScripts()) {
             // We have parsed until the end of the current input and so are now moving ahead of the preload scanner.
             // Clear the scanner so we know to scan starting from the current input point if we block again.
             m_preloadScanner.reset();
         } else {
             m_preloadScanner->appendToEnd(source);
             if (isWaitingForScripts())
                 m_preloadScanner->scanAndPreload(m_preloader.get(), document()->validBaseElementURL(), nullptr);
         }
@@ skipping 226 matching lines @@
 void HTMLDocumentParser::resumeScheduledTasks()
 {
     ASSERT(m_tasksWereSuspended);
     m_tasksWereSuspended = false;
     if (m_parserScheduler)
         m_parserScheduler->resume();
 }
 
 void HTMLDocumentParser::appendBytes(const char* data, size_t length)
 {
     if (!length || isStopped())

[Charlie Harrison, 2016/08/03 14:40:03]

Can you DCHECK(!document()->isPrefetchOnly()) here?

[droger, 2016/08/03 16:45:50]

Why? Is this because isStopped() should never be true when prefetching?

[Charlie Harrison, 2016/08/03 17:15:14]

This method should not run in the non-threaded model (pretty sure).

[droger, 2016/08/04 12:42:29]

Is "This method" appendBytes()? Or isStopped()?

appendBytes() is run for me, and looking at the "if (shouldUseThreading())" line
1069 below, I think this is expected.

[Charlie Harrison, 2016/08/04 12:44:41]

Ah you're right, I was misremember the control flow for the non-threaded case.
Apologies.

         return;
 
     if (shouldUseThreading()) {
         double bytesReceivedTime = monotonicallyIncreasingTimeMS();
         if (!m_haveBackgroundParser)
             startBackgroundParser();
 
         std::unique_ptr<Vector<char>> buffer = wrapUnique(new Vector<char>(length));
         memcpy(buffer->data(), data, length);
         TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("blink.debug"), "HTMLDocumentParser::appendBytes", "size", (unsigned)length);
@@ skipping 115 matching lines @@
         (*WTF::bind(function, std::forward<Ps>(parameters)...))();
         return;
     case Asynchronous:
         m_loadingTaskRunner->postTask(BLINK_FROM_HERE, WTF::bind(function, std::forward<Ps>(parameters)...));
         return;
     }
     NOTREACHED();
 }
 
 } // namespace blink