Instrument DomDistiller with timing information.

src/com/dom_distiller/client/ContentExtractor.java

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package com.dom_distiller.client;
 
+import com.dom_distiller.proto.DomDistillerProtos;
+import com.dom_distiller.proto.DomDistillerProtos.TimingInfo;
 import com.google.gwt.dom.client.AnchorElement;
 import com.google.gwt.dom.client.Document;
 import com.google.gwt.dom.client.Element;
 import com.google.gwt.dom.client.Node;
 import com.google.gwt.dom.client.NodeList;
 import com.google.gwt.dom.client.VideoElement;
 
 import de.l3s.boilerpipe.BoilerpipeProcessingException;
 import de.l3s.boilerpipe.document.TextBlock;
 import de.l3s.boilerpipe.document.TextDocument;
 import de.l3s.boilerpipe.extractors.CommonExtractors;
 import de.l3s.boilerpipe.labels.DefaultLabels;
 import de.l3s.boilerpipe.sax.BoilerpipeHTMLContentHandler;
 
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.logging.Logger;
 
 public class ContentExtractor {
-    static Logger logger = Logger.getLogger("DomDistiller");
+    private final Element documentElement;
 
-    private final Element documentElement;
+    private final List<String> candidateTitles;
+
+    private final TimingInfo mTimingInfo;
 
     private final MarkupParser parser;
 
-    private final List<String> candidateTitles;
-
     public ContentExtractor(Element root) {
         this.documentElement = root;
+        this.candidateTitles = new LinkedList<String>();
+        this.mTimingInfo = DomDistillerProtos.TimingInfo.create();
+
+        double startTime = DomUtil.getTime();
         this.parser = new MarkupParser(root);
-        this.candidateTitles = new LinkedList<String>();
+        this.mTimingInfo.setMarkupParsingTime(DomUtil.getTime() - startTime);
     }
 
     // Grabs a list of candidate titles in descending priority order:
     // 1) meta-information
     // 2) The document's title element, modified based on some readability heuristics
     // 3) The document's title element, if it's a string
     private void ensureTitleInitialized() {
         if (candidateTitles.size() > 0) return;
 
         String title = parser.getTitle();
@@ skipping 13 matching lines @@
         ensureTitleInitialized();
         assert candidateTitles.size() > 0;
         return candidateTitles.get(0);
     }
 
     public String extractContent() {
         return extractContent(false);
     }
 
     public String extractContent(boolean textOnly) {
+        double now = DomUtil.getTime();
+        TextDocument document = createTextBlocksFromPage();
+        mTimingInfo.setDocumentConstructionTime(DomUtil.getTime() - now);
+
+        now = DomUtil.getTime();
+        List<Node> contentNodes = processTextBlocks(document);
+        mTimingInfo.setArticleProcessingTime(DomUtil.getTime() - now);
+
+        if (contentNodes.isEmpty()) {
+            return "";
+        }
+
+        now = DomUtil.getTime();
+        String html = formatExtractedNodes(textOnly, contentNodes);
+        mTimingInfo.setFormattingTime(DomUtil.getTime() - now);
+        return html;
+    }
+
+
+    /**
+     * Returns timing information about the most recent extraction run.
+     * @return an instance of DomDistillerProtos.TimingInfo with detailed timing statistics.
+     */
+    public TimingInfo getTimingInfo() {
+        return mTimingInfo;
+    }
+
+    /**
+     * Converts the original HTML page into a series of TextBlock for analysis.
+     * @return a document with the list of extracted TextBlocks and additional information
+     *         that can be useful for identifying the core elements of the page.
+     */
+    private TextDocument createTextBlocksFromPage() {
         BoilerpipeHTMLContentHandler htmlParser = new BoilerpipeHTMLContentHandler();
-
         htmlParser.startDocument();
         DomToSaxVisitor domToSaxVisitor = new DomToSaxVisitor(htmlParser);
         FilteringDomVisitor filteringDomVisitor = new FilteringDomVisitor(domToSaxVisitor);
         new DomWalker(filteringDomVisitor).walk(documentElement);
         htmlParser.endDocument();
-
         TextDocument document = htmlParser.toTextDocument();
         ensureTitleInitialized();
+        document.setCandidateTitles(candidateTitles);
+        document.setHiddenElements(filteringDomVisitor.getHiddenElements());
+        document.setDataTables(filteringDomVisitor.getDataTables());
+        return document;
+    }
 
-        document.setCanddiateTitles(candidateTitles);
+    /**
+     * Implements the actual analysis of the page content, identifying the core elements of the
+     * page.
+     * @param document the TextBlock representation of the page extracted from the DOM.
+     * @return a list of DOM nodes from the original document that contain the core elements of the
+     *         page.
+     */
+    private List<Node> processTextBlocks(TextDocument document) {
         try {
             CommonExtractors.ARTICLE_EXTRACTOR.process(document);
         } catch (BoilerpipeProcessingException e) {
-            logger.warning("Processing failed.");
-            return "";
+            LogUtil.logToConsole("DomDistiller Processing failed: " + e);
+            return new LinkedList<Node>();
         }
 
-
         List<Node> contentNodes = getContentNodesForTextDocument(document);
 
         List<Node> contentAndRelevantElements = RelevantElementsFinder.findAndAddElements(
-                contentNodes, filteringDomVisitor.getHiddenElements(),
-                filteringDomVisitor.getDataTables(), Document.get().getDocumentElement());
+                contentNodes, document.getHiddenElements(),
+                document.getDataTables(), Document.get().getDocumentElement());
+        return contentAndRelevantElements;
+    }
 
-        if (contentAndRelevantElements.isEmpty()) {
-            return "";
-        }
-
-        Node clonedSubtree = NodeListExpander.expand(contentAndRelevantElements).cloneSubtree();
-
+    /**
+     * Creates a new minimal HTML document containing copies of the DOM nodes identified as the
+     * core elements of the page. Some additional re-formatting hints may be included in the new
+     * document.
+     *
+     * @param textOnly indicates whether to simply return the aggregated text content instead of
+     *        HTML
+     * @param contentNodes the DOM nodes containing text to be included in the final docuemnt.
+     * @return A HTML or text document which includes the aggregated content of the provided HTML
+     *        nodes.
+     */
+    private String formatExtractedNodes(boolean textOnly, List<Node> contentNodes) {
+        Node clonedSubtree = NodeListExpander.expand(contentNodes).cloneSubtree();
         if (clonedSubtree.getNodeType() != Node.ELEMENT_NODE) {
             return "";
         }
 
         // The base URL in the distilled page viewer is different from that in
         // the live page.  This breaks all relative links (in anchors,
         // images, etc.), so make them absolute in the distilled content.
         makeAllLinksAbsolute(clonedSubtree);
-
         if (textOnly) {
             return getTextFromTree(clonedSubtree);
         }
 
         // TODO(cjhopman): this discards the top element and just returns its children. This might
         // break in some cases.
         return Element.as(clonedSubtree).getInnerHTML();
     }
 
     /**
@@ skipping 71 matching lines @@
             return;
         }
         var elementsWithSrc = root.querySelectorAll('img,source,track,video');
         for (var key in elementsWithSrc) {
             if (elementsWithSrc[key].src) {
                 elementsWithSrc[key].src = elementsWithSrc[key].src;
             }
         }
     }-*/;
 }