Fix final content and title extraction.

src/com/dom_distiller/client/ContentExtractor.java

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package com.dom_distiller.client;
 
 import com.google.gwt.dom.client.AnchorElement;
 import com.google.gwt.dom.client.Document;
 import com.google.gwt.dom.client.Element;
 import com.google.gwt.dom.client.ImageElement;
@@ skipping 55 matching lines @@
         List<Node> contentNodes = getContentNodesForTextDocument(document, textNodes);
 
         List<Node> contentAndImages = RelevantImageFinder.findAndAddImages(
                 contentNodes, Document.get().getDocumentElement());
 
         if (contentAndImages.isEmpty()) {
             return "";
         }
 
         Node clonedSubtree = NodeListExpander.expand(contentAndImages).cloneSubtree();
-
+        LogUtil.logToConsole(clonedSubtree.getNodeType() + " node:" + Node.ELEMENT_NODE);

[cjhopman, 2014/05/22 00:27:21]

probably don't need this. If it's been useful, at least guard it with some debug
flag.

[Yaron, 2014/05/22 17:05:24]

err right that was supposed to go. I was trying to investigate something with
small pages. I believe you can fail the below test if we return just a single
text node but can't repro it now.

         if (clonedSubtree.getNodeType() != Node.ELEMENT_NODE) {
             return "";
         }
 
         // The base URL in the distilled page viewer is different from that in
         // the live page.  This breaks all relative links (in anchors and
         // images), so make them absolute in the distilled content.
         makeAllLinksAbsolute(clonedSubtree);
 
         // TODO(cjhopman): this discards the top element and just returns its children. This might
         // break in some cases.
         return Element.as(clonedSubtree).getInnerHTML();
     }
 
     private static List<Node> parse(Element e, ContentHandler handler) {
         DomToSaxVisitor domToSaxVisitor = new DomToSaxVisitor(handler);
         FilteringDomVisitor filteringDomVisitor = new FilteringDomVisitor(domToSaxVisitor);
         new DomWalker(filteringDomVisitor).walk(e);
         return domToSaxVisitor.getTextNodes();
     }
 
     private static List<Node> getContentNodesForTextDocument(
             TextDocument document, List<Node> textNodes) {
         List<Integer> contentTextIndexes = new ArrayList<Integer>();
         for (TextBlock tb : document.getTextBlocks()) {
+            if (!tb.isContent()) {
+                continue;
+            }
             if (!tb.hasLabel(DefaultLabels.TITLE)) {
                 contentTextIndexes.addAll(tb.getContainedTextElements());
             }
         }
         Collections.sort(contentTextIndexes);
 
         // Boilerpipe's text node indexes start at 1.
         List<Node> contentNodes = new ArrayList<Node>(contentTextIndexes.size());
         for (Integer i : contentTextIndexes) {
             contentNodes.add(textNodes.get(i - 1));
@@ skipping 13 matching lines @@
             link.setHref(link.getHref());
         }
 
         NodeList<Element> allImages = root.getElementsByTagName("IMG");
         for (int i = 0; i < allImages.getLength(); i++) {
             ImageElement image = ImageElement.as(allImages.getItem(i));
             image.setSrc(image.getSrc());
         }
     }
 }