Discard hidden articles when using fast path

javatests/org/chromium/distiller/DomUtilTest.java

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package org.chromium.distiller;
 
 import org.chromium.distiller.webdocument.WebTable;
 
 import com.google.gwt.core.client.JsArray;
 import com.google.gwt.dom.client.Document;
 import com.google.gwt.dom.client.Element;
 import com.google.gwt.dom.client.Node;
+import com.google.gwt.dom.client.NodeList;
 
 import java.util.Map;
 import java.util.List;
 
 public class DomUtilTest extends DomDistillerJsTestCase {
+    private static final String CONTENT_TEXT = "Lorem Ipsum Lorem Ipsum Lorem Ipsum.";
+    private static final String TITLE_TEXT = "I am the document title";
+
     public void testGetAttributes() {
         Element e = Document.get().createDivElement();
         e.setInnerHTML("<div style=\"width:50px; height:100px\" id=\"f\" class=\"sdf\"></div>");
         e = Element.as(e.getChildNodes().getItem(0));
         JsArray<Node> jsAttrs = DomUtil.getAttributes(e);
         assertEquals(3, jsAttrs.length());
         assertEquals("style", jsAttrs.get(0).getNodeName());
         assertEquals("width:50px; height:100px", jsAttrs.get(0).getNodeValue());
         assertEquals("id", jsAttrs.get(1).getNodeName());
         assertEquals("f", jsAttrs.get(1).getNodeValue());
@@ skipping 75 matching lines @@
 
         Element currDiv = TestUtil.createDiv(3);
         div2.appendChild(currDiv);
         Element finalDiv1 = currDiv;
 
         currDiv = TestUtil.createDiv(4);
         div2.appendChild(currDiv);
         currDiv.appendChild(TestUtil.createDiv(5));
 
         assertEquals(div2, DomUtil.getNearestCommonAncestor(finalDiv1, currDiv.getChild(0)));
-        assertEquals(div2, DomUtil.getNearestCommonAncestor(
-                DomUtil.querySelectorAll(mRoot, "[id=\"3\"],[id=\"5\"]")));
+        NodeList<Element> nodeList = DomUtil.querySelectorAll(mRoot, "[id=\"3\"],[id=\"5\"]");
+        assertEquals(div2, DomUtil.getNearestCommonAncestor(TestUtil.nodeListToList(nodeList)));
+
     }
 
     /**
      * The tree graph is:
      * 1 - 2 - 3
      */
     public void testNearestCommonAncestorIsRoot() {
         Element div = TestUtil.createDiv(1);
         mBody.appendChild(div);
 
         Element div2 = TestUtil.createDiv(2);
         div.appendChild(div2);
 
         Element div3 = TestUtil.createDiv(3);
         div2.appendChild(div3);
 
         assertEquals(div, DomUtil.getNearestCommonAncestor(div, div3));
-        assertEquals(div, DomUtil.getNearestCommonAncestor(
-                DomUtil.querySelectorAll(mRoot, "[id=\"1\"],[id=\"3\"]")));
+        NodeList<Element> nodeList = DomUtil.querySelectorAll(mRoot, "[id=\"1\"],[id=\"3\"]");
+        assertEquals(div, DomUtil.getNearestCommonAncestor(TestUtil.nodeListToList(nodeList)));
     }
 
     public void testNodeDepth() {
         Element div = TestUtil.createDiv(1);
 
         Element div2 = TestUtil.createDiv(2);
         div.appendChild(div2);
 
         Element div3 = TestUtil.createDiv(3);
         div2.appendChild(div3);
@@ skipping 194 matching lines @@
                         "<td>text</td>" +
                         "<td>text</td>" +
                     "</tr>" +
                 "</tbody>" +
             "</table>";
 
         mBody.setInnerHTML(html);
         DomUtil.stripStyleAttributes(mBody);
         assertEquals(expected, mBody.getInnerHTML());
     }
+
+    public void testIsVisibleByItsOffsetParentDisplayNone() {
+        String html =
+            "<div style=\"display: none;\">" +
+                "<div>Some Text</div>" +
+            "</div>";
+        mBody.setInnerHTML(html);
+        Element child = mBody.getFirstChildElement().getFirstChildElement();
+        assertFalse(DomUtil.isVisibleByItsOffset(child));
+    }
+
+    public void testIsVisibleByItsOffsetChildDisplayNone() {
+        String html =
+            "<div>" +
+                "<div style=\"display: none;\">Some Text</div>" +
+            "</div>";
+        mBody.setInnerHTML(html);
+        Element child = mBody.getFirstChildElement().getFirstChildElement();
+        assertFalse(DomUtil.isVisibleByItsOffset(child));
+    }
+
+    public void testIsVisibleByItsOffsetDisplayBlock() {
+        String html =
+            "<div>" +
+                "<div>Some Text</div>" +
+            "</div>";
+        mBody.setInnerHTML(html);
+        Element child = mBody.getFirstChildElement().getFirstChildElement();
+        assertTrue(DomUtil.isVisibleByItsOffset(child));
+    }
+
+    public void testOnlyProcessArticleElement() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";

[wychen, 2015/12/10 23:33:36]

The reason I inserted these text was for ContentExtractor to recognize them as
content. If we only test getArticleElement(), the html can be simplified to
empty elements. For example, <h1></h1><article></article> should work.

+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<article>" + article + "</article>";
+
+        assertNotNull(getArticleElement(htmlArticle));

[wychen, 2015/12/10 23:33:37]

Sorry for changing my mind back and forth. It might make sense to assert the
correct element is returned, instead of just not null.

+    }
+
+    public void testOnlyProcessArticleElementWithHiddenArticleElement() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<article>" + article + "</article>" +
+            "<article style=\"display:none\">" + article + "</article>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessArticleElementMultiple() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<article>" + article + "</article>" +
+            "<article>" + article + "</article>";
+
+        // The existence of multiple articles disables the fast path.
+        assertNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessArticleElementMultipleWithHiddenArticleElement() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<article>" + article + "</article>" +
+            "<article style=\"display:none\">" + article + "</article>" +
+            "<article>" + article + "</article>";
+
+        // The existence of multiple articles disables the fast path.
+        assertNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticle() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/Article\">" + article + "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticleWithHiddenArticleElement() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/Article\">" + article + "</div>" +
+            "<div itemscope itemtype=\"http://schema.org/Article\" style=\"display:none\">" +
+                article + "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticleNews() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/NewsArticle\">" + article + "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticleNewsWithHiddenArticleElement() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/NewsArticle\">" + article + "</div>" +
+            "<div itemscope itemtype=\"http://schema.org/NewsArticle\" style=\"display:none\">" +
+                article + "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticleBlog() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/BlogPosting\">" + article + "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticleBlogWithHiddenArticleElement() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/BlogPosting\">" + article + "</div>" +
+            "<div itemscope itemtype=\"http://schema.org/BlogPosting\" style=\"display:none\">" +
+                article + "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticleNested() {
+        final String paragraph = "<p>" + CONTENT_TEXT + "</p>";
+        final String article = paragraph + paragraph;
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/Article\">" +
+                paragraph +
+                "<div itemscope itemtype=\"http://schema.org/Article\">" + paragraph + "</div>" +
+            "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGArticleNestedWithNestedHiddenArticleElement() {
+        final String paragraph = "<p>" + CONTENT_TEXT + "</p>";
+        final String article = paragraph + paragraph;
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/Article\">" +
+                paragraph +
+                "<div itemscope itemtype=\"http://schema.org/Article\">" + paragraph + "</div>" +
+                "<div itemscope itemtype=\"http://schema.org/Article\" style=\"display:none\">" +
+                article + "</div>" +
+            "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));

[wychen, 2015/12/10 23:33:37]

Ditto. Making sure which element is picked makes even more sense for these
complicated cases.

+    }
+
+    public void testOnlyProcessOGArticleNestedWithHiddenArticleElement() {
+        final String paragraph = "<p>" + CONTENT_TEXT + "</p>";
+        final String article = paragraph + paragraph;
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/Article\">" +
+                paragraph +
+                "<div itemscope itemtype=\"http://schema.org/Article\">" + paragraph + "</div>" +
+            "</div>" +
+            "<div itemscope itemtype=\"http://schema.org/Article\" style=\"display:none\">" +
+                article + "</div>";
+
+        assertNotNull(getArticleElement(htmlArticle));
+    }
+
+    public void testOnlyProcessOGNonArticleMovie() {
+        final String article = "<p>" + CONTENT_TEXT + "</p><p>" + CONTENT_TEXT + "</p>";
+
+        final String htmlArticle =
+            "<h1>" + CONTENT_TEXT + "</h1>" +
+            "<div itemscope itemtype=\"http://schema.org/Movie\">" + article + "</div>";
+
+        // Non-article schema.org types should not use the fast path.
+        assertNull(getArticleElement(htmlArticle));
+    }
+
+    private Element getArticleElement(String html) {
+        mBody.setInnerHTML(html);
+        return DomUtil.getArticleElement(mRoot);
+    }
 }