Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(66)

Unified Diff: src/com/dom_distiller/client/SchemaOrgParser.java

Issue 240073007: recognize and parse Schema.org Markup (Closed) Base URL: https://code.google.com/p/dom-distiller/@master
Patch Set: fine-tune prev bug fix Created 6 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: src/com/dom_distiller/client/SchemaOrgParser.java
diff --git a/src/com/dom_distiller/client/SchemaOrgParser.java b/src/com/dom_distiller/client/SchemaOrgParser.java
new file mode 100644
index 0000000000000000000000000000000000000000..31b71c807413473574a790eb38f6577a34efdd3f
--- /dev/null
+++ b/src/com/dom_distiller/client/SchemaOrgParser.java
@@ -0,0 +1,557 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package com.dom_distiller.client;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.gwt.dom.client.AnchorElement;
+import com.google.gwt.dom.client.Element;
+import com.google.gwt.dom.client.ImageElement;
+import com.google.gwt.dom.client.MetaElement;
+import com.google.gwt.dom.client.Node;
+import com.google.gwt.dom.client.NodeList;
+
+/**
+ * This class recognizes and parses schema.org markup tags, and returns the properties that matter
+ * to distilled content.
+ * Schema.org markup (http://schema.org) is based on the microdata format
+ * (http://www.whatwg.org/specs/web-apps/current-work/multipage/microdata.html).
+ * For the basic Schema.org Thing type, the basic properties are: name, url, description, image.
+ * In addition, for each type that we support, we also parse more specific properties:
+ * - Article: headline (i.e. title), publisher, copyright year, copyright holder, date published,
+ * date modified, author, article section
+ * - ImageObject: headline (i.e. title), publisher, copyright year, copyright holder, content url,
+ * encoding format, caption, representative of page, width, height
+ * - Person: family name, given name
+ * - Organization: legal name.
+ * The value of a Schema.Org property can be a Schema.Org type, i.e. embedded. E.g., the author or
+ * publisher of article or publisher of image could be a Schema.Org Person or Organization type;
+ * in fact, this is the reason we support Person and Organization types.
+ */
+public class SchemaOrgParser implements MarkupParser.Parser {
cjhopman 2014/04/21 16:52:22 Can we split this class into two parts: 1. A Sche
kuan 2014/04/23 15:32:36 Done.
+ private static final String NAME_PROP = "name";
+ private static final String URL_PROP = "url";
+ private static final String DESCRIPTION_PROP = "description";
+ private static final String IMAGE_PROP = "image";
+ private static final String HEADLINE_PROP = "headline";
+ private static final String PUBLISHER_PROP = "publisher";
+ private static final String COPYRIGHT_HOLDER_PROP = "copyrightHolder";
+ private static final String COPYRIGHT_YEAR_PROP = "copyrightYear";
+ private static final String CONTENT_URL_PROP = "contentUrl";
+ private static final String ENCODING_FORMAT_PROP = "encodingFormat";
+ private static final String CAPTION_PROP = "caption";
+ private static final String REPRESENTATIVE_PROP = "representativeOfPage";
+ private static final String WIDTH_PROP = "width";
+ private static final String HEIGHT_PROP = "height";
+ private static final String DATE_PUBLISHED_PROP = "datePublished";
+ private static final String DATE_MODIFIED_PROP = "dateModified";
+ private static final String AUTHOR_PROP = "author";
+ private static final String SECTION_PROP = "articleSection";
+ private static final String FAMILY_NAME_PROP = "familyName";
+ private static final String GIVEN_NAME_PROP = "givenName";
+ private static final String LEGAL_NAME_PROP = "legalName";
+ private static final String AUTHOR_REL = "author";
+
+ private enum Type { // All these types are extended from Thing, directly or indirectly.
+ IMAGE,
+ ARTICLE,
+ PERSON,
+ ORGANIZATION,
+ UNSUPPORTED,
+ }
+
+ private static class ThingItem {
+ private final Type mType;
+ private final String[] mStringPropertyNames;
+ private final String[] mItemPropertyNames;
+ private final String[] mStringProperties;
+ private final ThingItem[] mItemProperties;
+
+ ThingItem(Type type, String[] stringPropertyNames, String[] itemPropertyNames) {
+ mType = type;
+ mStringPropertyNames = stringPropertyNames;
+ mItemPropertyNames = itemPropertyNames;
+ mStringProperties = new String[mStringPropertyNames.length];
+ mItemProperties = new ThingItem[mItemPropertyNames.length];
+ }
+
+ String toStringProperty() {
+ return "";
+ }
+
+ MarkupParser.Image getImage() {
+ // Use value of IMAGE_PROP to create a MarkupParser.Image.
+ String imageUrl = getStringProperty(IMAGE_PROP);
+ if (imageUrl.isEmpty()) return null;
+ MarkupParser.Image image = new MarkupParser.Image();
+ image.image = imageUrl;
+ image.url = imageUrl;
+ return image;
+ }
+
+ MarkupParser.Article getArticle() {
+ return null;
+ }
+
+ final boolean isSupported() { return mType != Type.UNSUPPORTED; }
+
+ final boolean isImageRepresentativeOfPage() {
+ String value = getStringProperty(REPRESENTATIVE_PROP);
+ return value.equalsIgnoreCase("true");
+ }
+
+ // Store |value| for property with |name|, unless the property already has a non-empty
+ // value, in which case, |value| will be ignored. This means we only keep the first value.
+ final void putStringValue(String name, String value) {
+ for (int i = 0; i < mStringPropertyNames.length; i++) {
+ if (name.equals(mStringPropertyNames[i])) {
+ String existing = mStringProperties[i];
+ if (existing == null || existing.isEmpty()) mStringProperties[i] = value;
+ break;
+ }
+ }
+ }
+
+ // Store |value| for property with |name|, unless the property already has a non-null value,
+ // in which case, |value| will be ignored. This means we only keep the first value.
+ final void putItemValue(String name, ThingItem value) {
+ for (int i = 0; i < mItemPropertyNames.length; i++) {
+ if (name.equals(mItemPropertyNames[i])) {
+ if (mItemProperties[i] == null) mItemProperties[i] = value;
+ break;
+ }
+ }
+ }
+
+ final String getStringProperty(String name) {
+ // Check if property exists in |mStringProperties|.
+ for (int i = 0; i < mStringPropertyNames.length; i++) {
+ if (name.equals(mStringPropertyNames[i])) {
+ String value = mStringProperties[i];
+ if (value != null && !value.isEmpty()) return value;
+ break;
+ }
+ }
+ // Otherwise, repeat for |mItemProperties|.
+ for (int i = 0; i < mItemPropertyNames.length; i++) {
+ if (!name.equals(mItemPropertyNames[i])) continue;
+ if (mItemProperties[i] != null) return mItemProperties[i].toStringProperty();
+ break;
+ }
+ return "";
+ }
+ }
+
+ private final List<ThingItem> mItemScopes = new ArrayList<ThingItem>();
+ private String mAuthorFromRel = "";
+ private static final Map<String, Type> sTypeUrls;
+ private static final Map<String, String[]> sTagAttributesMap;
+ private static final String[] sEmptyPropertyNames = {
+ // Intentionally empty, declared so that it's initialized statically.
+ };
+
+ static {
+ sTypeUrls = new HashMap<String, Type>();
+ sTypeUrls.put("http://schema.org/ImageObject", Type.IMAGE);
+ sTypeUrls.put("http://schema.org/Article", Type.ARTICLE);
cjhopman 2014/04/21 16:52:22 We should probably recognize schema.org/NewsArticl
kuan 2014/04/23 15:32:36 Done.
+ sTypeUrls.put("http://schema.org/Person", Type.PERSON);
+ sTypeUrls.put("http://schema.org/Organization", Type.ORGANIZATION);
cjhopman 2014/04/21 16:52:22 There are a whole bunch of subtypes of Organizatio
kuan 2014/04/23 15:32:36 i added more subtypes, but i'm not sure if they're
+
+ // The key for |sTagAttributesMap| is the tag name, while the entry value is an array of
+ // attributes in the specified tag from which to extract information:
+ // - 0th attribute: contains the value for the property specified in itemprop
+ // - 1st attribute: if available, contains the value for the author property.
+ sTagAttributesMap = new HashMap<String, String[]>();
+ sTagAttributesMap.put("IMG", new String[] { "SRC" });
+ sTagAttributesMap.put("AUDIO", new String[] { "SRC" });
+ sTagAttributesMap.put("EMBED", new String[] { "SRC" });
+ sTagAttributesMap.put("IFRAME", new String[] { "SRC" });
+ sTagAttributesMap.put("SOURCE", new String[] { "SRC" });
+ sTagAttributesMap.put("TRACK", new String[] { "SRC" });
+ sTagAttributesMap.put("VIDEO", new String[] { "SRC" });
+ sTagAttributesMap.put("A", new String[] { "HREF", "REL" });
+ sTagAttributesMap.put("LINK", new String[] { "HREF", "REL" });
+ sTagAttributesMap.put("AREA", new String[] { "HREF" });
+ sTagAttributesMap.put("META", new String[] { "CONTENT" });
+ sTagAttributesMap.put("TIME", new String[] { "DATETIME" });
+ sTagAttributesMap.put("OBJECT", new String[] { "DATA" });
+ sTagAttributesMap.put("DATA", new String[] { "VALUE" });
+ sTagAttributesMap.put("METER", new String[] { "VALUE" });
+ }
+
+ /**
+ * The object that extracts and verifies Schema.org markup tags from |root|.
+ */
+ public SchemaOrgParser(Element root) {
+ // TODO(kuan): Parsing all tags is pretty expensive, should we do so only lazily?
+ // If parse lazily, all get* methods will need to check for parsed state and, if necessary,
+ // parse before returning the requested properties.
+ // Note that the <html> element can also be the start of a Schema.org item, and hence needs
+ // to be parsed.
+ parse(root, null);
+ }
+
+ @Override
+ public String getTitle() {
+ String title = findStringProperty(HEADLINE_PROP);
+ if (title.isEmpty()) title = findStringProperty(NAME_PROP);
cjhopman 2014/04/21 16:52:22 This seems to mean that we will get the first item
kuan 2014/04/23 15:32:36 Done.
+ return title;
+ }
+
+ @Override
+ public String getType() {
+ if (mItemScopes.isEmpty()) return "";
+ // Assume the type of the first item is the page type.
+ return mItemScopes.get(0).mType.toString();
cjhopman 2014/04/21 16:52:22 Does schema.org specify that this is a good way to
kuan 2014/04/23 15:32:36 Done. for the record, as per our off-line discuss
+ }
+
+ @Override
+ public String getUrl() {
+ return findStringProperty(URL_PROP);
cjhopman 2014/04/21 16:52:22 I don't really understand what we expect the url f
kuan 2014/04/23 15:32:36 Done.
+ }
+
+ @Override
+ public MarkupParser.Image[] getImages() {
cjhopman 2014/04/21 16:52:22 We should be careful about what this returns, we o
kuan 2014/04/23 15:32:36 Done. per our discussion off-line, i won't impl t
+ if (mItemScopes.isEmpty()) return null;
+ List<MarkupParser.Image> images = new ArrayList<MarkupParser.Image>();
+ for (int i = 0; i < mItemScopes.size(); i++) {
+ ThingItem item = mItemScopes.get(i);
+ MarkupParser.Image image = item.getImage();
+ if (image != null) {
+ if (item.isImageRepresentativeOfPage()) {
+ // Image should be the dominant, i.e. first, one.
+ images.add(0, image);
+ } else {
+ images.add(image);
+ }
+ }
+ }
+ if (images.isEmpty()) return null;
+ return images.toArray(new MarkupParser.Image[images.size()]);
+ }
+
+ @Override
+ public String getDescription() {
+ return findStringProperty(DESCRIPTION_PROP);
cjhopman 2014/04/21 16:52:22 Again, this should probably only be the descriptio
kuan 2014/04/23 15:32:36 Done.
+ }
+
+ @Override
+ public String getPublisher() {
+ return findStringProperty(PUBLISHER_PROP);
cjhopman 2014/04/21 16:52:22 I think we would only want the publisher property
kuan 2014/04/23 15:32:36 Done.
+ }
+
+ @Override
+ public String getCopyright() {
+ if (mItemScopes.isEmpty()) return "";
+ // Returns a concatenated string of copyright year and copyright holder of the first item
+ // that has these properties, delimited by a whitespace.
+ String copyright = "";
+ for (int i = 0; i < mItemScopes.size() && copyright.isEmpty(); i++) {
+ ThingItem item = mItemScopes.get(i);
+ copyright = concat(item.getStringProperty(COPYRIGHT_YEAR_PROP),
+ item.getStringProperty(COPYRIGHT_HOLDER_PROP));
+ }
+ return copyright.isEmpty() ? copyright : "Copyright " + copyright;
+ }
+
+ @Override
+ public String getAuthor() {
+ String author = findStringProperty(AUTHOR_PROP);
+ return author.isEmpty() ? mAuthorFromRel : author;
+ }
+
+ @Override
+ public MarkupParser.Article getArticle() {
+ if (mItemScopes.isEmpty()) return null;
+ // Returns the first article.
+ MarkupParser.Article article = null;
+ for (int i = 0; i < mItemScopes.size() && article == null; i++) {
+ article = mItemScopes.get(i).getArticle();
+ }
+ return article;
+ }
+
+ @Override
+ public boolean optOut() {
+ return false;
+ }
+
+ private void parse(Element e, ThingItem parentItem) {
+ ThingItem newItem = null;
+ boolean isItemScope = isItemScope(e);
+ // A non-null |parentItem| means we're currently parsing the elements for a schema.org type.
+ String[] propertyNames = parentItem != null ? getItemProp(e) : new String[0];
+
+ if (isItemScope) {
+ // The "itemscope" and "itemtype" attributes of |e| indicate the start of an item.
+ // Create the corresponding extended-ThingItem, and add it to the list if:
+ // 1) its type is supported, and
+ // 2) if the parent is an unsupported type, it's not an "itemprop" attribute of the
+ // parent, based on the rule that an item is a top-level item if its element doesn't
+ // have an itemprop attribute.
+ newItem = createItemForElement(e);
+ if (newItem != null && newItem.isSupported() &&
+ (parentItem == null || parentItem.isSupported() || propertyNames.length == 0)) {
+ mItemScopes.add(newItem);
+ }
+ }
+
+ // If parent is a supported type, parse the element for >= 1 properties in "itemprop"
+ // attribute.
+ if (propertyNames.length > 0 && parentItem.isSupported() &&
+ (newItem == null || newItem.isSupported())) {
+ for (int i = 0; i < propertyNames.length; i++) {
+ // If a new item was created above, the property value of this "itemprop" attribute
+ // is an embedded item, so add it to the parent item.
+ if (newItem != null) {
+ parentItem.putItemValue(propertyNames[i], newItem);
+ } else {
+ // Otherwise, extract the property value from the tag itself, and add it to the
+ // parent item.
+ parentItem.putStringValue(propertyNames[i], getPropertyValue(e));
+ }
+ }
+ }
+
+ // If <a> or <link> tags specify rel="author", extract it.
cjhopman 2014/04/25 20:52:34 Where does this rel="author" stuff come from? I ca
+ if (mAuthorFromRel.isEmpty()) mAuthorFromRel = getAuthorFromRelAttribute(e);
+
+ // Now, parse each child element recursively.
+ NodeList<Node> children = e.getChildNodes();
+ for (int i = 0; i < children.getLength(); i++) {
+ Node child = children.getItem(i);
+ if (child.getNodeType() != Node.ELEMENT_NODE) continue;
+ parse(Element.as(child), newItem != null ? newItem : parentItem);
+ }
+ }
+
+ private Type getItemType(Element e) {
+ // "itemtype" attribute is case-sensitive.
+ String type = e.getAttribute("ITEMTYPE");
+ return sTypeUrls.containsKey(type) ? sTypeUrls.get(type) : Type.UNSUPPORTED;
+ }
+
+ private ThingItem createItemForElement(Element e) {
+ ThingItem newItem = null;
+ Type type = getItemType(e);
+ switch (type) {
+ case IMAGE:
+ newItem = new ImageItem();
+ break;
+ case ARTICLE:
+ newItem = new ArticleItem();
+ break;
+ case PERSON:
+ newItem = new PersonItem();
+ break;
+ case ORGANIZATION:
+ newItem = new OrganizationItem();
+ break;
+ case UNSUPPORTED:
+ newItem = new UnsupportedItem();
+ break;
+ default:
+ return null;
+ }
+ return newItem;
+ }
+
+ // Returns the first item that has the requested property value.
+ private String findStringProperty(String name) {
+ if (mItemScopes.isEmpty()) return "";
+ for (int i = 0; i < mItemScopes.size(); i++) {
+ String value = mItemScopes.get(i).getStringProperty(name);
+ if (!value.isEmpty()) return value;
+ }
+ return "";
+ }
+
+ private static class ImageItem extends ThingItem {
+ private static final String[] sStringPropertyNames = {
+ NAME_PROP,
+ URL_PROP,
+ DESCRIPTION_PROP,
+ IMAGE_PROP,
+ HEADLINE_PROP,
+ PUBLISHER_PROP,
+ COPYRIGHT_HOLDER_PROP,
+ COPYRIGHT_YEAR_PROP,
+ CONTENT_URL_PROP,
+ ENCODING_FORMAT_PROP,
+ CAPTION_PROP,
+ REPRESENTATIVE_PROP,
+ WIDTH_PROP,
+ HEIGHT_PROP,
+ };
+
+ private static final String[] sItemPropertyNames = {
+ PUBLISHER_PROP,
+ COPYRIGHT_HOLDER_PROP,
+ };
+
+ ImageItem() {
+ super(Type.IMAGE, sStringPropertyNames, sItemPropertyNames);
+ }
+
+ @Override
+ MarkupParser.Image getImage() {
+ MarkupParser.Image image = new MarkupParser.Image();
+ String url = getStringProperty(CONTENT_URL_PROP);
+ image.image = !url.isEmpty() ? url : getStringProperty(NAME_PROP);
+ image.url = image.image;
+ image.type = getStringProperty(ENCODING_FORMAT_PROP);
+ image.caption = getStringProperty(CAPTION_PROP);
+ try {
+ image.width = Integer.parseInt(getStringProperty(WIDTH_PROP), 10);
+ } catch (Exception e) {
+ }
+ try {
+ image.height = Integer.parseInt(getStringProperty(HEIGHT_PROP), 10);
+ } catch (Exception e) {
+ }
+ return image;
+ }
+ }
+
+ private static class ArticleItem extends ThingItem {
+ private static final String[] sStringPropertyNames = {
+ NAME_PROP,
+ URL_PROP,
+ DESCRIPTION_PROP,
+ IMAGE_PROP,
+ HEADLINE_PROP,
+ PUBLISHER_PROP,
+ COPYRIGHT_HOLDER_PROP,
+ COPYRIGHT_YEAR_PROP,
+ DATE_MODIFIED_PROP,
+ DATE_PUBLISHED_PROP,
+ AUTHOR_PROP,
+ SECTION_PROP,
+ };
+
+ private static final String[] sItemPropertyNames = {
+ PUBLISHER_PROP,
+ COPYRIGHT_HOLDER_PROP,
+ AUTHOR_PROP,
+ };
+
+ ArticleItem() {
+ super(Type.ARTICLE, sStringPropertyNames, sItemPropertyNames);
+ }
+
+ @Override
+ MarkupParser.Article getArticle() {
+ MarkupParser.Article article = new MarkupParser.Article();
+ article.publishedTime = getStringProperty(DATE_PUBLISHED_PROP);
+ article.modifiedTime = getStringProperty(DATE_MODIFIED_PROP);
+ article.section = getStringProperty(SECTION_PROP);
+ String author = getStringProperty(AUTHOR_PROP);
+ article.authors = author.isEmpty() ? new String[0] : new String[] { author };
+ return article;
+ }
+ }
+
+ private static class PersonItem extends ThingItem {
+ private static final String[] sStringPropertyNames = {
+ NAME_PROP,
+ URL_PROP,
+ DESCRIPTION_PROP,
+ IMAGE_PROP,
+ FAMILY_NAME_PROP,
+ GIVEN_NAME_PROP,
+ };
+
+ PersonItem() {
+ super(Type.PERSON, sStringPropertyNames, sEmptyPropertyNames);
+ }
+
+ // Returns either the value of NAME_PROP, or concatenated values of GIVEN_NAME_PROP and
+ // FAILY_NAME_PROP delimited by a whitespace.
cjhopman 2014/04/21 16:52:22 s/FAILY/FAMILY
kuan 2014/04/23 15:32:36 Done.
+ @Override
+ String toStringProperty() {
+ String fullname = getStringProperty(NAME_PROP);
+ if (fullname.isEmpty()) {
+ fullname = concat(getStringProperty(GIVEN_NAME_PROP),
+ getStringProperty(FAMILY_NAME_PROP));
+ }
+ return fullname;
+ }
+ }
+
+ private static class OrganizationItem extends ThingItem {
+ private static final String[] sStringPropertyNames = {
+ NAME_PROP,
+ URL_PROP,
+ DESCRIPTION_PROP,
+ IMAGE_PROP,
+ LEGAL_NAME_PROP,
+ };
+
+ OrganizationItem() {
+ super(Type.ORGANIZATION, sStringPropertyNames, sEmptyPropertyNames);
+ }
+
+ // Returns either the value of NAME_PROP or LEGAL_NAME_PROP.
+ @Override
+ String toStringProperty() {
+ String name = getStringProperty(NAME_PROP);
+ if (name.isEmpty()) name = getStringProperty(LEGAL_NAME_PROP);
+ return name;
+ }
+ }
+
+ private static class UnsupportedItem extends ThingItem {
+ UnsupportedItem(){
+ super(Type.UNSUPPORTED, sEmptyPropertyNames, sEmptyPropertyNames);
+ }
+ }
+
+ private static boolean isItemScope(Element e) {
+ return e.hasAttribute("ITEMSCOPE") && e.hasAttribute("ITEMTYPE");
+ }
+
+ private static String[] getItemProp(Element e) {
+ // "itemprop" attribute is case-sensitive, and can have multiple properties.
+ String itemprop = e.getAttribute("ITEMPROP");
+ if (itemprop.isEmpty()) return new String[0];
+ String[] splits = StringUtil.split(itemprop, "\\s+");
+ return splits.length > 0 ? splits : new String[] { itemprop };
+ }
+
+ // Extracts the property value from |e|. For some tags, the value is a specific attribute,
+ // while for others, it's the text between the start and end tags.
+ private static String getPropertyValue(Element e) {
+ String value = "";
+ String tagName = e.getTagName();
+ if (sTagAttributesMap.containsKey(tagName)) {
+ value = e.getAttribute(sTagAttributesMap.get(tagName)[0]);
+ }
+ if (value.isEmpty()) value = e.getInnerText();
+ return value;
+ }
+
+ // Extracts the author property from |e|'s "rel=author" attribute.
+ private static String getAuthorFromRelAttribute(Element e) {
+ String author = "";
+ String tagName = e.getTagName();
+ if (sTagAttributesMap.containsKey(tagName)) {
+ String[] attrs = sTagAttributesMap.get(tagName);
+ if (attrs.length > 1 && e.getAttribute(attrs[1]).equals(AUTHOR_REL)) {
+ author = e.getInnerText();
+ }
+ }
+ return author;
+ }
+
+ private static String concat(String first, String second) {
+ String concat = first;
+ if (!concat.isEmpty() && !second.isEmpty()) concat += " ";
+ concat += second;
+ return concat;
+ }
+}
« no previous file with comments | « src/com/dom_distiller/client/MarkupParser.java ('k') | test/com/dom_distiller/client/SchemaOrgParserTest.java » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698