Fix word count issue for Chinese and Japanese

javatests/org/chromium/distiller/StringUtilTest.java

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package org.chromium.distiller;
 
 import com.google.gwt.regexp.shared.RegExp;
 
 public class StringUtilTest extends JsTestCase {
     public void testCountWords() {
         assertEquals(0, StringUtil.countWords(""));
         assertEquals(0, StringUtil.countWords("  -@# ';]"));
         assertEquals(1, StringUtil.countWords("word"));
         assertEquals(1, StringUtil.countWords("b'fore"));
         assertEquals(1, StringUtil.countWords(" _word.under_score_ "));
         assertEquals(2, StringUtil.countWords(" \ttwo\nwords"));
         assertEquals(2, StringUtil.countWords(" \ttwo @^@^&(@#$([][;;\nwords"));
         assertEquals(5, StringUtil.countWords("dør når på svært dårlig"));
         assertEquals(5, StringUtil.countWords("svært få dør av blåbærsyltetøy"));
+
+        // One Chinese sentence, or a series of Japanese glyphs should not be treated
+        // as a single word.
+        assertTrue(StringUtil.countWords("一個中文句子不應該當成一個字") > 1);
+        assertTrue(StringUtil.countWords("ファイナルファンタジー") > 1);
+        // However, treating each Chinese/Japanese glyph as a word is also wrong.
+        assertTrue(StringUtil.countWords("一個中文句子不應該當成一個字") < 14);
+        assertTrue(StringUtil.countWords("ファイナルファンタジー") < 11);

[kuan, 2015/05/15 01:45:07]

this is a katakana sentence, how about also adding one for hiragana since u
mentioned that in the code? u only tested a single hiragana character @33.

what about kanji, and simplified chinese?

[wychen, 2015/05/18 18:49:20]

All added.

[kuan, 2015/05/21 17:46:15]

thanks!

+        // Even if they are separated by spaces.
+        assertTrue(StringUtil.countWords("一 個 中 文 句 子 不 應 該 當 成 一 個 字") < 14);
+        assertTrue(StringUtil.countWords("フ ァ イ ナ ル フ ァ ン タ ジ ー") < 11);
+
+        assertEquals(1, StringUtil.countWords("字"));
+        assertEquals(1, StringUtil.countWords("が"));
+
+        // Mixing ASCII words and Chinese/Japanese glyphs
+        assertEquals(2, StringUtil.countWords("word字"));
+        assertEquals(2, StringUtil.countWords("word 字"));
+
+        // Hangul uses space as word delimiter like English.
+        assertEquals(1, StringUtil.countWords("어"));
+        assertEquals(2, StringUtil.countWords("한국어 단어"));
+        assertEquals(5, StringUtil.countWords("한 국 어 단 어"));
+        assertEquals(8, StringUtil.countWords(
+                "예비군 훈련장 총기 난사범 최모씨의 군복에서 발견된 유서."));
     }
 
     public void testIsWhitespace() {
         assertTrue(StringUtil.isWhitespace(' '));
         assertTrue(StringUtil.isWhitespace('\t'));
         assertTrue(StringUtil.isWhitespace('\n'));
         assertTrue(StringUtil.isWhitespace('\u00a0'));
         assertFalse(StringUtil.isWhitespace('a'));
         assertFalse(StringUtil.isWhitespace('$'));
         assertFalse(StringUtil.isWhitespace('_'));
@@ skipping 77 matching lines @@
         assertEquals(-1, StringUtil.toNumber("'8_"));
         assertEquals(-1, StringUtil.toNumber(""));
         assertEquals(-1, StringUtil.toNumber(" "));
         assertEquals(-1, StringUtil.toNumber("\u00a0\u0460"));
         assertEquals(-1, StringUtil.toNumber("abc"));
         assertEquals(-1, StringUtil.toNumber("$"));
         assertEquals(-1, StringUtil.toNumber("_"));
     }
 
 }