Creates BreakIterator::GetWordBreakStatus.

base/i18n/break_iterator_unittest.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/i18n/break_iterator.h"
 
 #include "base/strings/string_piece.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ skipping 351 matching lines @@
   EXPECT_TRUE(iter.Advance());
   EXPECT_EQ(iter.GetString(), iter.GetStringPiece().as_string());
   EXPECT_EQ(StringPiece16(ASCIIToUTF16("some")), iter.GetStringPiece());
 
   EXPECT_TRUE(iter.Advance());
   EXPECT_TRUE(iter.Advance());
   EXPECT_EQ(iter.GetString(), iter.GetStringPiece().as_string());
   EXPECT_EQ(StringPiece16(ASCIIToUTF16("string")), iter.GetStringPiece());
 }
 
+// Make sure that when not in RULE_BASED or BREAK_WORD mode we're getting
+// IS_LINE_OR_CHAR_BREAK.
+TEST(BreakIteratorTest, GetWordBreakStatusBreakLine) {
+  // The string "foo ទេ \nCan Ми..." which contains English, Khmer, and Russian

[jungshik at Google, 2015/08/11 21:43:50]

Due to an issue with Visual Studio, we cannot use non-ASCII characters even in
comments. Visual Studio would barf in East Asian locales (CJK) when it's asked
to compile this source code. That really sucks, but ...Perhaps, we should
disable that warning (that would be treated as an error). 

[Julius, 2015/08/12 01:22:21]

I got rid of the non-ASCII characters in the comments and tried to make them
clear as to what's in the string.

+  // characters, in that order.
+  base::string16 text(
+      base::WideToUTF16(L"foo \x1791\x17c1 \nCan \x041C\x0438..."));
+  BreakIterator iter(text, BreakIterator::BREAK_LINE);
+  ASSERT_TRUE(iter.Init());
+
+  EXPECT_TRUE(iter.Advance());
+  // Finds "foo" and the space.
+  EXPECT_EQ(base::UTF8ToUTF16("foo "), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_LINE_OR_CHAR_BREAK);
+  EXPECT_TRUE(iter.Advance());
+  // Finds the Khmer characters, the next space, and the newline.
+  EXPECT_EQ(base::WideToUTF16(L"\x1791\x17c1 \n"), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_LINE_OR_CHAR_BREAK);
+  EXPECT_TRUE(iter.Advance());
+  // Finds "Can" and the space.
+  EXPECT_EQ(base::UTF8ToUTF16("Can "), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_LINE_OR_CHAR_BREAK);
+  EXPECT_TRUE(iter.Advance());
+  // Finds the Russian characters and periods.
+  EXPECT_EQ(base::WideToUTF16(L"\x041C\x0438..."), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_LINE_OR_CHAR_BREAK);
+  EXPECT_FALSE(iter.Advance());
+}
+
+// Make sure that in BREAK_WORD mode we're getting IS_WORD_BREAK and
+// IS_SKIPPABLE_WORD when we should be. IS_WORD_BREAK should be returned when we
+// finish going over non-punctuation characters while IS_SKIPPABLE_WORD should
+// be returned on punctuation and spaces.
+TEST(BreakIteratorTest, GetWordBreakStatusBreakWord) {
+  // The string "foo ទេ \nCan Ми..." which contains English, Khmer, and Russian
+  // characters, in that order.
+  base::string16 text(
+      base::WideToUTF16(L"foo \x1791\x17c1 \nCan \x041C\x0438..."));
+  BreakIterator iter(text, BreakIterator::BREAK_WORD);
+  ASSERT_TRUE(iter.Init());
+
+  EXPECT_TRUE(iter.Advance());
+  // Finds "foo".
+  EXPECT_EQ(base::UTF8ToUTF16("foo"), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
+  EXPECT_TRUE(iter.Advance());
+  // Finds the space, and the Khmer characters.
+  EXPECT_EQ(base::UTF8ToUTF16(" "), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
+  EXPECT_TRUE(iter.Advance());
+  EXPECT_EQ(base::WideToUTF16(L"\x1791\x17c1"), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
+  EXPECT_TRUE(iter.Advance());
+  // Finds the space and the newline.
+  EXPECT_EQ(base::UTF8ToUTF16(" "), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
+  EXPECT_TRUE(iter.Advance());
+  EXPECT_EQ(base::UTF8ToUTF16("\n"), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
+  EXPECT_TRUE(iter.Advance());
+  // Finds "Can".
+  EXPECT_EQ(base::UTF8ToUTF16("Can"), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
+  EXPECT_TRUE(iter.Advance());
+  // Finds the space and the Russian characters.
+  EXPECT_EQ(base::UTF8ToUTF16(" "), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
+  EXPECT_TRUE(iter.Advance());
+  EXPECT_EQ(base::WideToUTF16(L"\x041C\x0438"), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
+  EXPECT_TRUE(iter.Advance());
+  // Finds the trailing periods.
+  EXPECT_EQ(base::UTF8ToUTF16("."), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
+  EXPECT_TRUE(iter.Advance());
+  EXPECT_EQ(base::UTF8ToUTF16("."), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
+  EXPECT_TRUE(iter.Advance());
+  EXPECT_EQ(base::UTF8ToUTF16("."), iter.GetString());
+  EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
+  EXPECT_FALSE(iter.Advance());
+}
+
 }  // namespace i18n
 }  // namespace base