Update ICU to 54.1 step 1

source/test/intltest/rbbiapts.cpp

 /********************************************************************
- * Copyright (c) 1999-2013, International Business Machines
+ * Copyright (c) 1999-2014, International Business Machines
  * Corporation and others. All Rights Reserved.
  ********************************************************************
  *   Date        Name        Description
  *   12/14/99    Madhu        Creation.
  *   01/12/2000  Madhu        updated for changed API
  ********************************************************************/
 
 #include "unicode/utypes.h"
 
 #if !UCONFIG_NO_BREAK_ITERATION
 
 #include "unicode/uchar.h"
 #include "intltest.h"
 #include "unicode/rbbi.h"
 #include "unicode/schriter.h"
 #include "rbbiapts.h"
 #include "rbbidata.h"
 #include "cstring.h"
 #include "ubrkimpl.h"
 #include "unicode/locid.h"
 #include "unicode/ustring.h"
 #include "unicode/utext.h"
 #include "cmemory.h"
-
+#if !UCONFIG_NO_BREAK_ITERATION && U_HAVE_STD_STRING
+#include "unicode/filteredbrk.h"
+#include <stdio.h> // for sprintf
+#endif
 /**
  * API Test the RuleBasedBreakIterator class
  */
 
 
 #define TEST_ASSERT_SUCCESS(status) {if (U_FAILURE(status)) {\
 dataerrln("Failure at file %s, line %d, error = %s", __FILE__, __LINE__, u_errorName(status));}}
 
 #define TEST_ASSERT(expr) {if ((expr) == FALSE) { \
     errln("Test Failure at file %s, line %d: \"%s\" is false.\n", __FILE__, __LINE__, #expr);};}
@@ skipping 599 matching lines @@
 
 //
 //  TestRuleStatus
 //      Test word break rule status constants.
 //
 void RBBIAPITest::TestRuleStatus() {
      UChar str[30];
      //no longer test Han or hiragana breaking here: ruleStatusVec would return nothing
      // changed UBRK_WORD_KANA to UBRK_WORD_IDEO
      u_unescape("plain word 123.45 \\u30a1\\u30a2 ",
-              // 012345678901234567  8      9    0      
-              //                     Katakana      
+              // 012345678901234567  8      9    0
+              //                     Katakana
                 str, 30);
      UnicodeString testString1(str);
      int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 20, 21};
      int32_t tag_lo[]  = {UBRK_WORD_NONE,     UBRK_WORD_LETTER, UBRK_WORD_NONE,    UBRK_WORD_LETTER,
                           UBRK_WORD_NONE,     UBRK_WORD_NUMBER, UBRK_WORD_NONE,
                           UBRK_WORD_IDEO,     UBRK_WORD_NONE};
 
      int32_t tag_hi[]  = {UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT,
                           UBRK_WORD_NONE_LIMIT, UBRK_WORD_NUMBER_LIMIT, UBRK_WORD_NONE_LIMIT,
                           UBRK_WORD_IDEO_LIMIT, UBRK_WORD_NONE_LIMIT};
@@ skipping 213 matching lines @@
 
 
 void RBBIAPITest::TestRegistration() {
 #if !UCONFIG_NO_SERVICE
     UErrorCode status = U_ZERO_ERROR;
     BreakIterator* ja_word = BreakIterator::createWordInstance("ja_JP", status);
     // ok to not delete these if we exit because of error?
     BreakIterator* ja_char = BreakIterator::createCharacterInstance("ja_JP", status);
     BreakIterator* root_word = BreakIterator::createWordInstance("", status);
     BreakIterator* root_char = BreakIterator::createCharacterInstance("", status);
-    
+
     if (status == U_MISSING_RESOURCE_ERROR || status == U_FILE_ACCESS_ERROR) {
         dataerrln("Error creating instances of break interactors - %s", u_errorName(status));
 
         delete ja_word;
         delete ja_char;
         delete root_word;
         delete root_char;
-        
+
         return;
     }
 
     URegistryKey key = BreakIterator::registerInstance(ja_word, "xx", UBRK_WORD, status);
     {
 #if 0 // With a dictionary based word breaking, ja_word is identical to root.
         if (ja_word && *ja_word == *root_word) {
             errln("japan not different from root");
         }
 #endif
@@ skipping 150 matching lines @@
     RoundtripRule("sent");
     RoundtripRule("line");
     RoundtripRule("char");
     if (!quick) {
         RoundtripRule("word_POSIX");
     }
 }
 
 // Try out the RuleBasedBreakIterator constructors that take RBBIDataHeader*
 // (these are protected so we access them via a local class RBBIWithProtectedFunctions).
-// This is just a sanity check, not a thorough test (e.g. we don't check that the 
+// This is just a sanity check, not a thorough test (e.g. we don't check that the
 // first delete actually frees rulesCopy).
 void RBBIAPITest::TestCreateFromRBBIData() {
     // Get some handy RBBIData
     const char *brkName = "word"; // or "sent", "line", "char", etc.
     UErrorCode status = U_ZERO_ERROR;
     LocalUDataMemoryPointer data(udata_open(U_ICUDATA_BRKITR, "brk", brkName, &status));
     if ( U_SUCCESS(status) ) {
         const RBBIDataHeader * builtRules = (const RBBIDataHeader *)udata_getMemory(data.getAlias());
         uint32_t length = builtRules->fLength;
         RBBIWithProtectedFunctions * brkItr;
 
         // Try the memory-adopting constructor, need to copy the data first
         RBBIDataHeader * rulesCopy = (RBBIDataHeader *) uprv_malloc(length);
         if ( rulesCopy ) {
             uprv_memcpy( rulesCopy, builtRules, length );
 
             brkItr = new RBBIWithProtectedFunctions(rulesCopy, status);
             if ( U_SUCCESS(status) ) {
                 delete brkItr; // this should free rulesCopy
             } else {
                 errln("create RuleBasedBreakIterator from RBBIData (adopted): ICU Error \"%s\"\n", u_errorName(status) );
                 status = U_ZERO_ERROR;// reset for the next test
                 uprv_free( rulesCopy );
             }
         }
-        
+
         // Now try the non-adopting constructor
         brkItr = new RBBIWithProtectedFunctions(builtRules, RBBIWithProtectedFunctions::kDontAdopt, status);
         if ( U_SUCCESS(status) ) {
             delete brkItr; // this should NOT attempt to free builtRules
             if (builtRules->fLength != length) { // sanity check
                 errln("create RuleBasedBreakIterator from RBBIData (non-adopted): delete affects data\n" );
             }
         } else {
             errln("create RuleBasedBreakIterator from RBBIData (non-adopted): ICU Error \"%s\"\n", u_errorName(status) );
         }
@@ skipping 64 matching lines @@
         TEST_ASSERT_SUCCESS(status);
         RuleBasedBreakIterator *returnedBI = &bi->refreshInputText(&ut2, status);
         TEST_ASSERT_SUCCESS(status);
         TEST_ASSERT(bi == returnedBI);
 
         /* Find the following matches, now working in the moved string. */
         TEST_ASSERT(5 == bi->next());
         TEST_ASSERT(7 == bi->next());
         TEST_ASSERT(8 == bi->next());
         TEST_ASSERT(UBRK_DONE == bi->next());
-    
+
         utext_close(&ut1);
         utext_close(&ut2);
     }
     delete bi;
 
 }
 
+#if !UCONFIG_NO_BREAK_ITERATION && U_HAVE_STD_STRING && !UCONFIG_NO_FILTERED_BREAK_ITERATION
+static void prtbrks(BreakIterator* brk, const UnicodeString &ustr, IntlTest &it) {
+  static const UChar PILCROW=0x00B6, CHSTR=0x3010, CHEND=0x3011; // lenticular brackets
+  it.logln(UnicodeString("String:'")+ustr+UnicodeString("'"));
+
+  int32_t *pos = new int32_t[ustr.length()];
+  int32_t posCount = 0;
+
+  // calculate breaks up front, so we can print out
+  // sans any debugging
+  for(int32_t n = 0; (n=brk->next())!=UBRK_DONE; ) {
+    pos[posCount++] = n;
+    if(posCount>=ustr.length()) {
+      it.errln("brk count exceeds string length!");
+      return;
+    }
+  }
+  UnicodeString out;
+  out.append((UChar)CHSTR);
+  int32_t prev = 0;
+  for(int32_t i=0;i<posCount;i++) {
+    int32_t n=pos[i];
+    out.append(ustr.tempSubString(prev,n-prev));
+    out.append((UChar)PILCROW);
+    prev=n;
+  }
+  out.append(ustr.tempSubString(prev,ustr.length()-prev));
+  out.append((UChar)CHEND);
+  it.logln(out);
+
+  out.remove();
+  for(int32_t i=0;i<posCount;i++) {
+    char tmp[100];
+    sprintf(tmp,"%d ",pos[i]);
+    out.append(UnicodeString(tmp));
+  }
+  it.logln(out);
+  delete [] pos;
+}
+#endif
+
+void RBBIAPITest::TestFilteredBreakIteratorBuilder() {
+#if !UCONFIG_NO_BREAK_ITERATION && U_HAVE_STD_STRING && !UCONFIG_NO_FILTERED_BREAK_ITERATION
+  UErrorCode status = U_ZERO_ERROR;
+  LocalPointer<FilteredBreakIteratorBuilder> builder;
+  LocalPointer<BreakIterator> baseBI;
+  LocalPointer<BreakIterator> filteredBI;
+
+  const UnicodeString text("In the meantime Mr. Weston arrived with his small ship, which he had now recovered. Capt. Gorges, who informed the Sgt. here that one purpose of his going east was to meet with Mr. Weston, took this opportunity to call him to account for some abuses he had to lay to his charge."); // (William Bradford, public domain. http://catalog.hathitrust.org/Record/008651224 ) - edited.
+  const UnicodeString ABBR_MR("Mr.");
+  const UnicodeString ABBR_CAPT("Capt.");
+
+  {
+    logln("Constructing empty builder\n");
+    builder.adoptInstead(FilteredBreakIteratorBuilder::createInstance(status));
+    TEST_ASSERT_SUCCESS(status);
+
+    logln("Constructing base BI\n");
+    baseBI.adoptInstead(BreakIterator::createSentenceInstance(Locale::getEnglish(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    logln("Building new BI\n");
+    filteredBI.adoptInstead(builder->build(baseBI.orphan(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    if (U_SUCCESS(status)) {
+        logln("Testing:");
+        filteredBI->setText(text);
+        TEST_ASSERT(20 == filteredBI->next()); // Mr.
+        TEST_ASSERT(84 == filteredBI->next()); // recovered.
+        TEST_ASSERT(90 == filteredBI->next()); // Capt.
+        TEST_ASSERT(181 == filteredBI->next()); // Mr.
+        TEST_ASSERT(278 == filteredBI->next()); // charge.
+        filteredBI->first();
+        prtbrks(filteredBI.getAlias(), text, *this);
+    }
+  }
+
+  {
+    logln("Constructing empty builder\n");
+    builder.adoptInstead(FilteredBreakIteratorBuilder::createInstance(status));
+    TEST_ASSERT_SUCCESS(status);
+
+    if (U_SUCCESS(status)) {
+        logln("Adding Mr. as an exception\n");
+        TEST_ASSERT(TRUE == builder->suppressBreakAfter(ABBR_MR, status));
+        TEST_ASSERT(FALSE == builder->suppressBreakAfter(ABBR_MR, status)); // already have it
+        TEST_ASSERT(TRUE == builder->unsuppressBreakAfter(ABBR_MR, status));
+        TEST_ASSERT(FALSE == builder->unsuppressBreakAfter(ABBR_MR, status)); // already removed it
+        TEST_ASSERT(TRUE == builder->suppressBreakAfter(ABBR_MR, status));
+        TEST_ASSERT_SUCCESS(status);
+
+        logln("Constructing base BI\n");
+        baseBI.adoptInstead(BreakIterator::createSentenceInstance(Locale::getEnglish(), status));
+        TEST_ASSERT_SUCCESS(status);
+
+        logln("Building new BI\n");
+        filteredBI.adoptInstead(builder->build(baseBI.orphan(), status));
+        TEST_ASSERT_SUCCESS(status);
+
+        logln("Testing:");
+        filteredBI->setText(text);
+        TEST_ASSERT(84 == filteredBI->next());
+        TEST_ASSERT(90 == filteredBI->next());// Capt.
+        TEST_ASSERT(278 == filteredBI->next());
+        filteredBI->first();
+        prtbrks(filteredBI.getAlias(), text, *this);
+    }
+  }
+
+
+  {
+    logln("Constructing empty builder\n");
+    builder.adoptInstead(FilteredBreakIteratorBuilder::createInstance(status));
+    TEST_ASSERT_SUCCESS(status);
+
+    if (U_SUCCESS(status)) {
+        logln("Adding Mr. and Capt as an exception\n");
+        TEST_ASSERT(TRUE == builder->suppressBreakAfter(ABBR_MR, status));
+        TEST_ASSERT(TRUE == builder->suppressBreakAfter(ABBR_CAPT, status));
+        TEST_ASSERT_SUCCESS(status);
+
+        logln("Constructing base BI\n");
+        baseBI.adoptInstead(BreakIterator::createSentenceInstance(Locale::getEnglish(), status));
+        TEST_ASSERT_SUCCESS(status);
+
+        logln("Building new BI\n");
+        filteredBI.adoptInstead(builder->build(baseBI.orphan(), status));
+        TEST_ASSERT_SUCCESS(status);
+
+        logln("Testing:");
+        filteredBI->setText(text);
+        TEST_ASSERT(84 == filteredBI->next());
+        TEST_ASSERT(278 == filteredBI->next());
+        filteredBI->first();
+        prtbrks(filteredBI.getAlias(), text, *this);
+    }
+  }
+
+
+  {
+    logln("Constructing English builder\n");
+    builder.adoptInstead(FilteredBreakIteratorBuilder::createInstance(Locale::getEnglish(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    logln("Constructing base BI\n");
+    baseBI.adoptInstead(BreakIterator::createSentenceInstance(Locale::getEnglish(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    if (U_SUCCESS(status)) {
+        logln("unsuppressing 'Capt'");
+        TEST_ASSERT(TRUE == builder->unsuppressBreakAfter(ABBR_CAPT, status));
+
+        logln("Building new BI\n");
+        filteredBI.adoptInstead(builder->build(baseBI.orphan(), status));
+        TEST_ASSERT_SUCCESS(status);
+
+        if(filteredBI.isValid()) {
+          logln("Testing:");
+          filteredBI->setText(text);
+          TEST_ASSERT(84 == filteredBI->next());
+          TEST_ASSERT(90 == filteredBI->next());
+          TEST_ASSERT(278 == filteredBI->next());
+          filteredBI->first();
+          prtbrks(filteredBI.getAlias(), text, *this);
+        }
+    }
+  }
+
+
+  {
+    logln("Constructing English builder\n");
+    builder.adoptInstead(FilteredBreakIteratorBuilder::createInstance(Locale::getEnglish(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    logln("Constructing base BI\n");
+    baseBI.adoptInstead(BreakIterator::createSentenceInstance(Locale::getEnglish(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    if (U_SUCCESS(status)) {
+        logln("Building new BI\n");
+        filteredBI.adoptInstead(builder->build(baseBI.orphan(), status));
+        TEST_ASSERT_SUCCESS(status);
+
+        if(filteredBI.isValid()) {
+          logln("Testing:");
+          filteredBI->setText(text);
+          TEST_ASSERT(84 == filteredBI->next());
+          TEST_ASSERT(278 == filteredBI->next());
+          filteredBI->first();
+          prtbrks(filteredBI.getAlias(), text, *this);
+        }
+    }
+  }
+
+#if 0
+  // reenable once french is in
+  {
+    logln("Constructing French builder");
+    builder.adoptInstead(FilteredBreakIteratorBuilder::createInstance(Locale::getFrench(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    logln("Constructing base BI\n");
+    baseBI.adoptInstead(BreakIterator::createSentenceInstance(Locale::getFrench(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    logln("Building new BI\n");
+    filteredBI.adoptInstead(builder->build(baseBI.orphan(), status));
+    TEST_ASSERT_SUCCESS(status);
+
+    if(filteredBI.isValid()) {
+      logln("Testing:");
+      filteredBI->setText(text);
+      TEST_ASSERT(20 == filteredBI->next());
+      TEST_ASSERT(84 == filteredBI->next());
+      filteredBI->first();
+      prtbrks(filteredBI.getAlias(), text, *this);
+    }
+  }
+#endif
+
+#else
+  logln("Skipped- not: !UCONFIG_NO_BREAK_ITERATION && U_HAVE_STD_STRING && !UCONFIG_NO_FILTERED_BREAK_ITERATION");
+#endif
+}
 
 //---------------------------------------------
 // runIndexedTest
 //---------------------------------------------
 
 void RBBIAPITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
 {
     if (exec) logln((UnicodeString)"TestSuite RuleBasedBreakIterator API ");
     switch (index) {
      //   case 0: name = "TestConstruction"; if (exec) TestConstruction(); break;
@@ skipping 14 matching lines @@
         case  9: name = "TestRegistration"; if (exec) TestRegistration(); break;
         case 10: name = "TestBoilerPlate"; if (exec) TestBoilerPlate(); break;
         case 11: name = "TestRuleStatus"; if (exec) TestRuleStatus(); break;
         case 12: name = "TestRoundtripRules"; if (exec) TestRoundtripRules(); break;
         case 13: name = "TestCreateFromRBBIData"; if (exec) TestCreateFromRBBIData(); break;
 #else
         case  9: case 10: case 11: case 12: case 13: name = "skip"; break;
 #endif
         case 14: name = "TestRefreshInputText"; if (exec) TestRefreshInputText(); break;
 
+#if !UCONFIG_NO_BREAK_ITERATION && U_HAVE_STD_STRING
+    case 15: name = "TestFilteredBreakIteratorBuilder"; if(exec) TestFilteredBreakIteratorBuilder(); break;
+#else
+    case 15: name="skip"; break;
+#endif
         default: name = ""; break; // needed to end loop
     }
 }
 
 //---------------------------------------------
 //Internal subroutines
 //---------------------------------------------
 
 void RBBIAPITest::doBoundaryTest(BreakIterator& bi, UnicodeString& text, int32_t *boundaries){
      logln((UnicodeString)"testIsBoundary():");
@@ skipping 40 matching lines @@
     : RuleBasedBreakIterator(data, status)
 {
 }
 
 RBBIWithProtectedFunctions::RBBIWithProtectedFunctions(const RBBIDataHeader* data, enum EDontAdopt, UErrorCode &status)
     : RuleBasedBreakIterator(data, RuleBasedBreakIterator::kDontAdopt, status)
 {
 }
 
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */