Import SQLite 3.8.7.4.

third_party/sqlite/src/test/fts3atoken.test

 # 2007 June 21
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
 #
 #    May you do good and not evil.
 #    May you find forgiveness for yourself and forgive others.
 #    May you share freely, never taking more than you give.
 #
 #*************************************************************************
 # This file implements regression tests for SQLite library. The focus 
 # of this script is testing the pluggable tokeniser feature of the 
 # FTS3 module.
 #
 # $Id: fts3atoken.test,v 1.1 2007/08/20 17:38:42 shess Exp $
 #
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
 # If SQLITE_ENABLE_FTS3 is defined, omit this file.
 ifcapable !fts3 {
   finish_test
   return
 }
 
+set ::testprefix fts3token
+
 proc escape_string {str} {
   set out ""
   foreach char [split $str ""] {
     scan $char %c i
     if {$i<=127} {
       append out $char
     } else {
       append out [format {\x%.4x} $i]
     }
   }
@@ skipping 121 matching lines @@
   set    input "short tokens then "
   append input $longtoken
   set    output "0 short short "
   append output "1 tokens tokens "
   append output "2 then then "
   append output "3 [string tolower $longtoken] $longtoken"
 
   do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
   do_icu_test fts3token-4.7 th_TH  $input $output
   do_icu_test fts3token-4.8 en_US  $input $output
+
+  do_execsql_test 5.1 {
+    CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
+    insert into x1 (name) values (NULL);
+    insert into x1 (name) values (NULL);
+    delete from x1;
+  }
+
+  proc cp_to_str {codepoint_list} {
+    set fmt [string repeat %c [llength $codepoint_list]]
+    eval [list format $fmt] $codepoint_list
+  }
+
+  do_test 5.2 {
+    set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
+    execsql { INSERT INTO x1 VALUES($str) }
+  } {}
 }
 
+
 do_test fts3token-internal {
   execsql { SELECT fts3_tokenizer_internal_test() }
 } {ok}
 
+
 finish_test