[sql] Remove SQLite 3.10.2 reference directory.

third_party/sqlite/sqlite-src-3100200/test/fts4unicode.test

-# 2012 May 25
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-#
-# The tests in this file focus on testing the "unicode" FTS tokenizer.
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-ifcapable !fts3_unicode { finish_test ; return }
-set ::testprefix fts4unicode
-
-proc do_unicode_token_test {tn input res} {
-  set input [string map {' ''} $input]
-  uplevel [list do_execsql_test $tn "
-    SELECT fts3_tokenizer_test('unicode61', 'remove_diacritics=0', '$input');
-  " [list [list {*}$res]]]
-}
-
-proc do_unicode_token_test2 {tn input res} {
-  set input [string map {' ''} $input]
-  uplevel [list do_execsql_test $tn "
-    SELECT fts3_tokenizer_test('unicode61', '$input');
-  " [list [list {*}$res]]]
-}
-
-proc do_unicode_token_test3 {tn args} {
-  set res   [lindex $args end]
-  set sql "SELECT fts3_tokenizer_test('unicode61'"
-  foreach a [lrange $args 0 end-1] {
-    append sql ", '"
-    append sql [string map {' ''} $a]
-    append sql "'"
-  }
-  append sql ")"
-  uplevel [list do_execsql_test $tn $sql [list [list {*}$res]]]
-}
-
-do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D}
-
-do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \
-    "0 \uE4 \uC4 1 \uF6 \uD6 2 \uFC \uDC"
-
-do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \
-    "0 x\uE4x x\uC4x 1 x\uF6x x\uD6x 2 x\uFCx x\uDCx"
-
-# 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s.
-do_unicode_token_test 1.3 "\uDF" "0 \uDF \uDF"
-do_unicode_token_test 1.4 "\u1E9E" "0 \uDF \u1E9E"
-
-do_unicode_token_test 1.5 "The quick brown fox" {
-  0 the The 1 quick quick 2 brown brown 3 fox fox
-}
-do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" {
-  0 the The 1 quick quick 2 brown brown 3 fox fox
-}
-
-do_unicode_token_test2 1.7  {a B c D} {0 a a 1 b B 2 c c 3 d D}
-do_unicode_token_test2 1.8  "\uC4 \uD6 \uDC" "0 a \uC4 1 o \uD6 2 u \uDC"
-
-do_unicode_token_test2 1.9  "x\uC4x x\uD6x x\uDCx" \
-    "0 xax x\uC4x 1 xox x\uD6x 2 xux x\uDCx"
-
-# Check that diacritics are removed if remove_diacritics=1 is specified.
-# And that they do not break tokens.
-do_unicode_token_test2 1.10 "xx\u0301xx" "0 xxxx xx\u301xx"
-
-# Title-case mappings work
-do_unicode_token_test 1.11 "\u01c5" "0 \u01c6 \u01c5"
-
-#-------------------------------------------------------------------------
-#
-set docs [list {
-  Enhance the INSERT syntax to allow multiple rows to be inserted via the
-  VALUES clause.
-} {
-  Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
-} {
-  Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
-} {
-  Added the sqlite3_db_readonly() interface.
-} {
-  Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the
-  ability to add new PRAGMA statements or to override built-in PRAGMAs.  
-} {
-  Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
-  the same row that contains the maximum x value.
-} {
-  Added support for the FTS4 languageid option.
-} {
-  Documented support for the FTS4 content option. This feature has actually
-  been in the code since version 3.7.9 but is only now considered to be
-  officially supported.  
-} {
-  Pending statements no longer block ROLLBACK. Instead, the pending statement
-  will return SQLITE_ABORT upon next access after the ROLLBACK.  
-} {
-  Improvements to the handling of CSV inputs in the command-line shell
-} {
-  Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be
-  incorrectly converted into an INNER JOIN if the WHERE clause indexable terms
-  connected by OR.  
-}]
-
-set map(a) [list "\u00C4" "\u00E4"]  ; # LATIN LETTER A WITH DIAERESIS
-set map(e) [list "\u00CB" "\u00EB"]  ; # LATIN LETTER E WITH DIAERESIS
-set map(i) [list "\u00CF" "\u00EF"]  ; # LATIN LETTER I WITH DIAERESIS
-set map(o) [list "\u00D6" "\u00F6"]  ; # LATIN LETTER O WITH DIAERESIS
-set map(u) [list "\u00DC" "\u00FC"]  ; # LATIN LETTER U WITH DIAERESIS
-set map(y) [list "\u0178" "\u00FF"]  ; # LATIN LETTER Y WITH DIAERESIS
-set map(h) [list "\u1E26" "\u1E27"]  ; # LATIN LETTER H WITH DIAERESIS
-set map(w) [list "\u1E84" "\u1E85"]  ; # LATIN LETTER W WITH DIAERESIS
-set map(x) [list "\u1E8C" "\u1E8D"]  ; # LATIN LETTER X WITH DIAERESIS
-foreach k [array names map] {
-  lappend mappings [string toupper $k] [lindex $map($k) 0] 
-  lappend mappings $k [lindex $map($k) 1]
-}
-proc mapdoc {doc} { 
-  set doc [regsub -all {[[:space:]]+} $doc " "]
-  string map $::mappings [string trim $doc] 
-}
-
-do_test 2.0 {
-  execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); }
-  foreach doc $docs {
-    set d [mapdoc $doc]
-    execsql { INSERT INTO t2 VALUES($d) }
-  }
-} {}
-
-do_test 2.1 {
-  set q [mapdoc "row"]
-  execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
-} [list [mapdoc {
-  Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
-  the same row that contains the maximum x value.
-}]]
-
-foreach {tn query snippet} {
-  2 "row" {
-     ...returns the value of y on the same [row] that contains 
-     the maximum x value.
-  }
-  3 "ROW" {
-     ...returns the value of y on the same [row] that contains 
-     the maximum x value.
-  }
-  4 "rollback" {
-     ...[ROLLBACK]. Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
-  }
-  5 "rOllback" {
-     ...[ROLLBACK]. Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
-  }
-  6 "lang*" {
-     Added support for the FTS4 [languageid] option.
-  }
-} {
-  do_test 2.$tn {
-    set q [mapdoc $query]
-    execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q }
-  } [list [mapdoc $snippet]]
-}
-
-#-------------------------------------------------------------------------
-# Make sure the unicode61 tokenizer does not crash if it is passed a 
-# NULL pointer.
-reset_db
-do_execsql_test 3.1 {
-  CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x, y);
-  INSERT INTO t1 VALUES(NULL, 'a b c');
-}
-
-do_execsql_test 3.2 {
-  SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'b'
-} {{a [b] c}}
-
-do_execsql_test 3.3 {
-  BEGIN;
-  DELETE FROM t1;
-  INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b');
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 SELECT * FROM t1;
-  INSERT INTO t1 VALUES('a b c', NULL);
-  INSERT INTO t1 VALUES('a x c', NULL);
-  COMMIT;
-}
-
-do_execsql_test 3.4 {
-  SELECT * FROM t1 WHERE t1 MATCH 'a b';
-} {{a b c} {}}
-
-#-------------------------------------------------------------------------
-#
-reset_db
-
-do_test 4.1 {
-  set a "abc\uFFFEdef"
-  set b "abc\uD800def"
-  set c "\uFFFEdef"
-  set d "\uD800def"
-  execsql {
-    CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x);
-    INSERT INTO t1 VALUES($a);
-    INSERT INTO t1 VALUES($b);
-    INSERT INTO t1 VALUES($c);
-    INSERT INTO t1 VALUES($d);
-  }
-} {}
-
-do_test 4.2 {
-  set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
-  set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
-  set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
-  set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
-  execsql {
-    INSERT INTO t1 VALUES($a);
-    INSERT INTO t1 VALUES($b);
-    INSERT INTO t1 VALUES($c);
-    INSERT INTO t1 VALUES($d);
-  }
-} {}
-
-do_test 4.3 {
-  set a [binary format c* {0xF7 0xBF 0xBF 0xBF}]
-  set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}]
-  set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}]
-  set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}]
-  execsql {
-    INSERT INTO t1 VALUES($a);
-    INSERT INTO t1 VALUES($b);
-    INSERT INTO t1 VALUES($c);
-    INSERT INTO t1 VALUES($d);
-  }
-} {}
-
-#-------------------------------------------------------------------------
-
-do_unicode_token_test3 5.1 {tokenchars=} {
-  sqlite3_reset sqlite3_column_int
-} {
-  0 sqlite3 sqlite3 
-  1 reset reset 
-  2 sqlite3 sqlite3 
-  3 column column 
-  4 int int
-}
-
-do_unicode_token_test3 5.2 {tokenchars=_} {
-  sqlite3_reset sqlite3_column_int
-} {
-  0 sqlite3_reset sqlite3_reset 
-  1 sqlite3_column_int sqlite3_column_int
-}
-
-do_unicode_token_test3 5.3 {separators=xyz} {
-  Laotianxhorseyrunszfast
-} {
-  0 laotian Laotian
-  1 horse horse
-  2 runs runs
-  3 fast fast
-}
-
-do_unicode_token_test3 5.4 {tokenchars=xyz} {
-  Laotianxhorseyrunszfast
-} {
-  0 laotianxhorseyrunszfast Laotianxhorseyrunszfast
-}
-
-do_unicode_token_test3 5.5 {tokenchars=_} {separators=zyx} {
-  sqlite3_resetxsqlite3_column_intyhonda_phantom
-} {
-  0 sqlite3_reset sqlite3_reset 
-  1 sqlite3_column_int sqlite3_column_int
-  2 honda_phantom honda_phantom
-}
-
-do_unicode_token_test3 5.6 "separators=\u05D1" "abc\u05D1def" {
-  0 abc abc 1 def def
-}
-
-do_unicode_token_test3 5.7                             \
-  "tokenchars=\u2444\u2445"                            \
-  "separators=\u05D0\u05D1\u05D2"                      \
-  "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \
-  [list                                                \
-    0 \u2444fre\u2445sh \u2444fre\u2445sh              \
-    1 water water                                      \
-    2 fish fish                                        \
-    3 \u2445timer \u2445timer                          \
-  ]
-
-# Check that it is not possible to add a standalone diacritic codepoint 
-# to either separators or tokenchars.
-do_unicode_token_test3 5.8 "separators=\u0301" \
-  "hello\u0301world \u0301helloworld"          \
-  "0 helloworld hello\u0301world 1 helloworld helloworld"
-
-do_unicode_token_test3 5.9 "tokenchars=\u0301" \
-  "hello\u0301world \u0301helloworld"          \
-  "0 helloworld hello\u0301world 1 helloworld helloworld"
-
-do_unicode_token_test3 5.10 "separators=\u0301" \
-  "remove_diacritics=0"                        \
-  "hello\u0301world \u0301helloworld"          \
-  "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
-
-do_unicode_token_test3 5.11 "tokenchars=\u0301" \
-  "remove_diacritics=0"                         \
-  "hello\u0301world \u0301helloworld"           \
-  "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
-
-
-#-------------------------------------------------------------------------
-
-proc do_tokenize {tokenizer txt} {
-  set res [list]
-  foreach {a b c} [db one {SELECT fts3_tokenizer_test($tokenizer, $txt)}] {
-    lappend res $b
-  }
-  set res
-}
-
-# Argument $lCodepoint must be a list of codepoints (integers) that 
-# correspond to whitespace characters. This command creates a string
-# $W from the codepoints, then tokenizes "${W}hello{$W}world${W}" 
-# using tokenizer $tokenizer. The test passes if the tokenizer successfully
-# extracts the two 5 character tokens.
-#
-proc do_isspace_test {tn tokenizer lCp} {
-  set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] 
-  set txt "${whitespace}hello${whitespace}world${whitespace}"
-  uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}]
-}
-
-set tokenizers [list unicode61]
-ifcapable icu { lappend tokenizers icu }
-
-# Some tests to check that the tokenizers can both identify white-space 
-# codepoints. All codepoints tested below are of type "Zs" in the
-# UnicodeData.txt file.
-#
-# Note that codepoint 6158 has changed from Zs to Cf in recent versions
-# of UnicodeData.txt.  So take that into account for the "icu" tests.
-#
-foreach T $tokenizers {
-  do_isspace_test 6.$T.1 $T    32
-  do_isspace_test 6.$T.2 $T    160
-  do_isspace_test 6.$T.3 $T    5760
-  if {$T!="icu"} {
-    do_isspace_test 6.$T.4 $T    6158
-  }
-  do_isspace_test 6.$T.5 $T    8192
-  do_isspace_test 6.$T.6 $T    8193
-  do_isspace_test 6.$T.7 $T    8194
-  do_isspace_test 6.$T.8 $T    8195
-  do_isspace_test 6.$T.9 $T    8196
-  do_isspace_test 6.$T.10 $T    8197
-  do_isspace_test 6.$T.11 $T    8198
-  do_isspace_test 6.$T.12 $T    8199
-  do_isspace_test 6.$T.13 $T    8200
-  do_isspace_test 6.$T.14 $T    8201
-  do_isspace_test 6.$T.15 $T    8202
-  do_isspace_test 6.$T.16 $T    8239
-  do_isspace_test 6.$T.17 $T    8287
-  do_isspace_test 6.$T.18 $T   12288
-
-  if {$T!="icu"} {
-    do_isspace_test 6.$T.19 $T   {32 160 5760 6158}
-  } else {
-    do_isspace_test 6.$T.19 $T   {32 160 5760 8192}
-  }
-  do_isspace_test 6.$T.20 $T   {8192 8193 8194 8195}
-  do_isspace_test 6.$T.21 $T   {8196 8197 8198 8199}
-  do_isspace_test 6.$T.22 $T   {8200 8201 8202 8239}
-  do_isspace_test 6.$T.23 $T   {8287 12288}
-}
-
-#-------------------------------------------------------------------------
-# Test that the private use ranges are treated as alphanumeric.
-#
-foreach {tn1 c} {
-  1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff
-} {
-  foreach {tn2 config res} {
-    1 ""             "0 hello*world hello*world"
-    2 "separators=*" "0 hello hello 1 world world"
-  } {
-    set config [string map [list * $c] $config]
-    set input  [string map [list * $c] "hello*world"]
-    set output [string map [list * $c] $res]
-    do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output
-  }
-}
-
-#-------------------------------------------------------------------------
-# Cursory test of remove_diacritics=0.
-#
-# 00C4;LATIN CAPITAL LETTER A WITH DIAERESIS
-# 00D6;LATIN CAPITAL LETTER O WITH DIAERESIS
-# 00E4;LATIN SMALL LETTER A WITH DIAERESIS
-# 00F6;LATIN SMALL LETTER O WITH DIAERESIS
-#
-do_execsql_test 8.1.1 "
-  CREATE VIRTUAL TABLE t3 USING fts4(tokenize=unicode61 'remove_diacritics=1');
-  INSERT INTO t3 VALUES('o');
-  INSERT INTO t3 VALUES('a');
-  INSERT INTO t3 VALUES('O');
-  INSERT INTO t3 VALUES('A');
-  INSERT INTO t3 VALUES('\xD6');
-  INSERT INTO t3 VALUES('\xC4');
-  INSERT INTO t3 VALUES('\xF6');
-  INSERT INTO t3 VALUES('\xE4');
-"
-do_execsql_test 8.1.2 {
-  SELECT rowid FROM t3 WHERE t3 MATCH 'o';
-} {1 3 5 7}
-do_execsql_test 8.1.3 {
-  SELECT rowid FROM t3 WHERE t3 MATCH 'a';
-} {2 4 6 8}
-do_execsql_test 8.2.1 {
-  CREATE VIRTUAL TABLE t4 USING fts4(tokenize=unicode61 "remove_diacritics=0");
-  INSERT INTO t4 SELECT * FROM t3;
-}
-do_execsql_test 8.2.2 {
-  SELECT rowid FROM t4 WHERE t4 MATCH 'o';
-} {1 3}
-do_execsql_test 8.2.3 {
-  SELECT rowid FROM t4 WHERE t4 MATCH 'a';
-} {2 4}
-
-#-------------------------------------------------------------------------
-#
-foreach {tn sql} {
-  1 {
-    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]);
-    CREATE VIRTUAL TABLE t6 USING fts4(
-        tokenize=unicode61 [tokenchars=="] "tokenchars=[]");
-    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]);
-  }
-  2 {
-    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= .");
-    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]");
-    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4");
-  }
-  3 {
-    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .');
-    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]');
-    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4');
-  }
-  4 {
-    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`);
-    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`);
-    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`);
-  }
-} {
-  do_execsql_test 9.$tn.0 { 
-    DROP TABLE IF EXISTS t5;
-    DROP TABLE IF EXISTS t5aux;
-    DROP TABLE IF EXISTS t6;
-    DROP TABLE IF EXISTS t6aux;
-    DROP TABLE IF EXISTS t7;
-    DROP TABLE IF EXISTS t7aux;
-  }
-  do_execsql_test 9.$tn.1 $sql
-
-  do_execsql_test 9.$tn.2 {
-    CREATE VIRTUAL TABLE t5aux USING fts4aux(t5);
-    INSERT INTO t5 VALUES('one two three/four.five.six');
-    SELECT * FROM t5aux;
-  } {
-    four.five.six   * 1 1 four.five.six   0 1 1 
-    {one two three} * 1 1 {one two three} 0 1 1
-  }
-
-  do_execsql_test 9.$tn.3 {
-    CREATE VIRTUAL TABLE t6aux USING fts4aux(t6);
-    INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta');
-    SELECT * FROM t6aux;
-  } {
-    {alpha=beta"gamma}   * 1 1 {alpha=beta"gamma} 0 1 1 
-    {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1
-  }
-
-  do_execsql_test 9.$tn.4 {
-    CREATE VIRTUAL TABLE t7aux USING fts4aux(t7);
-    INSERT INTO t7 VALUES('alephxbeth\xC4gimel');
-    SELECT * FROM t7aux;
-  } {
-    aleph * 1 1 aleph 0 1 1 
-    beth  * 1 1 beth  0 1 1 
-    gimel * 1 1 gimel 0 1 1
-  }
-}
-
-# Check that multiple options are handled correctly.
-#
-do_execsql_test 10.1 {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61
-    "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy"
-    "separators=a" "separators=a" "tokenchars=a" "tokenchars=a"
-  );
-
-  INSERT INTO t1 VALUES('oneatwoxthreeyfour');
-  INSERT INTO t1 VALUES('a.single=word');
-  CREATE VIRTUAL TABLE t1aux USING fts4aux(t1);
-  SELECT * FROM t1aux;
-} {
-  .single=word * 1 1 .single=word 0 1 1 
-  four         * 1 1 four         0 1 1 
-  one          * 1 1 one          0 1 1 
-  three        * 1 1 three        0 1 1 
-  two          * 1 1 two          0 1 1
-}
-
-# Test that case folding happens after tokenization, not before.
-#
-do_execsql_test 10.2 {
-  DROP TABLE IF EXISTS t2;
-  CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB");
-  INSERT INTO t2 VALUES('oneatwoBthree');
-  INSERT INTO t2 VALUES('onebtwoAthree');
-  CREATE VIRTUAL TABLE t2aux USING fts4aux(t2);
-  SELECT * FROM t2aux;
-} {
-  one           * 1 1 one           0 1 1 
-  onebtwoathree * 1 1 onebtwoathree 0 1 1 
-  three         * 1 1 three         0 1 1 
-  two           * 1 1 two           0 1 1
-}
-
-# Test that the tokenchars and separators options work with the 
-# fts3tokenize table.
-#
-do_execsql_test 11.1 {
-  CREATE VIRTUAL TABLE ft1 USING fts3tokenize(
-    "unicode61", "tokenchars=@.", "separators=1234567890"
-  );
-  SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road';
-} {
-  berlin@street sydney.road
-}
-
-finish_test