[sql] Import reference version of SQLite 3.17..

third_party/sqlite/sqlite-src-3170000/ext/fts5/test/fts5unicode2.test

+# 2012 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# The tests in this file focus on testing the "unicode" FTS tokenizer.
+#
+# This is a modified copy of FTS4 test file "fts4_unicode.test".
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5unicode2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc do_unicode_token_test {tn input res} {
+  uplevel [list do_test $tn [list \
+    sqlite3_fts5_tokenize -subst db "unicode61 remove_diacritics 0" $input
+  ] [list {*}$res]]
+}
+
+proc do_unicode_token_test2 {tn input res} {
+  uplevel [list do_test $tn [list \
+    sqlite3_fts5_tokenize -subst db "unicode61" $input
+  ] [list {*}$res]]
+}
+
+proc do_unicode_token_test3 {tn args} {
+  set tokenizer [concat unicode61 {*}[lrange $args 0 end-2]]
+  set input [lindex $args end-1]
+  set res [lindex $args end]
+  uplevel [list do_test $tn [list \
+    sqlite3_fts5_tokenize -subst db $tokenizer $input
+  ] [list {*}$res]]
+}
+
+do_unicode_token_test 1.0 {a B c D} {a a b B c c d D}
+
+do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \
+    "\uE4 \uC4 \uF6 \uD6 \uFC \uDC"
+
+do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \
+    "x\uE4x x\uC4x x\uF6x x\uD6x x\uFCx x\uDCx"
+
+# 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s.
+do_unicode_token_test 1.3 "\uDF" "\uDF \uDF"
+do_unicode_token_test 1.4 "\u1E9E" "\uDF \u1E9E"
+
+do_unicode_token_test 1.5 "The quick brown fox" {
+  the The quick quick brown brown fox fox
+}
+do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" {
+  the The quick quick brown brown fox fox
+}
+
+do_unicode_token_test2 1.7  {a B c D} {a a b B c c d D}
+do_unicode_token_test2 1.8  "\uC4 \uD6 \uDC" "a \uC4 o \uD6 u \uDC"
+
+do_unicode_token_test2 1.9  "x\uC4x x\uD6x x\uDCx" \
+    "xax x\uC4x xox x\uD6x xux x\uDCx"
+
+# Check that diacritics are removed if remove_diacritics=1 is specified.
+# And that they do not break tokens.
+do_unicode_token_test2 1.10 "xx\u0301xx" "xxxx xx\u301xx"
+
+# Title-case mappings work
+do_unicode_token_test 1.11 "\u01c5" "\u01c6 \u01c5"
+
+do_unicode_token_test 1.12 "\u00C1abc\u00C2 \u00D1def\u00C3" \
+    "\u00E1abc\u00E2 \u00C1abc\u00C2 \u00F1def\u00E3 \u00D1def\u00C3"
+
+do_unicode_token_test 1.13 "\u00A2abc\u00A3 \u00A4def\u00A5" \
+    "abc abc def def"
+
+#-------------------------------------------------------------------------
+#
+set docs [list {
+  Enhance the INSERT syntax to allow multiple rows to be inserted via the
+  VALUES clause.
+} {
+  Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
+} {
+  Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
+} {
+  Added the sqlite3_db_readonly() interface.
+} {
+  Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the
+  ability to add new PRAGMA statements or to override built-in PRAGMAs.  
+} {
+  Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
+  the same row that contains the maximum x value.
+} {
+  Added support for the FTS4 languageid option.
+} {
+  Documented support for the FTS4 content option. This feature has actually
+  been in the code since version 3.7.9 but is only now considered to be
+  officially supported.  
+} {
+  Pending statements no longer block ROLLBACK. Instead, the pending statement
+  will return SQLITE_ABORT upon next access after the ROLLBACK.  
+} {
+  Improvements to the handling of CSV inputs in the command-line shell
+} {
+  Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be
+  incorrectly converted into an INNER JOIN if the WHERE clause indexable terms
+  connected by OR.  
+}]
+
+set map(a) [list "\u00C4" "\u00E4"]  ; # LATIN LETTER A WITH DIAERESIS
+set map(e) [list "\u00CB" "\u00EB"]  ; # LATIN LETTER E WITH DIAERESIS
+set map(i) [list "\u00CF" "\u00EF"]  ; # LATIN LETTER I WITH DIAERESIS
+set map(o) [list "\u00D6" "\u00F6"]  ; # LATIN LETTER O WITH DIAERESIS
+set map(u) [list "\u00DC" "\u00FC"]  ; # LATIN LETTER U WITH DIAERESIS
+set map(y) [list "\u0178" "\u00FF"]  ; # LATIN LETTER Y WITH DIAERESIS
+set map(h) [list "\u1E26" "\u1E27"]  ; # LATIN LETTER H WITH DIAERESIS
+set map(w) [list "\u1E84" "\u1E85"]  ; # LATIN LETTER W WITH DIAERESIS
+set map(x) [list "\u1E8C" "\u1E8D"]  ; # LATIN LETTER X WITH DIAERESIS
+foreach k [array names map] {
+  lappend mappings [string toupper $k] [lindex $map($k) 0] 
+  lappend mappings $k [lindex $map($k) 1]
+}
+proc mapdoc {doc} { 
+  set doc [regsub -all {[[:space:]]+} $doc " "]
+  string map $::mappings [string trim $doc] 
+}
+
+do_test 2.0 {
+  execsql { CREATE VIRTUAL TABLE t2 USING fts5(tokenize=unicode61, x); }
+  foreach doc $docs {
+    set d [mapdoc $doc]
+    execsql { INSERT INTO t2 VALUES($d) }
+  }
+} {}
+
+do_test 2.1 {
+  set q [mapdoc "row"]
+  execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
+} [list [mapdoc {
+  Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
+  the same row that contains the maximum x value.
+}]]
+
+foreach {tn query snippet} {
+  2 "row" {
+     ...returns the value of y on the same [row] that contains 
+     the maximum x value.
+  }
+  3 "ROW" {
+     ...returns the value of y on the same [row] that contains 
+     the maximum x value.
+  }
+  4 "rollback" {
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
+  }
+  5 "rOllback" {
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
+  }
+  6 "lang*" {
+     Added support for the FTS4 [languageid] option.
+  }
+} {
+  do_test 2.$tn {
+    set q [mapdoc $query]
+    execsql { 
+      SELECT snippet(t2, -1, '[', ']', '...', 15) FROM t2 WHERE t2 MATCH $q 
+    }
+  } [list [mapdoc $snippet]]
+}
+
+#-------------------------------------------------------------------------
+# Make sure the unicode61 tokenizer does not crash if it is passed a 
+# NULL pointer.
+reset_db
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x, y);
+  INSERT INTO t1 VALUES(NULL, 'a b c');
+}
+
+do_execsql_test 3.2 {
+  SELECT snippet(t1, -1, '[', ']', '...', 15) FROM t1 WHERE t1 MATCH 'b'
+} {{a [b] c}}
+
+do_execsql_test 3.3 {
+  BEGIN;
+  DELETE FROM t1;
+  INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b');
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 SELECT * FROM t1;
+  INSERT INTO t1 VALUES('a b c', NULL);
+  INSERT INTO t1 VALUES('a x c', NULL);
+  COMMIT;
+}
+
+do_execsql_test 3.4 {
+  SELECT * FROM t1 WHERE t1 MATCH 'a b';
+} {{a b c} {}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+
+do_test 4.1 {
+  set a "abc\uFFFEdef"
+  set b "abc\uD800def"
+  set c "\uFFFEdef"
+  set d "\uD800def"
+  execsql {
+    CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x);
+    INSERT INTO t1 VALUES($a);
+    INSERT INTO t1 VALUES($b);
+    INSERT INTO t1 VALUES($c);
+    INSERT INTO t1 VALUES($d);
+  }
+
+  execsql "CREATE VIRTUAL TABLE t8 USING fts5(
+      a, b, tokenize=\"unicode61 separators '\uFFFE\uD800\u00BF'\"
+  )"
+} {}
+
+do_test 4.2 {
+  set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
+  set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
+  set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
+  set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
+  execsql {
+    INSERT INTO t1 VALUES($a);
+    INSERT INTO t1 VALUES($b);
+    INSERT INTO t1 VALUES($c);
+    INSERT INTO t1 VALUES($d);
+  }
+} {}
+
+do_test 4.3 {
+  set a [binary format c* {0xF7 0xBF 0xBF 0xBF}]
+  set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}]
+  set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}]
+  set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}]
+  execsql {
+    INSERT INTO t1 VALUES($a);
+    INSERT INTO t1 VALUES($b);
+    INSERT INTO t1 VALUES($c);
+    INSERT INTO t1 VALUES($d);
+  }
+} {}
+
+do_test 4.4 {
+  sqlite3_exec_hex db {
+    CREATE VIRTUAL TABLE t9 USING fts5(a, b, 
+      tokenize="unicode61 separators '%C09004'"
+    );
+    INSERT INTO t9(a) VALUES('abc%88def %89ghi%90');
+  }
+} {0 {}}
+
+
+#-------------------------------------------------------------------------
+
+breakpoint
+do_unicode_token_test3 5.1 {tokenchars {}} {
+  sqlite3_reset sqlite3_column_int
+} {
+  sqlite3 sqlite3 
+  reset reset 
+  sqlite3 sqlite3 
+  column column 
+  int int
+}
+
+do_unicode_token_test3 5.2 {tokenchars _} {
+  sqlite3_reset sqlite3_column_int
+} {
+  sqlite3_reset sqlite3_reset 
+  sqlite3_column_int sqlite3_column_int
+}
+
+do_unicode_token_test3 5.3 {separators xyz} {
+  Laotianxhorseyrunszfast
+} {
+  laotian Laotian
+  horse horse
+  runs runs
+  fast fast
+}
+
+do_unicode_token_test3 5.4 {tokenchars xyz} {
+  Laotianxhorseyrunszfast
+} {
+  laotianxhorseyrunszfast Laotianxhorseyrunszfast
+}
+
+do_unicode_token_test3 5.5 {tokenchars _} {separators zyx} {
+  sqlite3_resetxsqlite3_column_intyhonda_phantom
+} {
+  sqlite3_reset sqlite3_reset 
+  sqlite3_column_int sqlite3_column_int
+  honda_phantom honda_phantom
+}
+
+do_unicode_token_test3 5.6 "separators \u05D1" "abc\u05D1def" {
+  abc abc def def
+}
+
+do_unicode_token_test3 5.7                             \
+  "tokenchars \u2444\u2445"                            \
+  "separators \u05D0\u05D1\u05D2"                      \
+  "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \
+  [list                                                \
+    \u2444fre\u2445sh \u2444fre\u2445sh              \
+    water water                                      \
+    fish fish                                        \
+    \u2445timer \u2445timer                          \
+  ]
+
+# Check that it is not possible to add a standalone diacritic codepoint 
+# to either separators or tokenchars.
+do_unicode_token_test3 5.8 "separators \u0301" \
+  "hello\u0301world \u0301helloworld"          \
+  "helloworld hello\u0301world helloworld helloworld"
+
+do_unicode_token_test3 5.9 "tokenchars \u0301" \
+  "hello\u0301world \u0301helloworld"          \
+  "helloworld hello\u0301world helloworld helloworld"
+
+do_unicode_token_test3 5.10 "separators \u0301" \
+  "remove_diacritics 0"                        \
+  "hello\u0301world \u0301helloworld"          \
+  "hello\u0301world hello\u0301world helloworld helloworld"
+
+do_unicode_token_test3 5.11 "tokenchars \u0301" \
+  "remove_diacritics 0"                         \
+  "hello\u0301world \u0301helloworld"           \
+  "hello\u0301world hello\u0301world helloworld helloworld"
+
+#-------------------------------------------------------------------------
+
+proc do_tokenize {tokenizer txt} {
+  set res [list]
+  foreach {b c} [sqlite3_fts5_tokenize -subst db $tokenizer $txt] {
+    lappend res $b
+  }
+  set res
+}
+
+# Argument $lCodepoint must be a list of codepoints (integers) that 
+# correspond to whitespace characters. This command creates a string
+# $W from the codepoints, then tokenizes "${W}hello{$W}world${W}" 
+# using tokenizer $tokenizer. The test passes if the tokenizer successfully
+# extracts the two 5 character tokens.
+#
+proc do_isspace_test {tn tokenizer lCp} {
+  set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] 
+  set txt "${whitespace}hello${whitespace}world${whitespace}"
+  uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}]
+}
+
+set tokenizers [list unicode61]
+#ifcapable icu { lappend tokenizers icu }
+
+# Some tests to check that the tokenizers can both identify white-space 
+# codepoints. All codepoints tested below are of type "Zs" in the
+# UnicodeData.txt file.
+foreach T $tokenizers {
+  do_isspace_test 6.$T.1 $T    32
+  do_isspace_test 6.$T.2 $T    160
+  do_isspace_test 6.$T.3 $T    5760
+  do_isspace_test 6.$T.4 $T    6158
+  do_isspace_test 6.$T.5 $T    8192
+  do_isspace_test 6.$T.6 $T    8193
+  do_isspace_test 6.$T.7 $T    8194
+  do_isspace_test 6.$T.8 $T    8195
+  do_isspace_test 6.$T.9 $T    8196
+  do_isspace_test 6.$T.10 $T    8197
+  do_isspace_test 6.$T.11 $T    8198
+  do_isspace_test 6.$T.12 $T    8199
+  do_isspace_test 6.$T.13 $T    8200
+  do_isspace_test 6.$T.14 $T    8201
+  do_isspace_test 6.$T.15 $T    8202
+  do_isspace_test 6.$T.16 $T    8239
+  do_isspace_test 6.$T.17 $T    8287
+  do_isspace_test 6.$T.18 $T   12288
+
+  do_isspace_test 6.$T.19 $T   {32 160 5760 6158}
+  do_isspace_test 6.$T.20 $T   {8192 8193 8194 8195}
+  do_isspace_test 6.$T.21 $T   {8196 8197 8198 8199}
+  do_isspace_test 6.$T.22 $T   {8200 8201 8202 8239}
+  do_isspace_test 6.$T.23 $T   {8287 12288}
+}
+
+
+#-------------------------------------------------------------------------
+# Test that the private use ranges are treated as alphanumeric.
+#
+foreach {tn1 c} {
+  1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff
+} {
+  foreach {tn2 config res} {
+    1 ""             "hello*world hello*world"
+    2 "separators *" "hello hello world world"
+  } {
+    set config [string map [list * $c] $config]
+    set input  [string map [list * $c] "hello*world"]
+    set output [string map [list * $c] $res]
+    do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output
+  }
+}
+
+#-------------------------------------------------------------------------
+# Cursory test of remove_diacritics=0.
+#
+# 00C4;LATIN CAPITAL LETTER A WITH DIAERESIS
+# 00D6;LATIN CAPITAL LETTER O WITH DIAERESIS
+# 00E4;LATIN SMALL LETTER A WITH DIAERESIS
+# 00F6;LATIN SMALL LETTER O WITH DIAERESIS
+#
+do_execsql_test 8.1.1 "
+  CREATE VIRTUAL TABLE t3 USING fts5(
+    content, tokenize='unicode61 remove_diacritics 1'
+  );
+  INSERT INTO t3 VALUES('o');
+  INSERT INTO t3 VALUES('a');
+  INSERT INTO t3 VALUES('O');
+  INSERT INTO t3 VALUES('A');
+  INSERT INTO t3 VALUES('\xD6');
+  INSERT INTO t3 VALUES('\xC4');
+  INSERT INTO t3 VALUES('\xF6');
+  INSERT INTO t3 VALUES('\xE4');
+"
+do_execsql_test 8.1.2 {
+  SELECT rowid FROM t3 WHERE t3 MATCH 'o' ORDER BY rowid ASC;
+} {1 3 5 7}
+do_execsql_test 8.1.3 {
+  SELECT rowid FROM t3 WHERE t3 MATCH 'a' ORDER BY rowid ASC;
+} {2 4 6 8}
+do_execsql_test 8.2.1 {
+  CREATE VIRTUAL TABLE t4 USING fts5(
+    content, tokenize='unicode61 remove_diacritics 0'
+  );
+  INSERT INTO t4 SELECT * FROM t3 ORDER BY rowid ASC;
+}
+do_execsql_test 8.2.2 {
+  SELECT rowid FROM t4 WHERE t4 MATCH 'o' ORDER BY rowid ASC;
+} {1 3}
+do_execsql_test 8.2.3 {
+  SELECT rowid FROM t4 WHERE t4 MATCH 'a' ORDER BY rowid ASC;
+} {2 4}
+
+#-------------------------------------------------------------------------
+#
+if 0 {
+foreach {tn sql} {
+  1 {
+    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]);
+    CREATE VIRTUAL TABLE t6 USING fts4(
+        tokenize=unicode61 [tokenchars=="] "tokenchars=[]");
+    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]);
+  }
+  2 {
+    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= .");
+    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]");
+    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4");
+  }
+  3 {
+    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .');
+    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]');
+    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4');
+  }
+  4 {
+    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`);
+    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`);
+    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`);
+  }
+} {
+  do_execsql_test 9.$tn.0 { 
+    DROP TABLE IF EXISTS t5;
+    DROP TABLE IF EXISTS t5aux;
+    DROP TABLE IF EXISTS t6;
+    DROP TABLE IF EXISTS t6aux;
+    DROP TABLE IF EXISTS t7;
+    DROP TABLE IF EXISTS t7aux;
+  }
+  do_execsql_test 9.$tn.1 $sql
+
+  do_execsql_test 9.$tn.2 {
+    CREATE VIRTUAL TABLE t5aux USING fts4aux(t5);
+    INSERT INTO t5 VALUES('one two three/four.five.six');
+    SELECT * FROM t5aux;
+  } {
+    four.five.six   * 1 1 four.five.six   0 1 1 
+    {one two three} * 1 1 {one two three} 0 1 1
+  }
+
+  do_execsql_test 9.$tn.3 {
+    CREATE VIRTUAL TABLE t6aux USING fts4aux(t6);
+    INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta');
+    SELECT * FROM t6aux;
+  } {
+    {alpha=beta"gamma}   * 1 1 {alpha=beta"gamma} 0 1 1 
+    {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1
+  }
+
+  do_execsql_test 9.$tn.4 {
+    CREATE VIRTUAL TABLE t7aux USING fts4aux(t7);
+    INSERT INTO t7 VALUES('alephxbeth\xC4gimel');
+    SELECT * FROM t7aux;
+  } {
+    aleph * 1 1 aleph 0 1 1 
+    beth  * 1 1 beth  0 1 1 
+    gimel * 1 1 gimel 0 1 1
+  }
+}
+
+# Check that multiple options are handled correctly.
+#
+do_execsql_test 10.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61
+    "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy"
+    "separators=a" "separators=a" "tokenchars=a" "tokenchars=a"
+  );
+
+  INSERT INTO t1 VALUES('oneatwoxthreeyfour');
+  INSERT INTO t1 VALUES('a.single=word');
+  CREATE VIRTUAL TABLE t1aux USING fts4aux(t1);
+  SELECT * FROM t1aux;
+} {
+  .single=word * 1 1 .single=word 0 1 1 
+  four         * 1 1 four         0 1 1 
+  one          * 1 1 one          0 1 1 
+  three        * 1 1 three        0 1 1 
+  two          * 1 1 two          0 1 1
+}
+
+# Test that case folding happens after tokenization, not before.
+#
+do_execsql_test 10.2 {
+  DROP TABLE IF EXISTS t2;
+  CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB");
+  INSERT INTO t2 VALUES('oneatwoBthree');
+  INSERT INTO t2 VALUES('onebtwoAthree');
+  CREATE VIRTUAL TABLE t2aux USING fts4aux(t2);
+  SELECT * FROM t2aux;
+} {
+  one           * 1 1 one           0 1 1 
+  onebtwoathree * 1 1 onebtwoathree 0 1 1 
+  three         * 1 1 three         0 1 1 
+  two           * 1 1 two           0 1 1
+}
+
+# Test that the tokenchars and separators options work with the 
+# fts3tokenize table.
+#
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE ft1 USING fts3tokenize(
+    "unicode61", "tokenchars=@.", "separators=1234567890"
+  );
+  SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road';
+} {
+  berlin@street sydney.road
+}
+
+}
+
+finish_test