[sql] Import reference version of SQLite 3.10.2.

third_party/sqlite/sqlite-src-3100200/ext/fts5/test/fts5unicode2.test

 # 2012 May 25
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
 #
 #    May you do good and not evil.
 #    May you find forgiveness for yourself and forgive others.
 #    May you share freely, never taking more than you give.
 #
 #*************************************************************************
 #
 # The tests in this file focus on testing the "unicode" FTS tokenizer.
 #
+# This is a modified copy of FTS4 test file "fts4_unicode.test".
+#
 
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-ifcapable !fts3_unicode { finish_test ; return }
-set ::testprefix fts4unicode
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5unicode2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
 
 proc do_unicode_token_test {tn input res} {
-  set input [string map {' ''} $input]
-  uplevel [list do_execsql_test $tn "
-    SELECT fts3_tokenizer_test('unicode61', 'remove_diacritics=0', '$input');
-  " [list [list {*}$res]]]
+  uplevel [list do_test $tn [list \
+    sqlite3_fts5_tokenize -subst db "unicode61 remove_diacritics 0" $input
+  ] [list {*}$res]]
 }
 
 proc do_unicode_token_test2 {tn input res} {
-  set input [string map {' ''} $input]
-  uplevel [list do_execsql_test $tn "
-    SELECT fts3_tokenizer_test('unicode61', '$input');
-  " [list [list {*}$res]]]
+  uplevel [list do_test $tn [list \
+    sqlite3_fts5_tokenize -subst db "unicode61" $input
+  ] [list {*}$res]]
 }
 
 proc do_unicode_token_test3 {tn args} {
-  set res   [lindex $args end]
-  set sql "SELECT fts3_tokenizer_test('unicode61'"
-  foreach a [lrange $args 0 end-1] {
-    append sql ", '"
-    append sql [string map {' ''} $a]
-    append sql "'"
-  }
-  append sql ")"
-  uplevel [list do_execsql_test $tn $sql [list [list {*}$res]]]
+  set tokenizer [concat unicode61 {*}[lrange $args 0 end-2]]
+  set input [lindex $args end-1]
+  set res [lindex $args end]
+  uplevel [list do_test $tn [list \
+    sqlite3_fts5_tokenize -subst db $tokenizer $input
+  ] [list {*}$res]]
 }
 
-do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D}
+do_unicode_token_test 1.0 {a B c D} {a a b B c c d D}
 
 do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \
-    "0 \uE4 \uC4 1 \uF6 \uD6 2 \uFC \uDC"
+    "\uE4 \uC4 \uF6 \uD6 \uFC \uDC"
 
 do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \
-    "0 x\uE4x x\uC4x 1 x\uF6x x\uD6x 2 x\uFCx x\uDCx"
+    "x\uE4x x\uC4x x\uF6x x\uD6x x\uFCx x\uDCx"
 
 # 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s.
-do_unicode_token_test 1.3 "\uDF" "0 \uDF \uDF"
-do_unicode_token_test 1.4 "\u1E9E" "0 \uDF \u1E9E"
+do_unicode_token_test 1.3 "\uDF" "\uDF \uDF"
+do_unicode_token_test 1.4 "\u1E9E" "\uDF \u1E9E"
 
 do_unicode_token_test 1.5 "The quick brown fox" {
-  0 the The 1 quick quick 2 brown brown 3 fox fox
+  the The quick quick brown brown fox fox
 }
 do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" {
-  0 the The 1 quick quick 2 brown brown 3 fox fox
+  the The quick quick brown brown fox fox
 }
 
-do_unicode_token_test2 1.7  {a B c D} {0 a a 1 b B 2 c c 3 d D}
-do_unicode_token_test2 1.8  "\uC4 \uD6 \uDC" "0 a \uC4 1 o \uD6 2 u \uDC"
+do_unicode_token_test2 1.7  {a B c D} {a a b B c c d D}
+do_unicode_token_test2 1.8  "\uC4 \uD6 \uDC" "a \uC4 o \uD6 u \uDC"
 
 do_unicode_token_test2 1.9  "x\uC4x x\uD6x x\uDCx" \
-    "0 xax x\uC4x 1 xox x\uD6x 2 xux x\uDCx"
+    "xax x\uC4x xox x\uD6x xux x\uDCx"
 
 # Check that diacritics are removed if remove_diacritics=1 is specified.
 # And that they do not break tokens.
-do_unicode_token_test2 1.10 "xx\u0301xx" "0 xxxx xx\u301xx"
+do_unicode_token_test2 1.10 "xx\u0301xx" "xxxx xx\u301xx"
 
 # Title-case mappings work
-do_unicode_token_test 1.11 "\u01c5" "0 \u01c6 \u01c5"
+do_unicode_token_test 1.11 "\u01c5" "\u01c6 \u01c5"
+
+do_unicode_token_test 1.12 "\u00C1abc\u00C2 \u00D1def\u00C3" \
+    "\u00E1abc\u00E2 \u00C1abc\u00C2 \u00F1def\u00E3 \u00D1def\u00C3"
+
+do_unicode_token_test 1.13 "\u00A2abc\u00A3 \u00A4def\u00A5" \
+    "abc abc def def"
 
 #-------------------------------------------------------------------------
 #
 set docs [list {
   Enhance the INSERT syntax to allow multiple rows to be inserted via the
   VALUES clause.
 } {
   Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
 } {
   Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
@@ skipping 34 matching lines @@
 foreach k [array names map] {
   lappend mappings [string toupper $k] [lindex $map($k) 0] 
   lappend mappings $k [lindex $map($k) 1]
 }
 proc mapdoc {doc} { 
   set doc [regsub -all {[[:space:]]+} $doc " "]
   string map $::mappings [string trim $doc] 
 }
 
 do_test 2.0 {
-  execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); }
+  execsql { CREATE VIRTUAL TABLE t2 USING fts5(tokenize=unicode61, x); }
   foreach doc $docs {
     set d [mapdoc $doc]
     execsql { INSERT INTO t2 VALUES($d) }
   }
 } {}
 
 do_test 2.1 {
   set q [mapdoc "row"]
   execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
 } [list [mapdoc {
@@ skipping 17 matching lines @@
   5 "rOllback" {
      ...[ROLLBACK]. Instead, the pending statement
      will return SQLITE_ABORT upon next access after the [ROLLBACK].
   }
   6 "lang*" {
      Added support for the FTS4 [languageid] option.
   }
 } {
   do_test 2.$tn {
     set q [mapdoc $query]
-    execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q }
+    execsql { 
+      SELECT snippet(t2, -1, '[', ']', '...', 15) FROM t2 WHERE t2 MATCH $q 
+    }
   } [list [mapdoc $snippet]]
 }
 
 #-------------------------------------------------------------------------
 # Make sure the unicode61 tokenizer does not crash if it is passed a 
 # NULL pointer.
 reset_db
 do_execsql_test 3.1 {
-  CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x, y);
+  CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x, y);
   INSERT INTO t1 VALUES(NULL, 'a b c');
 }
 
 do_execsql_test 3.2 {
-  SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'b'
+  SELECT snippet(t1, -1, '[', ']', '...', 15) FROM t1 WHERE t1 MATCH 'b'
 } {{a [b] c}}
 
 do_execsql_test 3.3 {
   BEGIN;
   DELETE FROM t1;
   INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b');
   INSERT INTO t1 SELECT * FROM t1;
   INSERT INTO t1 SELECT * FROM t1;
   INSERT INTO t1 SELECT * FROM t1;
   INSERT INTO t1 SELECT * FROM t1;
@@ skipping 21 matching lines @@
 #-------------------------------------------------------------------------
 #
 reset_db
 
 do_test 4.1 {
   set a "abc\uFFFEdef"
   set b "abc\uD800def"
   set c "\uFFFEdef"
   set d "\uD800def"
   execsql {
-    CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x);
+    CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x);
     INSERT INTO t1 VALUES($a);
     INSERT INTO t1 VALUES($b);
     INSERT INTO t1 VALUES($c);
     INSERT INTO t1 VALUES($d);
   }
+
+  execsql "CREATE VIRTUAL TABLE t8 USING fts5(
+      a, b, tokenize=\"unicode61 separators '\uFFFE\uD800\u00BF'\"
+  )"
 } {}
 
 do_test 4.2 {
   set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
   set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
   set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
   set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
   execsql {
     INSERT INTO t1 VALUES($a);
     INSERT INTO t1 VALUES($b);
     INSERT INTO t1 VALUES($c);
     INSERT INTO t1 VALUES($d);
   }
 } {}
 
 do_test 4.3 {
   set a [binary format c* {0xF7 0xBF 0xBF 0xBF}]
   set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}]
   set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}]
   set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}]
   execsql {
     INSERT INTO t1 VALUES($a);
     INSERT INTO t1 VALUES($b);
     INSERT INTO t1 VALUES($c);
     INSERT INTO t1 VALUES($d);
   }
 } {}
 
+do_test 4.4 {
+  sqlite3_exec_hex db {
+    CREATE VIRTUAL TABLE t9 USING fts5(a, b, 
+      tokenize="unicode61 separators '%C09004'"
+    );
+    INSERT INTO t9(a) VALUES('abc%88def %89ghi%90');
+  }
+} {0 {}}
+
+
 #-------------------------------------------------------------------------
 
-do_unicode_token_test3 5.1 {tokenchars=} {
+breakpoint
+do_unicode_token_test3 5.1 {tokenchars {}} {
   sqlite3_reset sqlite3_column_int
 } {
-  0 sqlite3 sqlite3 
-  1 reset reset 
-  2 sqlite3 sqlite3 
-  3 column column 
-  4 int int
+  sqlite3 sqlite3 
+  reset reset 
+  sqlite3 sqlite3 
+  column column 
+  int int
 }
 
-do_unicode_token_test3 5.2 {tokenchars=_} {
+do_unicode_token_test3 5.2 {tokenchars _} {
   sqlite3_reset sqlite3_column_int
 } {
-  0 sqlite3_reset sqlite3_reset 
-  1 sqlite3_column_int sqlite3_column_int
+  sqlite3_reset sqlite3_reset 
+  sqlite3_column_int sqlite3_column_int
 }
 
-do_unicode_token_test3 5.3 {separators=xyz} {
+do_unicode_token_test3 5.3 {separators xyz} {
   Laotianxhorseyrunszfast
 } {
-  0 laotian Laotian
-  1 horse horse
-  2 runs runs
-  3 fast fast
+  laotian Laotian
+  horse horse
+  runs runs
+  fast fast
 }
 
-do_unicode_token_test3 5.4 {tokenchars=xyz} {
+do_unicode_token_test3 5.4 {tokenchars xyz} {
   Laotianxhorseyrunszfast
 } {
-  0 laotianxhorseyrunszfast Laotianxhorseyrunszfast
+  laotianxhorseyrunszfast Laotianxhorseyrunszfast
 }
 
-do_unicode_token_test3 5.5 {tokenchars=_} {separators=zyx} {
+do_unicode_token_test3 5.5 {tokenchars _} {separators zyx} {
   sqlite3_resetxsqlite3_column_intyhonda_phantom
 } {
-  0 sqlite3_reset sqlite3_reset 
-  1 sqlite3_column_int sqlite3_column_int
-  2 honda_phantom honda_phantom
+  sqlite3_reset sqlite3_reset 
+  sqlite3_column_int sqlite3_column_int
+  honda_phantom honda_phantom
 }
 
-do_unicode_token_test3 5.6 "separators=\u05D1" "abc\u05D1def" {
-  0 abc abc 1 def def
+do_unicode_token_test3 5.6 "separators \u05D1" "abc\u05D1def" {
+  abc abc def def
 }
 
 do_unicode_token_test3 5.7                             \
-  "tokenchars=\u2444\u2445"                            \
-  "separators=\u05D0\u05D1\u05D2"                      \
+  "tokenchars \u2444\u2445"                            \
+  "separators \u05D0\u05D1\u05D2"                      \
   "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \
   [list                                                \
-    0 \u2444fre\u2445sh \u2444fre\u2445sh              \
-    1 water water                                      \
-    2 fish fish                                        \
-    3 \u2445timer \u2445timer                          \
+    \u2444fre\u2445sh \u2444fre\u2445sh              \
+    water water                                      \
+    fish fish                                        \
+    \u2445timer \u2445timer                          \
   ]
 
 # Check that it is not possible to add a standalone diacritic codepoint 
 # to either separators or tokenchars.
-do_unicode_token_test3 5.8 "separators=\u0301" \
+do_unicode_token_test3 5.8 "separators \u0301" \
   "hello\u0301world \u0301helloworld"          \
-  "0 helloworld hello\u0301world 1 helloworld helloworld"
+  "helloworld hello\u0301world helloworld helloworld"
 
-do_unicode_token_test3 5.9 "tokenchars=\u0301" \
+do_unicode_token_test3 5.9 "tokenchars \u0301" \
   "hello\u0301world \u0301helloworld"          \
-  "0 helloworld hello\u0301world 1 helloworld helloworld"
+  "helloworld hello\u0301world helloworld helloworld"
 
-do_unicode_token_test3 5.10 "separators=\u0301" \
-  "remove_diacritics=0"                        \
+do_unicode_token_test3 5.10 "separators \u0301" \
+  "remove_diacritics 0"                        \
   "hello\u0301world \u0301helloworld"          \
-  "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
+  "hello\u0301world hello\u0301world helloworld helloworld"
 
-do_unicode_token_test3 5.11 "tokenchars=\u0301" \
-  "remove_diacritics=0"                         \
+do_unicode_token_test3 5.11 "tokenchars \u0301" \
+  "remove_diacritics 0"                         \
   "hello\u0301world \u0301helloworld"           \
-  "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
-
+  "hello\u0301world hello\u0301world helloworld helloworld"
 
 #-------------------------------------------------------------------------
 
 proc do_tokenize {tokenizer txt} {
   set res [list]
-  foreach {a b c} [db one {SELECT fts3_tokenizer_test($tokenizer, $txt)}] {
+  foreach {b c} [sqlite3_fts5_tokenize -subst db $tokenizer $txt] {
     lappend res $b
   }
   set res
 }
 
 # Argument $lCodepoint must be a list of codepoints (integers) that 
 # correspond to whitespace characters. This command creates a string
 # $W from the codepoints, then tokenizes "${W}hello{$W}world${W}" 
 # using tokenizer $tokenizer. The test passes if the tokenizer successfully
 # extracts the two 5 character tokens.
 #
 proc do_isspace_test {tn tokenizer lCp} {
   set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] 
   set txt "${whitespace}hello${whitespace}world${whitespace}"
   uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}]
 }
 
 set tokenizers [list unicode61]
-ifcapable icu { lappend tokenizers icu }
+#ifcapable icu { lappend tokenizers icu }
 
 # Some tests to check that the tokenizers can both identify white-space 
 # codepoints. All codepoints tested below are of type "Zs" in the
 # UnicodeData.txt file.
 foreach T $tokenizers {
   do_isspace_test 6.$T.1 $T    32
   do_isspace_test 6.$T.2 $T    160
   do_isspace_test 6.$T.3 $T    5760
   do_isspace_test 6.$T.4 $T    6158
   do_isspace_test 6.$T.5 $T    8192
@@ skipping 11 matching lines @@
   do_isspace_test 6.$T.17 $T    8287
   do_isspace_test 6.$T.18 $T   12288
 
   do_isspace_test 6.$T.19 $T   {32 160 5760 6158}
   do_isspace_test 6.$T.20 $T   {8192 8193 8194 8195}
   do_isspace_test 6.$T.21 $T   {8196 8197 8198 8199}
   do_isspace_test 6.$T.22 $T   {8200 8201 8202 8239}
   do_isspace_test 6.$T.23 $T   {8287 12288}
 }
 
+
 #-------------------------------------------------------------------------
 # Test that the private use ranges are treated as alphanumeric.
 #
 foreach {tn1 c} {
   1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff
 } {
   foreach {tn2 config res} {
-    1 ""             "0 hello*world hello*world"
-    2 "separators=*" "0 hello hello 1 world world"
+    1 ""             "hello*world hello*world"
+    2 "separators *" "hello hello world world"
   } {
     set config [string map [list * $c] $config]
     set input  [string map [list * $c] "hello*world"]
     set output [string map [list * $c] $res]
     do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output
   }
 }
 
 #-------------------------------------------------------------------------
 # Cursory test of remove_diacritics=0.
 #
 # 00C4;LATIN CAPITAL LETTER A WITH DIAERESIS
 # 00D6;LATIN CAPITAL LETTER O WITH DIAERESIS
 # 00E4;LATIN SMALL LETTER A WITH DIAERESIS
 # 00F6;LATIN SMALL LETTER O WITH DIAERESIS
 #
 do_execsql_test 8.1.1 "
-  CREATE VIRTUAL TABLE t3 USING fts4(tokenize=unicode61 'remove_diacritics=1');
+  CREATE VIRTUAL TABLE t3 USING fts5(
+    content, tokenize='unicode61 remove_diacritics 1'
+  );
   INSERT INTO t3 VALUES('o');
   INSERT INTO t3 VALUES('a');
   INSERT INTO t3 VALUES('O');
   INSERT INTO t3 VALUES('A');
   INSERT INTO t3 VALUES('\xD6');
   INSERT INTO t3 VALUES('\xC4');
   INSERT INTO t3 VALUES('\xF6');
   INSERT INTO t3 VALUES('\xE4');
 "
 do_execsql_test 8.1.2 {
-  SELECT rowid FROM t3 WHERE t3 MATCH 'o';
+  SELECT rowid FROM t3 WHERE t3 MATCH 'o' ORDER BY rowid ASC;
 } {1 3 5 7}
 do_execsql_test 8.1.3 {
-  SELECT rowid FROM t3 WHERE t3 MATCH 'a';
+  SELECT rowid FROM t3 WHERE t3 MATCH 'a' ORDER BY rowid ASC;
 } {2 4 6 8}
 do_execsql_test 8.2.1 {
-  CREATE VIRTUAL TABLE t4 USING fts4(tokenize=unicode61 "remove_diacritics=0");
-  INSERT INTO t4 SELECT * FROM t3;
+  CREATE VIRTUAL TABLE t4 USING fts5(
+    content, tokenize='unicode61 remove_diacritics 0'
+  );
+  INSERT INTO t4 SELECT * FROM t3 ORDER BY rowid ASC;
 }
 do_execsql_test 8.2.2 {
-  SELECT rowid FROM t4 WHERE t4 MATCH 'o';
+  SELECT rowid FROM t4 WHERE t4 MATCH 'o' ORDER BY rowid ASC;
 } {1 3}
 do_execsql_test 8.2.3 {
-  SELECT rowid FROM t4 WHERE t4 MATCH 'a';
+  SELECT rowid FROM t4 WHERE t4 MATCH 'a' ORDER BY rowid ASC;
 } {2 4}
 
 #-------------------------------------------------------------------------
 #
+if 0 {
 foreach {tn sql} {
   1 {
     CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]);
     CREATE VIRTUAL TABLE t6 USING fts4(
         tokenize=unicode61 [tokenchars=="] "tokenchars=[]");
     CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]);
   }
   2 {
     CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= .");
     CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]");
@@ skipping 91 matching lines @@
 #
 do_execsql_test 11.1 {
   CREATE VIRTUAL TABLE ft1 USING fts3tokenize(
     "unicode61", "tokenchars=@.", "separators=1234567890"
   );
   SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road';
 } {
   berlin@street sydney.road
 }
 
+}
+
 finish_test