Import SQLite 3.10.2.

third_party/sqlite/src/test/fts3tok1.test

 # 2013 April 22
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
 #
 #    May you do good and not evil.
 #    May you find forgiveness for yourself and forgive others.
 #    May you share freely, never taking more than you give.
 #
 #*************************************************************************
@@ skipping 92 matching lines @@
 #
 do_catchsql_test 2.0 {
   CREATE VIRTUAL TABLE tX USING fts3tokenize(nosuchtokenizer);
 } {1 {unknown tokenizer: nosuchtokenizer}}
 
 do_catchsql_test 2.1 {
   CREATE VIRTUAL TABLE t4 USING fts3tokenize;
   SELECT * FROM t4;
 } {1 {SQL logic error or missing database}}
 
+do_catchsql_test 2.2 {
+  CREATE VIRTUAL TABLE t USING fts4(tokenize=simple""); 
+} {0 {}}
+
+ifcapable fts3_unicode {
+  do_catchsql_test 2.3 {
+    CREATE VIRTUAL TABLE u USING fts4(tokenize=unicode61""); 
+  } {1 {unknown tokenizer}}
+}
+
 
 finish_test