[sql] Import reference version of SQLite 3.10.2.

third_party/sqlite/sqlite-src-3100200/ext/fts3/fts3_tokenizer.c

Index: third_party/sqlite/sqlite-src-3100200/ext/fts3/fts3_tokenizer.c
diff --git a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c b/third_party/sqlite/sqlite-src-3100200/ext/fts3/fts3_tokenizer.c
similarity index 96%
copy from third_party/sqlite/src/ext/fts3/fts3_tokenizer.c
copy to third_party/sqlite/sqlite-src-3100200/ext/fts3/fts3_tokenizer.c
index 04f84460e87be324685a16a1c9dd8c08b6e4bf0a..64cfe07aac38fd2aa5bec8c499b1fce4b858b4a5 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c
+++ b/third_party/sqlite/sqlite-src-3100200/ext/fts3/fts3_tokenizer.c
@@ -69,7 +69,7 @@ static void scalarFunc(
   if( argc==2 ){
     void *pOld;
     int n = sqlite3_value_bytes(argv[1]);
-    if( n!=sizeof(pPtr) ){
+    if( zName==0 || n!=sizeof(pPtr) ){
       sqlite3_result_error(context, "argument type mismatch", -1);
       return;
     }
@@ -80,7 +80,9 @@ static void scalarFunc(
       return;
     }
   }else{
-    pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
+    if( zName ){
+      pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
+    }
     if( !pPtr ){
       char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
       sqlite3_result_error(context, zErr, -1);
@@ -161,12 +163,16 @@ int sqlite3Fts3InitTokenizer(
   zEnd = &zCopy[strlen(zCopy)];
 
   z = (char *)sqlite3Fts3NextToken(zCopy, &n);
+  if( z==0 ){
+    assert( n==0 );
+    z = zCopy;
+  }
   z[n] = '\0';
   sqlite3Fts3Dequote(z);
 
   m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
   if( !m ){
-    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
+    sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z);
     rc = SQLITE_ERROR;
   }else{
     char const **aArg = 0;
@@ -189,7 +195,7 @@ int sqlite3Fts3InitTokenizer(
     rc = m->xCreate(iArg, aArg, ppTok);
     assert( rc!=SQLITE_OK || *ppTok );
     if( rc!=SQLITE_OK ){
-      *pzErr = sqlite3_mprintf("unknown tokenizer");
+      sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer");
     }else{
       (*ppTok)->pModule = m; 
     }
@@ -273,9 +279,9 @@ static void testFunc(
   p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
 
   if( !p ){
-    char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
-    sqlite3_result_error(context, zErr, -1);
-    sqlite3_free(zErr);
+    char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName);
+    sqlite3_result_error(context, zErr2, -1);
+    sqlite3_free(zErr2);
     return;
   }