[sql] Import SQLite 3.17.0.

third_party/sqlite/src/ext/fts5/fts5_tokenize.c

 /*
 ** 2014 May 31
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
@@ skipping 44 matching lines @@
 ** Delete a "ascii" tokenizer.
 */
 static void fts5AsciiDelete(Fts5Tokenizer *p){
   sqlite3_free(p);
 }
 
 /*
 ** Create an "ascii" tokenizer.
 */
 static int fts5AsciiCreate(
-  void *pCtx, 
+  void *pUnused, 
   const char **azArg, int nArg,
   Fts5Tokenizer **ppOut
 ){
   int rc = SQLITE_OK;
   AsciiTokenizer *p = 0;
+  UNUSED_PARAM(pUnused);
   if( nArg%2 ){
     rc = SQLITE_ERROR;
   }else{
     p = sqlite3_malloc(sizeof(AsciiTokenizer));
     if( p==0 ){
       rc = SQLITE_NOMEM;
     }else{
       int i;
       memset(p, 0, sizeof(AsciiTokenizer));
       memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
@@ skipping 28 matching lines @@
     aOut[i] = c;
   }
 }
 
 /*
 ** Tokenize some text using the ascii tokenizer.
 */
 static int fts5AsciiTokenize(
   Fts5Tokenizer *pTokenizer,
   void *pCtx,
-  int flags,
+  int iUnused,
   const char *pText, int nText,
   int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
 ){
   AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
   int rc = SQLITE_OK;
   int ie;
   int is = 0;
 
   char aFold[64];
   int nFold = sizeof(aFold);
   char *pFold = aFold;
   unsigned char *a = p->aTokenChar;
 
+  UNUSED_PARAM(iUnused);
+
   while( is<nText && rc==SQLITE_OK ){
     int nByte;
 
     /* Skip any leading divider characters. */
     while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){
       is++;
     }
     if( is==nText ) break;
 
     /* Count the token characters */
@@ skipping 173 matching lines @@
     sqlite3_free(p->aFold);
     sqlite3_free(p);
   }
   return;
 }
 
 /*
 ** Create a "unicode61" tokenizer.
 */
 static int fts5UnicodeCreate(
-  void *pCtx, 
+  void *pUnused, 
   const char **azArg, int nArg,
   Fts5Tokenizer **ppOut
 ){
   int rc = SQLITE_OK;             /* Return code */
   Unicode61Tokenizer *p = 0;      /* New tokenizer object */ 
 
+  UNUSED_PARAM(pUnused);
+
   if( nArg%2 ){
     rc = SQLITE_ERROR;
   }else{
     p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
     if( p ){
       int i;
       memset(p, 0, sizeof(Unicode61Tokenizer));
       memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
       p->bRemoveDiacritic = 1;
       p->nFold = 64;
@@ skipping 36 matching lines @@
 ** character (not a separator).
 */
 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
   assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
   return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
 }
 
 static int fts5UnicodeTokenize(
   Fts5Tokenizer *pTokenizer,
   void *pCtx,
-  int flags,
+  int iUnused,
   const char *pText, int nText,
   int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
 ){
   Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
   int rc = SQLITE_OK;
   unsigned char *a = p->aTokenChar;
 
   unsigned char *zTerm = (unsigned char*)&pText[nText];
   unsigned char *zCsr = (unsigned char *)pText;
 
   /* Output buffer */
   char *aFold = p->aFold;
   int nFold = p->nFold;
   const char *pEnd = &aFold[nFold-6];
 
+  UNUSED_PARAM(iUnused);
+
   /* Each iteration of this loop gobbles up a contiguous run of separators,
   ** then the next token.  */
   while( rc==SQLITE_OK ){
     int iCode;                    /* non-ASCII codepoint read from input */
     char *zOut = aFold;
     int is;
     int ie;
 
     /* Skip any separator characters. */
     while( 1 ){
@@ skipping 798 matching lines @@
     fts5_tokenizer x;
   } aBuiltin[] = {
     { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
     { "ascii",     {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
     { "porter",    {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
   };
   
   int rc = SQLITE_OK;             /* Return code */
   int i;                          /* To iterate through builtin functions */
 
-  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
+  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
     rc = pApi->xCreateTokenizer(pApi,
         aBuiltin[i].zName,
         (void*)pApi,
         &aBuiltin[i].x,
         0
     );
   }
 
   return rc;
 }