Import SQLite 3.7.6.3.

third_party/sqlite/src/ext/fts2/fts2_tokenizer1.c

 /*
 ** 2006 Oct 10
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
@@ skipping 11 matching lines @@
 **     * The FTS2 module is being built into the core of
 **       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
 */
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
 
 
 #include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
 
 #include "fts2_tokenizer.h"
 
 typedef struct simple_tokenizer {
   sqlite3_tokenizer base;
   char delim[128];             /* flag ASCII delimiters */
 } simple_tokenizer;
 
 typedef struct simple_tokenizer_cursor {
   sqlite3_tokenizer_cursor base;
@@ skipping 39 matching lines @@
       if( ch>=0x80 ){
         sqlite3_free(t);
         return SQLITE_ERROR;
       }
       t->delim[ch] = 1;
     }
   } else {
     /* Mark non-alphanumeric ASCII characters as delimiters */
     int i;
     for(i=1; i<0x80; i++){
-      t->delim[i] = !isalnum(i);
+      t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
+                      (i>='a' && i<='z'));
     }
   }
 
   *ppTokenizer = &t->base;
   return SQLITE_OK;
 }
 
 /*
 ** Destroy a tokenizer
 */
@@ skipping 81 matching lines @@
       if( n>c->nTokenAllocated ){
         c->nTokenAllocated = n+20;
         c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
         if( c->pToken==NULL ) return SQLITE_NOMEM;
       }
       for(i=0; i<n; i++){
         /* TODO(shess) This needs expansion to handle UTF-8
         ** case-insensitivity.
         */
         unsigned char ch = p[iStartOffset+i];
-        c->pToken[i] = (ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch;
+        c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
       }
       *ppToken = c->pToken;
       *pnBytes = n;
       *piStartOffset = iStartOffset;
       *piEndOffset = c->iOffset;
       *piPosition = c->iToken++;
 
       return SQLITE_OK;
     }
   }
@@ skipping 16 matching lines @@
 ** Allocate a new simple tokenizer.  Return a pointer to the new
 ** tokenizer in *ppModule
 */
 void sqlite3Fts2SimpleTokenizerModule(
   sqlite3_tokenizer_module const**ppModule
 ){
   *ppModule = &simpleTokenizerModule;
 }
 
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */