Import SQLite 3.7.6.3.

third_party/sqlite/src/ext/fts3/fts3_porter.c

 /*
 ** 2006 September 30
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
 ** Implementation of the full-text-search tokenizer that implements
 ** a Porter stemmer.
 */
 
 /*
 ** The code in this file is only compiled if:
 **
 **     * The FTS3 module is being built as an extension
 **       (in which case SQLITE_CORE is not defined), or
 **
 **     * The FTS3 module is being built into the core of
 **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
 */
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
+#include "fts3Int.h"
 
 #include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 
 #include "fts3_tokenizer.h"
 
 /*
 ** Class derived from sqlite3_tokenizer
 */
 typedef struct porter_tokenizer {
   sqlite3_tokenizer base;      /* Base class */
 } porter_tokenizer;
 
 /*
 ** Class derived from sqlit3_tokenizer_cursor
 */
 typedef struct porter_tokenizer_cursor {
   sqlite3_tokenizer_cursor base;
   const char *zInput;          /* input we are tokenizing */
   int nInput;                  /* size of the input */
   int iOffset;                 /* current position in zInput */
   int iToken;                  /* index of next token to be returned */
   char *zToken;                /* storage for current token */
   int nAllocated;              /* space allocated to zToken buffer */
 } porter_tokenizer_cursor;
 
 
-/* Forward declaration */
-static const sqlite3_tokenizer_module porterTokenizerModule;
-
-
 /*
 ** Create a new tokenizer instance.
 */
 static int porterCreate(
   int argc, const char * const *argv,
   sqlite3_tokenizer **ppTokenizer
 ){
   porter_tokenizer *t;
+
+  UNUSED_PARAMETER(argc);
+  UNUSED_PARAMETER(argv);
+
   t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
   if( t==NULL ) return SQLITE_NOMEM;
   memset(t, 0, sizeof(*t));
   *ppTokenizer = &t->base;
   return SQLITE_OK;
 }
 
 /*
 ** Destroy a tokenizer
 */
 static int porterDestroy(sqlite3_tokenizer *pTokenizer){
   sqlite3_free(pTokenizer);
   return SQLITE_OK;
 }
 
 /*
 ** Prepare to begin tokenizing a particular string.  The input
 ** string to be tokenized is zInput[0..nInput-1].  A cursor
 ** used to incrementally tokenize this string is returned in 
 ** *ppCursor.
 */
 static int porterOpen(
   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
   const char *zInput, int nInput,        /* String to be tokenized */
   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
 ){
   porter_tokenizer_cursor *c;
 
+  UNUSED_PARAMETER(pTokenizer);
+
   c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
   if( c==NULL ) return SQLITE_NOMEM;
 
   c->zInput = zInput;
   if( zInput==0 ){
     c->nInput = 0;
   }else if( nInput<0 ){
     c->nInput = (int)strlen(zInput);
   }else{
     c->nInput = nInput;
@@ skipping 120 matching lines @@
   return *z!=0;
 }
 
 /*
 ** Return TRUE if the word ends in a double consonant.
 **
 ** The text is reversed here. So we are really looking at
 ** the first two characters of z[].
 */
 static int doubleConsonant(const char *z){
-  return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+  return isConsonant(z) && z[0]==z[1];
 }
 
 /*
 ** Return TRUE if the word ends with three letters which
 ** are consonant-vowel-consonent and where the final consonant
 ** is not 'w', 'x', or 'y'.
 **
 ** The word is reversed here.  So we are really checking the
 ** first three letters and the first one cannot be in [wxy].
 */
 static int star_oh(const char *z){
   return
-    z[0]!=0 && isConsonant(z) &&
+    isConsonant(z) &&
     z[0]!='w' && z[0]!='x' && z[0]!='y' &&
-    z[1]!=0 && isVowel(z+1) &&
-    z[2]!=0 && isConsonant(z+2);
+    isVowel(z+1) &&
+    isConsonant(z+2);
 }
 
 /*
 ** If the word ends with zFrom and xCond() is true for the stem
 ** of the word that preceeds the zFrom ending, then change the 
 ** ending to zTo.
 **
 ** The input word *pz and zFrom are both in reverse order.  zTo
 ** is in normal order. 
 **
@@ skipping 23 matching lines @@
 ** inappropriate.  The input word is copied into the output with
 ** US-ASCII case folding.  If the input word is too long (more
 ** than 20 bytes if it contains no digits or more than 6 bytes if
 ** it contains digits) then word is truncated to 20 or 6 bytes
 ** by taking 10 or 3 bytes from the beginning and end.
 */
 static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
   int i, mx, j;
   int hasDigit = 0;
   for(i=0; i<nIn; i++){
-    int c = zIn[i];
+    char c = zIn[i];
     if( c>='A' && c<='Z' ){
       zOut[i] = c - 'A' + 'a';
     }else{
       if( c>='0' && c<='9' ) hasDigit = 1;
       zOut[i] = c;
     }
   }
   mx = hasDigit ? 3 : 10;
   if( nIn>mx*2 ){
     for(j=mx, i=nIn-mx; i<nIn; i++, j++){
@@ skipping 23 matching lines @@
 ** 
 ** If the input word contains not digits but does characters not 
 ** in [a-zA-Z] then no stemming is attempted and this routine just 
 ** copies the input into the input into the output with US-ASCII
 ** case folding.
 **
 ** Stemming never increases the length of the word.  So there is
 ** no chance of overflowing the zOut buffer.
 */
 static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
-  int i, j, c;
+  int i, j;
   char zReverse[28];
   char *z, *z2;
-  if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+  if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){
     /* The word is too big or too small for the porter stemmer.
     ** Fallback to the copy stemmer */
     copy_stemmer(zIn, nIn, zOut, pnOut);
     return;
   }
   for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
-    c = zIn[i];
+    char c = zIn[i];
     if( c>='A' && c<='Z' ){
       zReverse[j] = c + 'a' - 'A';
     }else if( c>='a' && c<='z' ){
       zReverse[j] = c;
     }else{
       /* The use of a character not in [a-zA-Z] means that we fallback
       ** to the copy stemmer */
       copy_stemmer(zIn, nIn, zOut, pnOut);
       return;
     }
@@ skipping 178 matching lines @@
   }
 
   /* Step 5b */
   if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
     z++;
   }
 
   /* z[] is now the stemmed word in reverse order.  Flip it back
   ** around into forward order and return.
   */
-  *pnOut = i = strlen(z);
+  *pnOut = i = (int)strlen(z);
   zOut[i] = 0;
   while( *z ){
     zOut[--i] = *(z++);
   }
 }
 
 /*
 ** Characters that can be part of a token.  We assume any character
 ** whose value is greater than 0x80 (any UTF character) can be
 ** part of a token.  In other words, delimiters all must have
@@ skipping 34 matching lines @@
 
     /* Count non-delimiter characters. */
     iStartOffset = c->iOffset;
     while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
       c->iOffset++;
     }
 
     if( c->iOffset>iStartOffset ){
       int n = c->iOffset-iStartOffset;
       if( n>c->nAllocated ){
+        char *pNew;
         c->nAllocated = n+20;
-        c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
-        if( c->zToken==NULL ) return SQLITE_NOMEM;
+        pNew = sqlite3_realloc(c->zToken, c->nAllocated);
+        if( !pNew ) return SQLITE_NOMEM;
+        c->zToken = pNew;
       }
       porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
       *pzToken = c->zToken;
       *piStartOffset = iStartOffset;
       *piEndOffset = c->iOffset;
       *piPosition = c->iToken++;
       return SQLITE_OK;
     }
   }
   return SQLITE_DONE;
@@ skipping 15 matching lines @@
 ** Allocate a new porter tokenizer.  Return a pointer to the new
 ** tokenizer in *ppModule
 */
 void sqlite3Fts3PorterTokenizerModule(
   sqlite3_tokenizer_module const**ppModule
 ){
   *ppModule = &porterTokenizerModule;
 }
 
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */