Change sqlite fts2 tokenizer to recognize "foo*" as a prefix search, and not...

third_party/sqlite/fts2.c

 /* fts2 has a design flaw which can lead to database corruption (see
 ** below).  It is recommended not to use it any longer, instead use
 ** fts3 (or higher).  If you believe that your use of fts2 is safe,
 ** add -DSQLITE_ENABLE_BROKEN_FTS2=1 to your CFLAGS.
 */
 #if (!defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)) \
         && !defined(SQLITE_ENABLE_BROKEN_FTS2)
 #error fts2 has a design flaw and has been deprecated.
 #endif
 /* The flaw is that fts2 uses the content table's unaliased rowid as
@@ skipping 3520 matching lines @@
   sqlite3_tokenizer *pTokenizer,          /* The tokenizer to use */
   const char *pSegment, int nSegment,     /* Query expression being parsed */
   int inPhrase,                           /* True if within "..." */
   Query *pQuery                           /* Append results here */
 ){
   const sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
   sqlite3_tokenizer_cursor *pCursor;
   int firstIndex = pQuery->nTerms;
   int iCol;
   int nTerm = 1;
+  int iEndLast = -1;
   
   int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
   if( rc!=SQLITE_OK ) return rc;
   pCursor->pTokenizer = pTokenizer;
 
   while( 1 ){
     const char *pToken;
     int nToken, iBegin, iEnd, iPos;
 
     rc = pModule->xNext(pCursor,
                         &pToken, &nToken,
                         &iBegin, &iEnd, &iPos);
     if( rc!=SQLITE_OK ) break;
     if( !inPhrase &&
         pSegment[iEnd]==':' &&
          (iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){
       pQuery->nextColumn = iCol;
       continue;
     }
     if( !inPhrase && pQuery->nTerms>0 && nToken==2
          && pSegment[iBegin]=='O' && pSegment[iBegin+1]=='R' ){
       pQuery->nextIsOr = 1;
       continue;
     }
     
     // If a tokenizer recognizes "*" as a separate word, we mark the word
     // before it as a prefix search.  The other isPrefix code below that tests
     // for a word that's tokenized with the star as its last character.
-    // TODO(evanm): this is incorrect in that "foo *" parses as "foo*".
-    if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' ){
+    if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' &&
+         iEndLast==iBegin){

[Evan Martin, 2008/12/17 00:28:11]

tabbing off here?

is iEndLast supposed to be one past the end of the previous token?  i'm a little
surprised if this works otherwise.

[Matt Perry, 2008/12/17 00:29:58]

fixed.. though there doesn't seem to be a consistent way to indent here.
it seems that way.

       pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
       continue;
     }
-    
+    iEndLast = iEnd;
+
     queryAdd(pQuery, pToken, nToken);
     if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
       pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
     }
     if( iEnd<nSegment && pSegment[iEnd]=='*' ){
       pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
     }
     pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm;
     if( inPhrase ){
       nTerm++;
@@ skipping 3150 matching lines @@
   sqlite3 *db, 
   char **pzErrMsg,
   const sqlite3_api_routines *pApi
 ){
   SQLITE_EXTENSION_INIT2(pApi)
   return sqlite3Fts2Init(db);
 }
 #endif
 
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */