Change sqlite fts2 tokenizer to recognize "foo*" as a prefix search, and not...

third_party/sqlite/fts2.c

Index: third_party/sqlite/fts2.c
===================================================================
--- third_party/sqlite/fts2.c	(revision 7052)
+++ third_party/sqlite/fts2.c	(working copy)
@@ -3538,6 +3538,7 @@
   int firstIndex = pQuery->nTerms;
   int iCol;
   int nTerm = 1;
+  int iEndLast = -1;
   
   int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
   if( rc!=SQLITE_OK ) return rc;
@@ -3566,12 +3567,13 @@
     // If a tokenizer recognizes "*" as a separate word, we mark the word
     // before it as a prefix search.  The other isPrefix code below that tests
     // for a word that's tokenized with the star as its last character.
-    // TODO(evanm): this is incorrect in that "foo *" parses as "foo*".
-    if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' ){
+    if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' &&
+         iEndLast==iBegin){

[Evan Martin, 2008/12/17 00:28:11]

tabbing off here?

is iEndLast supposed to be one past the end of the previous token?  i'm a little
surprised if this works otherwise.

[Matt Perry, 2008/12/17 00:29:58]

fixed.. though there doesn't seem to be a consistent way to indent here.
it seems that way.

       pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
       continue;
     }
-    
+    iEndLast = iEnd;
+
     queryAdd(pQuery, pToken, nToken);
     if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
       pQuery->pTerms[pQuery->nTerms-1].isNot = 1;