Import SQLite 3.8.7.4.

third_party/sqlite/src/ext/fts3/fts3_snippet.c

 /*
 ** 2009 Oct 23
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 ******************************************************************************
 */
 
+#include "fts3Int.h"
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
-#include "fts3Int.h"
 #include <string.h>
 #include <assert.h>
 
 /*
 ** Characters that may appear in the second argument to matchinfo().
 */
 #define FTS3_MATCHINFO_NPHRASE   'p'        /* 1 value */
 #define FTS3_MATCHINFO_NCOL      'c'        /* 1 value */
 #define FTS3_MATCHINFO_NDOC      'n'        /* 1 value */
 #define FTS3_MATCHINFO_AVGLENGTH 'a'        /* nCol values */
@@ skipping 94 matching lines @@
 **
 ** are encoded.
 **
 ** When this function is called, *pp points to the start of an element of
 ** the list. *piPos contains the value of the previous entry in the list.
 ** After it returns, *piPos contains the value of the next element of the
 ** list and *pp is advanced to the following varint.
 */
 static void fts3GetDeltaPosition(char **pp, int *piPos){
   int iVal;
-  *pp += sqlite3Fts3GetVarint32(*pp, &iVal);
+  *pp += fts3GetVarint32(*pp, &iVal);
   *piPos += (iVal-2);
 }
 
 /*
 ** Helper function for fts3ExprIterate() (see below).
 */
 static int fts3ExprIterate2(
   Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   int *piPhrase,                  /* Pointer to phrase counter */
   int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
@@ skipping 28 matching lines @@
 static int fts3ExprIterate(
   Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   void *pCtx                      /* Second argument to pass to callback */
 ){
   int iPhrase = 0;                /* Variable used as the phrase counter */
   return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
 }
 
 /*
-** The argument to this function is always a phrase node. Its doclist 
-** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
-** to the left of this one in the query tree have already been loaded.
-**
-** If this phrase node is part of a series of phrase nodes joined by 
-** NEAR operators (and is not the left-most of said series), then elements are
-** removed from the phrases doclist consistent with the NEAR restriction. If
-** required, elements may be removed from the doclists of phrases to the
-** left of this one that are part of the same series of NEAR operator 
-** connected phrases.
-**
-** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
-*/
-static int fts3ExprNearTrim(Fts3Expr *pExpr){
-  int rc = SQLITE_OK;
-  Fts3Expr *pParent = pExpr->pParent;
-
-  assert( pExpr->eType==FTSQUERY_PHRASE );
-  while( rc==SQLITE_OK
-   && pParent 
-   && pParent->eType==FTSQUERY_NEAR 
-   && pParent->pRight==pExpr 
-  ){
-    /* This expression (pExpr) is the right-hand-side of a NEAR operator. 
-    ** Find the expression to the left of the same operator.
-    */
-    int nNear = pParent->nNear;
-    Fts3Expr *pLeft = pParent->pLeft;
-
-    if( pLeft->eType!=FTSQUERY_PHRASE ){
-      assert( pLeft->eType==FTSQUERY_NEAR );
-      assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
-      pLeft = pLeft->pRight;
-    }
-
-    rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
-
-    pExpr = pLeft;
-    pParent = pExpr->pParent;
-  }
-
-  return rc;
-}
-
-/*
 ** This is an fts3ExprIterate() callback used while loading the doclists
 ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
 ** fts3ExprLoadDoclists().
 */
 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   int rc = SQLITE_OK;
+  Fts3Phrase *pPhrase = pExpr->pPhrase;
   LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
 
   UNUSED_PARAMETER(iPhrase);
 
   p->nPhrase++;
-  p->nToken += pExpr->pPhrase->nToken;
-
-  if( pExpr->isLoaded==0 ){
-    rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
-    pExpr->isLoaded = 1;
-    if( rc==SQLITE_OK ){
-      rc = fts3ExprNearTrim(pExpr);
-    }
-  }
+  p->nToken += pPhrase->nToken;
 
   return rc;
 }
 
 /*
 ** Load the doclists for each phrase in the query associated with FTS3 cursor
 ** pCsr. 
 **
 ** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable 
 ** phrases in the expression (all phrases except those directly or 
@@ skipping 150 matching lines @@
 }
 
 /*
 ** This function is an fts3ExprIterate() callback used by fts3BestSnippet().
 ** Each invocation populates an element of the SnippetIter.aPhrase[] array.
 */
 static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
   SnippetIter *p = (SnippetIter *)ctx;
   SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
   char *pCsr;
+  int rc;
 
   pPhrase->nToken = pExpr->pPhrase->nToken;
-
-  pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol);
+  rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr);
+  assert( rc==SQLITE_OK || pCsr==0 );
   if( pCsr ){
     int iFirst = 0;
     pPhrase->pList = pCsr;
     fts3GetDeltaPosition(&pCsr, &iFirst);
+    assert( iFirst>=0 );
     pPhrase->pHead = pCsr;
     pPhrase->pTail = pCsr;
     pPhrase->iHead = iFirst;
     pPhrase->iTail = iFirst;
   }else{
-    assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
+    assert( rc!=SQLITE_OK || (
+       pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 
+    ));
   }
 
-  return SQLITE_OK;
+  return rc;
 }
 
 /*
 ** Select the fragment of text consisting of nFragment contiguous tokens 
 ** from column iCol that represent the "best" snippet. The best snippet
 ** is the snippet with the highest score, where scores are calculated
 ** by adding:
 **
-**   (a) +1 point for each occurence of a matchable phrase in the snippet.
+**   (a) +1 point for each occurrence of a matchable phrase in the snippet.
 **
-**   (b) +1000 points for the first occurence of each matchable phrase in 
+**   (b) +1000 points for the first occurrence of each matchable phrase in 
 **       the snippet for which the corresponding mCovered bit is not set.
 **
 ** The selected snippet parameters are stored in structure *pFragment before
 ** returning. The score of the selected snippet is stored in *piScore
 ** before returning.
 */
 static int fts3BestSnippet(
   int nSnippet,                   /* Desired snippet length */
   Fts3Cursor *pCsr,               /* Cursor to create snippet for */
   int iCol,                       /* Index of column to create snippet from */
@@ skipping 92 matching lines @@
   */
   if( pStr->n+nAppend+1>=pStr->nAlloc ){
     int nAlloc = pStr->nAlloc+nAppend+100;
     char *zNew = sqlite3_realloc(pStr->z, nAlloc);
     if( !zNew ){
       return SQLITE_NOMEM;
     }
     pStr->z = zNew;
     pStr->nAlloc = nAlloc;
   }
+  assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) );
 
   /* Append the data to the string buffer. */
   memcpy(&pStr->z[pStr->n], zAppend, nAppend);
   pStr->n += nAppend;
   pStr->z[pStr->n] = '\0';
 
   return SQLITE_OK;
 }
 
 /*
@@ skipping 11 matching lines @@
 **
 **     ....X.....X....
 **
 ** This is done as part of extracting the snippet text, not when selecting
 ** the snippet. Snippet selection is done based on doclists only, so there
 ** is no way for fts3BestSnippet() to know whether or not the document 
 ** actually contains terms that follow the final highlighted term. 
 */
 static int fts3SnippetShift(
   Fts3Table *pTab,                /* FTS3 table snippet comes from */
+  int iLangid,                    /* Language id to use in tokenizing */
   int nSnippet,                   /* Number of tokens desired for snippet */
   const char *zDoc,               /* Document text to extract snippet from */
   int nDoc,                       /* Size of buffer zDoc in bytes */
   int *piPos,                     /* IN/OUT: First token of snippet */
   u64 *pHlmask                    /* IN/OUT: Mask of tokens to highlight */
 ){
   u64 hlmask = *pHlmask;          /* Local copy of initial highlight-mask */
 
   if( hlmask ){
     int nLeft;                    /* Tokens to the left of first highlight */
@@ skipping 15 matching lines @@
       int nShift;                 /* Number of tokens to shift snippet by */
       int iCurrent = 0;           /* Token counter */
       int rc;                     /* Return Code */
       sqlite3_tokenizer_module *pMod;
       sqlite3_tokenizer_cursor *pC;
       pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
 
       /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired)
       ** or more tokens in zDoc/nDoc.
       */
-      rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+      rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC);
       if( rc!=SQLITE_OK ){
         return rc;
       }
-      pC->pTokenizer = pTab->pTokenizer;
       while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){
-        const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3;
+        const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0;
         rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
       }
       pMod->xClose(pC);
       if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; }
 
       nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet;
       assert( nShift<=nDesired );
       if( nShift>0 ){
         *piPos += nShift;
         *pHlmask = hlmask >> nShift;
@@ skipping 23 matching lines @@
   const char *zDoc;               /* Document text to extract snippet from */
   int nDoc;                       /* Size of zDoc in bytes */
   int iCurrent = 0;               /* Current token number of document */
   int iEnd = 0;                   /* Byte offset of end of current token */
   int isShiftDone = 0;            /* True after snippet is shifted */
   int iPos = pFragment->iPos;     /* First token of snippet */
   u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */
   int iCol = pFragment->iCol+1;   /* Query column to extract text from */
   sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */
   sqlite3_tokenizer_cursor *pC;   /* Tokenizer cursor open on zDoc/nDoc */
-  const char *ZDUMMY;             /* Dummy argument used with tokenizer */
-  int DUMMY1;                     /* Dummy argument used with tokenizer */
   
   zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol);
   if( zDoc==0 ){
     if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){
       return SQLITE_NOMEM;
     }
     return SQLITE_OK;
   }
   nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol);
 
   /* Open a token cursor on the document. */
   pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
-  rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+  rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC);
   if( rc!=SQLITE_OK ){
     return rc;
   }
-  pC->pTokenizer = pTab->pTokenizer;
 
   while( rc==SQLITE_OK ){
-    int iBegin;                   /* Offset in zDoc of start of token */
-    int iFin;                     /* Offset in zDoc of end of token */
-    int isHighlight;              /* True for highlighted terms */
+    const char *ZDUMMY;           /* Dummy argument used with tokenizer */
+    int DUMMY1 = -1;              /* Dummy argument used with tokenizer */
+    int iBegin = 0;               /* Offset in zDoc of start of token */
+    int iFin = 0;                 /* Offset in zDoc of end of token */
+    int isHighlight = 0;          /* True for highlighted terms */
 
+    /* Variable DUMMY1 is initialized to a negative value above. Elsewhere
+    ** in the FTS code the variable that the third argument to xNext points to
+    ** is initialized to zero before the first (*but not necessarily
+    ** subsequent*) call to xNext(). This is done for a particular application
+    ** that needs to know whether or not the tokenizer is being used for
+    ** snippet generation or for some other purpose.
+    **
+    ** Extreme care is required when writing code to depend on this
+    ** initialization. It is not a documented part of the tokenizer interface.
+    ** If a tokenizer is used directly by any code outside of FTS, this
+    ** convention might not be respected.  */
     rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent);
     if( rc!=SQLITE_OK ){
       if( rc==SQLITE_DONE ){
         /* Special case - the last token of the snippet is also the last token
         ** of the column. Append any punctuation that occurred between the end
         ** of the previous token and the end of the document to the output. 
         ** Then break out of the loop. */
         rc = fts3StringAppend(pOut, &zDoc[iEnd], -1);
       }
       break;
     }
     if( iCurrent<iPos ){ continue; }
 
     if( !isShiftDone ){
       int n = nDoc - iBegin;
-      rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask);
+      rc = fts3SnippetShift(
+          pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask
+      );
       isShiftDone = 1;
 
       /* Now that the shift has been done, check if the initial "..." are
       ** required. They are required if (a) this is not the first fragment,
       ** or (b) this fragment does not begin at position 0 of its column. 
       */
       if( rc==SQLITE_OK && (iPos>0 || iFragment>0) ){
         rc = fts3StringAppend(pOut, zEllipsis, -1);
       }
       if( rc!=SQLITE_OK || iCurrent<iPos ) continue;
@@ skipping 43 matching lines @@
   /* A column-list is terminated by either a 0x01 or 0x00. */
   while( 0xFE & (*pEnd | c) ){
     c = *pEnd++ & 0x80;
     if( !c ) nEntry++;
   }
 
   *ppCollist = pEnd;
   return nEntry;
 }
 
-static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
-  char *pCsr = *pp;
-  while( *pCsr ){
-    int nHit;
-    sqlite3_int64 iCol = 0;
-    if( *pCsr==0x01 ){
-      pCsr++;
-      pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
-    }
-    nHit = fts3ColumnlistCount(&pCsr);
-    assert( nHit>0 );
-    if( isGlobal ){
-      aOut[iCol*3+1]++;
-    }
-    aOut[iCol*3] += nHit;
-  }
-  pCsr++;
-  *pp = pCsr;
-}
-
 /*
 ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
 ** for a single query. 
 **
 ** fts3ExprIterate() callback to load the 'global' elements of a
 ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
 ** of the matchinfo array that are constant for all rows returned by the 
 ** current query.
 **
 ** Argument pCtx is actually a pointer to a struct of type MatchInfo. This
@@ skipping 13 matching lines @@
 ** file system. This is done because the full-text index doclist is required
 ** to calculate these values properly, and the full-text index doclist is
 ** not available for deferred tokens.
 */
 static int fts3ExprGlobalHitsCb(
   Fts3Expr *pExpr,                /* Phrase expression node */
   int iPhrase,                    /* Phrase number (numbered from zero) */
   void *pCtx                      /* Pointer to MatchInfo structure */
 ){
   MatchInfo *p = (MatchInfo *)pCtx;
-  Fts3Cursor *pCsr = p->pCursor;
-  char *pIter;
-  char *pEnd;
-  char *pFree = 0;
-  u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
-
-  assert( pExpr->isLoaded );
-  assert( pExpr->eType==FTSQUERY_PHRASE );
-
-  if( pCsr->pDeferred ){
-    Fts3Phrase *pPhrase = pExpr->pPhrase;
-    int ii;
-    for(ii=0; ii<pPhrase->nToken; ii++){
-      if( pPhrase->aToken[ii].bFulltext ) break;
-    }
-    if( ii<pPhrase->nToken ){
-      int nFree = 0;
-      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
-      if( rc!=SQLITE_OK ) return rc;
-      pIter = pFree;
-      pEnd = &pFree[nFree];
-    }else{
-      int iCol;                   /* Column index */
-      for(iCol=0; iCol<p->nCol; iCol++){
-        aOut[iCol*3 + 1] = (u32)p->nDoc;
-        aOut[iCol*3 + 2] = (u32)p->nDoc;
-      }
-      return SQLITE_OK;
-    }
-  }else{
-    pIter = pExpr->aDoclist;
-    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
-  }
-
-  /* Fill in the global hit count matrix row for this phrase. */
-  while( pIter<pEnd ){
-    while( *pIter++ & 0x80 );      /* Skip past docid. */
-    fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
-  }
-
-  sqlite3_free(pFree);
-  return SQLITE_OK;
+  return sqlite3Fts3EvalPhraseStats(
+      p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
+  );
 }
 
 /*
 ** fts3ExprIterate() callback used to collect the "local" part of the
 ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the 
 ** array that are different for each row returned by the query.
 */
 static int fts3ExprLocalHitsCb(
   Fts3Expr *pExpr,                /* Phrase expression node */
   int iPhrase,                    /* Phrase number */
   void *pCtx                      /* Pointer to MatchInfo structure */
 ){
+  int rc = SQLITE_OK;
   MatchInfo *p = (MatchInfo *)pCtx;
   int iStart = iPhrase * p->nCol * 3;
   int i;
 
-  for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
-
-  if( pExpr->aDoclist ){
+  for(i=0; i<p->nCol && rc==SQLITE_OK; i++){
     char *pCsr;
-
-    pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1);
+    rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr);
     if( pCsr ){
-      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
+      p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
+    }else{
+      p->aMatchinfo[iStart+i*3] = 0;
     }
   }
 
-  return SQLITE_OK;
+  return rc;
 }
 
 static int fts3MatchinfoCheck(
   Fts3Table *pTab, 
   char cArg,
   char **pzErr
 ){
   if( (cArg==FTS3_MATCHINFO_NPHRASE)
    || (cArg==FTS3_MATCHINFO_NCOL)
-   || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
-   || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
+   || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4)
+   || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4)
    || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
    || (cArg==FTS3_MATCHINFO_LCS)
    || (cArg==FTS3_MATCHINFO_HITS)
   ){
     return SQLITE_OK;
   }
   *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
   return SQLITE_ERROR;
 }
 
@@ skipping 34 matching lines @@
 
   if( !*ppStmt ){
     int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
     if( rc!=SQLITE_OK ) return rc;
   }
   pStmt = *ppStmt;
   assert( sqlite3_data_count(pStmt)==1 );
 
   a = sqlite3_column_blob(pStmt, 0);
   a += sqlite3Fts3GetVarint(a, &nDoc);
-  if( nDoc==0 ) return SQLITE_CORRUPT;
+  if( nDoc==0 ) return FTS_CORRUPT_VTAB;
   *pnDoc = (u32)nDoc;
 
   if( paLen ) *paLen = a;
   return SQLITE_OK;
 }
 
 /*
 ** An instance of the following structure is used to store state while 
 ** iterating through a multi-column position-list corresponding to the
 ** hits for a single phrase on a single row in order to calculate the
 ** values for a matchinfo() FTS3_MATCHINFO_LCS request.
 */
 typedef struct LcsIterator LcsIterator;
 struct LcsIterator {
   Fts3Expr *pExpr;                /* Pointer to phrase expression */
+  int iPosOffset;                 /* Tokens count up to end of this phrase */
   char *pRead;                    /* Cursor used to iterate through aDoclist */
-  int iPosOffset;                 /* Tokens count up to end of this phrase */
-  int iCol;                       /* Current column number */
   int iPos;                       /* Current position */
 };
 
 /* 
 ** If LcsIterator.iCol is set to the following value, the iterator has
 ** finished iterating through all offsets for all columns.
 */
 #define LCS_ITERATOR_FINISHED 0x7FFFFFFF;
 
 static int fts3MatchinfoLcsCb(
@@ skipping 10 matching lines @@
 ** Advance the iterator passed as an argument to the next position. Return
 ** 1 if the iterator is at EOF or if it now points to the start of the
 ** position list for the next column.
 */
 static int fts3LcsIteratorAdvance(LcsIterator *pIter){
   char *pRead = pIter->pRead;
   sqlite3_int64 iRead;
   int rc = 0;
 
   pRead += sqlite3Fts3GetVarint(pRead, &iRead);
-  if( iRead==0 ){
-    pIter->iCol = LCS_ITERATOR_FINISHED;
+  if( iRead==0 || iRead==1 ){
+    pRead = 0;
     rc = 1;
   }else{
-    if( iRead==1 ){
-      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
-      pIter->iCol = (int)iRead;
-      pIter->iPos = pIter->iPosOffset;
-      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
-      rc = 1;
-    }
     pIter->iPos += (int)(iRead-2);
   }
 
   pIter->pRead = pRead;
   return rc;
 }
   
 /*
 ** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. 
 **
@@ skipping 11 matching lines @@
   int iCol;
   int nToken = 0;
 
   /* Allocate and populate the array of LcsIterator objects. The array
   ** contains one element for each matchable phrase in the query.
   **/
   aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
   if( !aIter ) return SQLITE_NOMEM;
   memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
   (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
+
   for(i=0; i<pInfo->nPhrase; i++){
     LcsIterator *pIter = &aIter[i];
     nToken -= pIter->pExpr->pPhrase->nToken;
     pIter->iPosOffset = nToken;
-    pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1);
-    if( pIter->pRead ){
-      pIter->iPos = pIter->iPosOffset;
-      fts3LcsIteratorAdvance(&aIter[i]);
-    }else{
-      pIter->iCol = LCS_ITERATOR_FINISHED;
-    }
   }
 
   for(iCol=0; iCol<pInfo->nCol; iCol++){
     int nLcs = 0;                 /* LCS value for this column */
     int nLive = 0;                /* Number of iterators in aIter not at EOF */
 
-    /* Loop through the iterators in aIter[]. Set nLive to the number of
-    ** iterators that point to a position-list corresponding to column iCol.
-    */
     for(i=0; i<pInfo->nPhrase; i++){
-      assert( aIter[i].iCol>=iCol );
-      if( aIter[i].iCol==iCol ) nLive++;
+      int rc;
+      LcsIterator *pIt = &aIter[i];
+      rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead);
+      if( rc!=SQLITE_OK ) return rc;
+      if( pIt->pRead ){
+        pIt->iPos = pIt->iPosOffset;
+        fts3LcsIteratorAdvance(&aIter[i]);
+        nLive++;
+      }
     }
 
-    /* The following loop runs until all iterators in aIter[] have finished
-    ** iterating through positions in column iCol. Exactly one of the 
-    ** iterators is advanced each time the body of the loop is run.
-    */
     while( nLive>0 ){
       LcsIterator *pAdv = 0;      /* The iterator to advance by one position */
       int nThisLcs = 0;           /* LCS for the current iterator positions */
 
       for(i=0; i<pInfo->nPhrase; i++){
         LcsIterator *pIter = &aIter[i];
-        if( iCol!=pIter->iCol ){  
+        if( pIter->pRead==0 ){
           /* This iterator is already at EOF for this column. */
           nThisLcs = 0;
         }else{
           if( pAdv==0 || pIter->iPos<pAdv->iPos ){
             pAdv = pIter;
           }
           if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
             nThisLcs++;
           }else{
             nThisLcs = 1;
@@ skipping 45 matching lines @@
       case FTS3_MATCHINFO_NPHRASE:
         if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
         break;
 
       case FTS3_MATCHINFO_NCOL:
         if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
         break;
         
       case FTS3_MATCHINFO_NDOC:
         if( bGlobal ){
-          sqlite3_int64 nDoc;
+          sqlite3_int64 nDoc = 0;
           rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
           pInfo->aMatchinfo[0] = (u32)nDoc;
         }
         break;
 
       case FTS3_MATCHINFO_AVGLENGTH: 
         if( bGlobal ){
           sqlite3_int64 nDoc;     /* Number of rows in table */
           const char *a;          /* Aggregate column length array */
 
@@ skipping 235 matching lines @@
 typedef struct TermOffset TermOffset;
 typedef struct TermOffsetCtx TermOffsetCtx;
 
 struct TermOffset {
   char *pList;                    /* Position-list */
   int iPos;                       /* Position just read from pList */
   int iOff;                       /* Offset of this term from read positions */
 };
 
 struct TermOffsetCtx {
+  Fts3Cursor *pCsr;
   int iCol;                       /* Column of table to populate aTerm for */
   int iTerm;
   sqlite3_int64 iDocid;
   TermOffset *aTerm;
 };
 
 /*
 ** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
 */
 static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
   TermOffsetCtx *p = (TermOffsetCtx *)ctx;
   int nTerm;                      /* Number of tokens in phrase */
   int iTerm;                      /* For looping through nTerm phrase terms */
   char *pList;                    /* Pointer to position list for phrase */
   int iPos = 0;                   /* First position in position-list */
+  int rc;
 
   UNUSED_PARAMETER(iPhrase);
-  pList = sqlite3Fts3FindPositions(pExpr, p->iDocid, p->iCol);
+  rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList);
   nTerm = pExpr->pPhrase->nToken;
   if( pList ){
     fts3GetDeltaPosition(&pList, &iPos);
     assert( iPos>=0 );
   }
 
   for(iTerm=0; iTerm<nTerm; iTerm++){
     TermOffset *pT = &p->aTerm[p->iTerm++];
     pT->iOff = nTerm-iTerm-1;
     pT->pList = pList;
     pT->iPos = iPos;
   }
 
-  return SQLITE_OK;
+  return rc;
 }
 
 /*
 ** Implementation of offsets() function.
 */
 void sqlite3Fts3Offsets(
   sqlite3_context *pCtx,          /* SQLite function call context */
   Fts3Cursor *pCsr                /* Cursor object */
 ){
   Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
   sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule;
-  const char *ZDUMMY;             /* Dummy argument used with xNext() */
-  int NDUMMY;                     /* Dummy argument used with xNext() */
   int rc;                         /* Return Code */
   int nToken;                     /* Number of tokens in query */
   int iCol;                       /* Column currently being processed */
   StrBuffer res = {0, 0, 0};      /* Result string */
   TermOffsetCtx sCtx;             /* Context for fts3ExprTermOffsetInit() */
 
   if( !pCsr->pExpr ){
     sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
     return;
   }
 
   memset(&sCtx, 0, sizeof(sCtx));
   assert( pCsr->isRequireSeek==0 );
 
   /* Count the number of terms in the query */
   rc = fts3ExprLoadDoclists(pCsr, 0, &nToken);
   if( rc!=SQLITE_OK ) goto offsets_out;
 
   /* Allocate the array of TermOffset iterators. */
   sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken);
   if( 0==sCtx.aTerm ){
     rc = SQLITE_NOMEM;
     goto offsets_out;
   }
   sCtx.iDocid = pCsr->iPrevId;
+  sCtx.pCsr = pCsr;
 
   /* Loop through the table columns, appending offset information to 
   ** string-buffer res for each column.
   */
   for(iCol=0; iCol<pTab->nColumn; iCol++){
     sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */
-    int iStart;
-    int iEnd;
-    int iCurrent;
+    const char *ZDUMMY;           /* Dummy argument used with xNext() */
+    int NDUMMY = 0;               /* Dummy argument used with xNext() */
+    int iStart = 0;
+    int iEnd = 0;
+    int iCurrent = 0;
     const char *zDoc;
     int nDoc;
 
     /* Initialize the contents of sCtx.aTerm[] for column iCol. There is 
     ** no way that this operation can fail, so the return code from
     ** fts3ExprIterate() can be discarded.
     */
     sCtx.iCol = iCol;
     sCtx.iTerm = 0;
     (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
 
     /* Retreive the text stored in column iCol. If an SQL NULL is stored 
     ** in column iCol, jump immediately to the next iteration of the loop.
     ** If an OOM occurs while retrieving the data (this can happen if SQLite
     ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 
     ** to the caller. 
     */
     zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
     nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
     if( zDoc==0 ){
       if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){
         continue;
       }
       rc = SQLITE_NOMEM;
       goto offsets_out;
     }
 
     /* Initialize a tokenizer iterator to iterate through column iCol. */
-    rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+    rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid,
+        zDoc, nDoc, &pC
+    );
     if( rc!=SQLITE_OK ) goto offsets_out;
-    pC->pTokenizer = pTab->pTokenizer;
 
     rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
     while( rc==SQLITE_OK ){
       int i;                      /* Used to loop through terms */
       int iMinPos = 0x7FFFFFFF;   /* Position of next token */
       TermOffset *pTerm = 0;      /* TermOffset associated with next token */
 
       for(i=0; i<nToken; i++){
         TermOffset *pT = &sCtx.aTerm[i];
         if( pT->pList && (pT->iPos-pT->iOff)<iMinPos ){
           iMinPos = pT->iPos-pT->iOff;
           pTerm = pT;
         }
       }
 
       if( !pTerm ){
         /* All offsets for this column have been gathered. */
-        break;
+        rc = SQLITE_DONE;
       }else{
         assert( iCurrent<=iMinPos );
         if( 0==(0xFE&*pTerm->pList) ){
           pTerm->pList = 0;
         }else{
           fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos);
         }
         while( rc==SQLITE_OK && iCurrent<iMinPos ){
           rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
         }
         if( rc==SQLITE_OK ){
           char aBuffer[64];
           sqlite3_snprintf(sizeof(aBuffer), aBuffer, 
               "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
           );
           rc = fts3StringAppend(&res, aBuffer, -1);
-        }else if( rc==SQLITE_DONE ){
-          rc = SQLITE_CORRUPT;
+        }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){
+          rc = FTS_CORRUPT_VTAB;
         }
       }
     }
     if( rc==SQLITE_DONE ){
       rc = SQLITE_OK;
     }
 
     pMod->xClose(pC);
     if( rc!=SQLITE_OK ) goto offsets_out;
   }
@@ skipping 49 matching lines @@
 
   if( rc!=SQLITE_OK ){
     sqlite3_result_error_code(pContext, rc);
   }else{
     int n = pCsr->nMatchinfo * sizeof(u32);
     sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
   }
 }
 
 #endif