Import SQLite 3.10.2.

third_party/sqlite/src/ext/fts3/fts3_write.c

 /*
 ** 2009 Oct 23
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
@@ skipping 308 matching lines @@
 /* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
 /* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
 /* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
 /* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
 /* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=?",
 /* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(?,?)",
 /* 24 */  "",
 /* 25 */  "",
 
 /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
-/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",
+/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'",
 
 /* This statement is used to determine which level to read the input from
 ** when performing an incremental merge. It returns the absolute level number
 ** of the oldest level in the db that contains at least ? segments. Or,
 ** if no level in the FTS index contains more than ? segments, the statement
 ** returns zero rows.  */
 /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
          "  ORDER BY (level %% 1024) ASC LIMIT 1",
 
 /* Estimate the upper limit on the number of leaf nodes in a new segment
@@ skipping 513 matching lines @@
   return (rc==SQLITE_DONE ? SQLITE_OK : rc);
 }
 
 /* 
 ** Calling this function indicates that subsequent calls to 
 ** fts3PendingTermsAdd() are to add term/position-list pairs for the
 ** contents of the document with docid iDocid.
 */
 static int fts3PendingTermsDocid(
   Fts3Table *p,                   /* Full-text table handle */
+  int bDelete,                    /* True if this op is a delete */
   int iLangid,                    /* Language id of row being written */
   sqlite_int64 iDocid             /* Docid of row being written */
 ){
   assert( iLangid>=0 );
+  assert( bDelete==1 || bDelete==0 );
 
   /* TODO(shess) Explore whether partially flushing the buffer on
   ** forced-flush would provide better performance.  I suspect that if
   ** we ordered the doclists by size and flushed the largest until the
   ** buffer was half empty, that would let the less frequent terms
   ** generate longer doclists.
   */
-  if( iDocid<=p->iPrevDocid 
+  if( iDocid<p->iPrevDocid 
+   || (iDocid==p->iPrevDocid && p->bPrevDelete==0)
    || p->iPrevLangid!=iLangid
    || p->nPendingData>p->nMaxPendingData 
   ){
     int rc = sqlite3Fts3PendingTermsFlush(p);
     if( rc!=SQLITE_OK ) return rc;
   }
   p->iPrevDocid = iDocid;
   p->iPrevLangid = iLangid;
+  p->bPrevDelete = bDelete;
   return SQLITE_OK;
 }
 
 /*
 ** Discard the contents of the pending-terms hash tables. 
 */
 void sqlite3Fts3PendingTermsClear(Fts3Table *p){
   int i;
   for(i=0; i<p->nIndex; i++){
     Fts3HashElem *pElem;
@@ skipping 169 matching lines @@
   int rc;
   sqlite3_stmt *pSelect;
 
   assert( *pbFound==0 );
   if( *pRC ) return;
   rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
   if( rc==SQLITE_OK ){
     if( SQLITE_ROW==sqlite3_step(pSelect) ){
       int i;
       int iLangid = langidFromSelect(p, pSelect);
-      rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
+      i64 iDocid = sqlite3_column_int64(pSelect, 0);
+      rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid);
       for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
         int iCol = i-1;
         if( p->abNotindexed[iCol]==0 ){
           const char *zText = (const char *)sqlite3_column_text(pSelect, i);
           rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]);
           aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
         }
       }
       if( rc!=SQLITE_OK ){
         sqlite3_reset(pSelect);
@@ skipping 227 matching lines @@
   if( !pReader->aDoclist ){
     pNext = pReader->aNode;
   }else{
     pNext = &pReader->aDoclist[pReader->nDoclist];
   }
 
   if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
 
     if( fts3SegReaderIsPending(pReader) ){
       Fts3HashElem *pElem = *(pReader->ppNextElem);
-      if( pElem==0 ){
-        pReader->aNode = 0;
-      }else{
+      sqlite3_free(pReader->aNode);
+      pReader->aNode = 0;
+      if( pElem ){
+        char *aCopy;
         PendingList *pList = (PendingList *)fts3HashData(pElem);
+        int nCopy = pList->nData+1;
         pReader->zTerm = (char *)fts3HashKey(pElem);
         pReader->nTerm = fts3HashKeysize(pElem);
-        pReader->nNode = pReader->nDoclist = pList->nData + 1;
-        pReader->aNode = pReader->aDoclist = pList->aData;
+        aCopy = (char*)sqlite3_malloc(nCopy);
+        if( !aCopy ) return SQLITE_NOMEM;
+        memcpy(aCopy, pList->aData, nCopy);
+        pReader->nNode = pReader->nDoclist = nCopy;
+        pReader->aNode = pReader->aDoclist = aCopy;
         pReader->ppNextElem++;
         assert( pReader->aNode );
       }
       return SQLITE_OK;
     }
 
     fts3SegReaderSetEof(pReader);
 
     /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 
     ** blocks have already been traversed.  */
@@ skipping 219 matching lines @@
   }
   *pnOvfl = nOvfl;
   return rc;
 }
 
 /*
 ** Free all allocations associated with the iterator passed as the 
 ** second argument.
 */
 void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
-  if( pReader && !fts3SegReaderIsPending(pReader) ){
-    sqlite3_free(pReader->zTerm);
+  if( pReader ){
+    if( !fts3SegReaderIsPending(pReader) ){
+      sqlite3_free(pReader->zTerm);
+    }
     if( !fts3SegReaderIsRootOnly(pReader) ){
       sqlite3_free(pReader->aNode);
-      sqlite3_blob_close(pReader->pBlob);
     }
+    sqlite3_blob_close(pReader->pBlob);
   }
   sqlite3_free(pReader);
 }
 
 /*
 ** Allocate a new SegReader object.
 */
 int sqlite3Fts3SegReaderNew(
   int iAge,                       /* Segment "age". */
   int bLookup,                    /* True for a lookup only */
@@ skipping 35 matching lines @@
   }
   *ppReader = pReader;
   return SQLITE_OK;
 }
 
 /*
 ** This is a comparison function used as a qsort() callback when sorting
 ** an array of pending terms by term. This occurs as part of flushing
 ** the contents of the pending-terms hash table to the database.
 */
-static int fts3CompareElemByTerm(const void *lhs, const void *rhs){
+static int SQLITE_CDECL fts3CompareElemByTerm(
+  const void *lhs,
+  const void *rhs
+){
   char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
   char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
   int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
   int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
 
   int n = (n1<n2 ? n1 : n2);
   int c = memcmp(z1, z2, n);
   if( c==0 ){
     c = n1 - n2;
   }
@@ skipping 1436 matching lines @@
         ** take place.  */
         bOk = 0;
         break;
       }
       bOk = 1;
     }
     rc = sqlite3_reset(pRange);
 
     if( bOk ){
       int iIdx = 0;
-      sqlite3_stmt *pUpdate1;
-      sqlite3_stmt *pUpdate2;
+      sqlite3_stmt *pUpdate1 = 0;
+      sqlite3_stmt *pUpdate2 = 0;
 
       if( rc==SQLITE_OK ){
         rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0);
       }
       if( rc==SQLITE_OK ){
         rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0);
       }
 
       if( rc==SQLITE_OK ){
 
@@ skipping 337 matching lines @@
 ** iIndex/iLangid combination.
 */
 static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
   int bSeenDone = 0;
   int rc;
   sqlite3_stmt *pAllLangid = 0;
 
   rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
   if( rc==SQLITE_OK ){
     int rc2;
-    sqlite3_bind_int(pAllLangid, 1, p->nIndex);
+    sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
+    sqlite3_bind_int(pAllLangid, 2, p->nIndex);
     while( sqlite3_step(pAllLangid)==SQLITE_ROW ){
       int i;
       int iLangid = sqlite3_column_int(pAllLangid, 0);
       for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
         rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL);
         if( rc==SQLITE_DONE ){
           bSeenDone = 1;
           rc = SQLITE_OK;
         }
       }
@@ skipping 46 matching lines @@
       }else{
         memset(aSz, 0, nByte);
         aSzIns = &aSz[p->nColumn+1];
         aSzDel = &aSzIns[p->nColumn+1];
       }
     }
 
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
       int iCol;
       int iLangid = langidFromSelect(p, pStmt);
-      rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0));
+      rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0));
       memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1));
       for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
         if( p->abNotindexed[iCol]==0 ){
           const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
           rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]);
           aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
         }
       }
       if( p->bHasDocsize ){
         fts3InsertDocsize(&rc, p, aSz);
@@ skipping 1244 matching lines @@
   const int nHint = pHint->n;
   int i;
 
   i = pHint->n-2;
   while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
   while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
 
   pHint->n = i;
   i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
   i += fts3GetVarint32(&pHint->a[i], pnInput);
-  if( i!=nHint ) return SQLITE_CORRUPT_VTAB;
+  if( i!=nHint ) return FTS_CORRUPT_VTAB;
 
   return SQLITE_OK;
 }
 
 
 /*
 ** Attempt an incremental merge that writes nMerge leaf blocks.
 **
 ** Incremental merges happen nMin segments at a time. The segments 
 ** to be merged are the nMin oldest segments (the ones with the smallest 
@@ skipping 347 matching lines @@
 static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
   int rc = SQLITE_OK;             /* Return code */
   u64 cksum1 = 0;                 /* Checksum based on FTS index contents */
   u64 cksum2 = 0;                 /* Checksum based on %_content contents */
   sqlite3_stmt *pAllLangid = 0;   /* Statement to return all language-ids */
 
   /* This block calculates the checksum according to the FTS index. */
   rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
   if( rc==SQLITE_OK ){
     int rc2;
-    sqlite3_bind_int(pAllLangid, 1, p->nIndex);
+    sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
+    sqlite3_bind_int(pAllLangid, 2, p->nIndex);
     while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
       int iLangid = sqlite3_column_int(pAllLangid, 0);
       int i;
       for(i=0; i<p->nIndex; i++){
         cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc);
       }
     }
     rc2 = sqlite3_reset(pAllLangid);
     if( rc==SQLITE_OK ) rc = rc2;
   }
 
   /* This block calculates the checksum according to the %_content table */
-  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
   if( rc==SQLITE_OK ){
     sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
     sqlite3_stmt *pStmt = 0;
     char *zSql;
    
     zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
     if( !zSql ){
       rc = SQLITE_NOMEM;
     }else{
       rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
@@ skipping 76 matching lines @@
 **
 ** If the two checksums are identical, the integrity-check is deemed to have
 ** passed.
 */
 static int fts3DoIntegrityCheck(
   Fts3Table *p                    /* FTS3 table handle */
 ){
   int rc;
   int bOk = 0;
   rc = fts3IntegrityCheck(p, &bOk);
-  if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
+  if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
   return rc;
 }
 
 /*
 ** Handle a 'special' INSERT of the form:
 **
 **   "INSERT INTO tbl(tbl) VALUES(<expr>)"
 **
 ** Argument pVal contains the result of <expr>. Currently the only 
 ** meaningful value to insert is the text 'optimize'.
@@ skipping 341 matching lines @@
   /* If this is an INSERT or UPDATE operation, insert the new record. */
   if( nArg>1 && rc==SQLITE_OK ){
     int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
     if( bInsertDone==0 ){
       rc = fts3InsertData(p, apVal, pRowid);
       if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
         rc = FTS_CORRUPT_VTAB;
       }
     }
     if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
-      rc = fts3PendingTermsDocid(p, iLangid, *pRowid);
+      rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
     }
     if( rc==SQLITE_OK ){
       assert( p->iPrevDocid==*pRowid );
       rc = fts3InsertTerms(p, iLangid, apVal, aSzIns);
     }
     if( p->bHasDocsize ){
       fts3InsertDocsize(&rc, p, aSzIns);
     }
     nChng++;
   }
@@ skipping 24 matching lines @@
     }else{
       sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
       sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
     }
   }
   sqlite3Fts3SegmentsClose(p);
   return rc;
 }
 
 #endif