[sql] Rewrite sqlite patching "system".

third_party/sqlite/patches/0023-fts2-Fix-numerous-out-of-bounds-bugs-reading-corrupt.patch

-diff -ru ext-orig/fts2/fts2.c ext/fts2/fts2.c
---- ext-orig/fts2/fts2.c        2009-09-04 13:37:41.000000000 -0700
-+++ ext/fts2/fts2.c     2009-09-30 14:48:14.000000000 -0700
-@@ -37,6 +37,20 @@
- ** This is an SQLite module implementing full-text search.
- */
- 
-+/* TODO(shess): To make it easier to spot changes without groveling
-+** through changelogs, I've defined GEARS_FTS2_CHANGES to call them
-+** out, and I will document them here.  On imports, these changes
-+** should be reviewed to make sure they are still present, or are
-+** dropped as appropriate.
-+**
-+** SQLite core adds the custom function fts2_tokenizer() to be used
-+** for defining new tokenizers.  The second parameter is a vtable
-+** pointer encoded as a blob.  Obviously this cannot be exposed to
-+** Gears callers for security reasons.  It could be suppressed in the
-+** authorizer, but for now I have simply commented the definition out.
-+*/
-+#define GEARS_FTS2_CHANGES 1
-+
- /*
- ** The code in this file is only compiled if:
- **
-@@ -326,8 +326,10 @@
- #include "fts2_hash.h"
- #include "fts2_tokenizer.h"
- #include "sqlite3.h"
--#include "sqlite3ext.h"
--SQLITE_EXTENSION_INIT1
-+#ifndef SQLITE_CORE 
-+# include "sqlite3ext.h"
-+  SQLITE_EXTENSION_INIT1
-+#endif
- 
- 
- /* TODO(shess) MAN, this thing needs some refactoring.  At minimum, it
-@@ -335,6 +349,16 @@
- # define TRACE(A)
- #endif
- 
-+#if 0
-+/* Useful to set breakpoints.  See main.c sqlite3Corrupt(). */
-+static int fts2Corrupt(void){
-+  return SQLITE_CORRUPT;
-+}
-+# define SQLITE_CORRUPT_BKPT fts2Corrupt()
-+#else
-+# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT
-+#endif
-+
- /* It is not safe to call isspace(), tolower(), or isalnum() on
- ** hi-bit-set characters.  This is the same solution used in the
- ** tokenizer.
-@@ -423,30 +447,41 @@
+From 106d71238a58d4dfbeb8cf1cba45a1c4e6f583e8 Mon Sep 17 00:00:00 2001
+From: Chris Evans <cevans@chromium.org>
+Date: Wed, 30 Sep 2009 23:10:34 +0000
+Subject: [PATCH 23/23] [fts2] Fix numerous out-of-bounds bugs reading corrupt
+ database.
+
+Fix numerous bugs in fts2 where a corrupt fts2 database could cause
+out-of-bounds reads and writes.
+
+Original review URL is more descriptive:
+http://codereview.chromium.org/216026
+---
+ third_party/sqlite/src/ext/fts2/fts2.c | 751 ++++++++++++++++++++++-----------
+ 1 file changed, 514 insertions(+), 237 deletions(-)
+
+diff --git a/third_party/sqlite/src/ext/fts2/fts2.c b/third_party/sqlite/src/ext/fts2/fts2.c
+index d5587b3..36d14ff 100644
+--- a/third_party/sqlite/src/ext/fts2/fts2.c
++++ b/third_party/sqlite/src/ext/fts2/fts2.c
+@@ -447,30 +447,41 @@ static int putVarint(char *p, sqlite_int64 v){
  /* Read a 64-bit variable-length integer from memory starting at p[0].
   * Return the number of bytes read, or 0 on error.
   * The value is stored in *v. */
 -static int getVarint(const char *p, sqlite_int64 *v){
 +static int getVarintSafe(const char *p, sqlite_int64 *v, int max){
    const unsigned char *q = (const unsigned char *) p;
    sqlite_uint64 x = 0, y = 1;
 -  while( (*q & 0x80) == 0x80 ){
 +  if( max>VARINT_MAX ) max = VARINT_MAX;
 +  while( max && (*q & 0x80) == 0x80 ){
@@ skipping 29 matching lines @@
   return ret;
  }
  
 +static int getVarint32(const char* p, int *pi){
 +  return getVarint32Safe(p, pi, VARINT_MAX);
 +}
 +
  /*******************************************************************/
  /* DataBuffer is used to collect data into a buffer in piecemeal
  ** fashion.  It implements the usual distinction between amount of
-@@ -615,7 +650,7 @@
+@@ -639,7 +650,7 @@ typedef struct DLReader {
  
  static int dlrAtEnd(DLReader *pReader){
    assert( pReader->nData>=0 );
 -  return pReader->nData==0;
 +  return pReader->nData<=0;
  }
  static sqlite_int64 dlrDocid(DLReader *pReader){
    assert( !dlrAtEnd(pReader) );
-@@ -639,7 +674,8 @@
+@@ -663,7 +674,8 @@ static int dlrAllDataBytes(DLReader *pReader){
  */
  static const char *dlrPosData(DLReader *pReader){
    sqlite_int64 iDummy;
 -  int n = getVarint(pReader->pData, &iDummy);
 +  int n = getVarintSafe(pReader->pData, &iDummy, pReader->nElement);
 +  if( !n ) return NULL;
    assert( !dlrAtEnd(pReader) );
    return pReader->pData+n;
  }
-@@ -649,7 +685,7 @@
+@@ -673,7 +685,7 @@ static int dlrPosDataLen(DLReader *pReader){
    assert( !dlrAtEnd(pReader) );
    return pReader->nElement-n;
  }
 -static void dlrStep(DLReader *pReader){
 +static int dlrStep(DLReader *pReader){
    assert( !dlrAtEnd(pReader) );
  
    /* Skip past current doclist element. */
-@@ -658,32 +694,48 @@
+@@ -682,32 +694,48 @@ static void dlrStep(DLReader *pReader){
    pReader->nData -= pReader->nElement;
  
    /* If there is more data, read the next doclist element. */
 -  if( pReader->nData!=0 ){
 +  if( pReader->nData>0 ){
      sqlite_int64 iDocidDelta;
 -    int iDummy, n = getVarint(pReader->pData, &iDocidDelta);
 +    int nTotal = 0;
 +    int iDummy, n = getVarintSafe(pReader->pData, &iDocidDelta, pReader->nData);
 +    if( !n ) return SQLITE_CORRUPT_BKPT;
@@ skipping 41 matching lines @@
 -                    const char *pData, int nData){
 +static void dlrDestroy(DLReader *pReader){
 +  SCRAMBLE(pReader);
 +}
 +static int dlrInit(DLReader *pReader, DocListType iType,
 +                   const char *pData, int nData){
 +  int rc;
    assert( pData!=NULL && nData!=0 );
    pReader->iType = iType;
    pReader->pData = pData;
-@@ -692,10 +744,9 @@
+@@ -716,10 +744,9 @@ static void dlrInit(DLReader *pReader, DocListType iType,
    pReader->iDocid = 0;
  
    /* Load the first element's data.  There must be a first element. */
 -  dlrStep(pReader);
 -}
 -static void dlrDestroy(DLReader *pReader){
 -  SCRAMBLE(pReader);
 +  rc = dlrStep(pReader);
 +  if( rc!=SQLITE_OK ) dlrDestroy(pReader);
 +  return rc;
  }
  
  #ifndef NDEBUG
-@@ -782,9 +833,9 @@
+@@ -806,9 +833,9 @@ static void dlwDestroy(DLWriter *pWriter){
  /* TODO(shess) This has become just a helper for docListMerge.
  ** Consider a refactor to make this cleaner.
  */
 -static void dlwAppend(DLWriter *pWriter,
 -                      const char *pData, int nData,
 -                      sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
 +static int dlwAppend(DLWriter *pWriter,
 +                     const char *pData, int nData,
 +                     sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
    sqlite_int64 iDocid = 0;
    char c[VARINT_MAX];
    int nFirstOld, nFirstNew;     /* Old and new varint len of first docid. */
-@@ -793,7 +844,8 @@
+@@ -817,7 +844,8 @@ static void dlwAppend(DLWriter *pWriter,
  #endif
  
    /* Recode the initial docid as delta from iPrevDocid. */
 -  nFirstOld = getVarint(pData, &iDocid);
 +  nFirstOld = getVarintSafe(pData, &iDocid, nData);
 +  if( !nFirstOld ) return SQLITE_CORRUPT_BKPT;
    assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
    nFirstNew = putVarint(c, iFirstDocid-pWriter->iPrevDocid);
  
-@@ -814,10 +866,11 @@
+@@ -838,10 +866,11 @@ static void dlwAppend(DLWriter *pWriter,
      dataBufferAppend(pWriter->b, c, nFirstNew);
    }
    pWriter->iPrevDocid = iLastDocid;
 +  return SQLITE_OK;
  }
 -static void dlwCopy(DLWriter *pWriter, DLReader *pReader){
 -  dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
 -            dlrDocid(pReader), dlrDocid(pReader));
 +static int dlwCopy(DLWriter *pWriter, DLReader *pReader){
 +  return dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
 +                   dlrDocid(pReader), dlrDocid(pReader));
  }
  static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
    char c[VARINT_MAX];
-@@ -878,45 +931,63 @@
+@@ -902,45 +931,63 @@ static int plrEndOffset(PLReader *pReader){
    assert( !plrAtEnd(pReader) );
    return pReader->iEndOffset;
  }
 -static void plrStep(PLReader *pReader){
 -  int i, n;
 +static int plrStep(PLReader *pReader){
 +  int i, n, nTotal = 0;
  
    assert( !plrAtEnd(pReader) );
  
@@ skipping 57 matching lines @@
 -static void plrInit(PLReader *pReader, DLReader *pDLReader){
 +static void plrDestroy(PLReader *pReader){
 +  SCRAMBLE(pReader);
 +}
 +
 +static int plrInit(PLReader *pReader, DLReader *pDLReader){
 +  int rc;
    pReader->pData = dlrPosData(pDLReader);
    pReader->nData = dlrPosDataLen(pDLReader);
    pReader->iType = pDLReader->iType;
-@@ -924,10 +995,9 @@
+@@ -948,10 +995,9 @@ static void plrInit(PLReader *pReader, DLReader *pDLReader){
    pReader->iPosition = 0;
    pReader->iStartOffset = 0;
    pReader->iEndOffset = 0;
 -  plrStep(pReader);
 -}
 -static void plrDestroy(PLReader *pReader){
 -  SCRAMBLE(pReader);
 +  rc = plrStep(pReader);
 +  if( rc!=SQLITE_OK ) plrDestroy(pReader);
 +  return rc;
  }
  
  /*******************************************************************/
-@@ -1113,14 +1183,16 @@
+@@ -1137,14 +1183,16 @@ static void dlcDelete(DLCollector *pCollector){
  ** deletion will be trimmed, and will thus not effect a deletion
  ** during the merge.
  */
 -static void docListTrim(DocListType iType, const char *pData, int nData,
 -                        int iColumn, DocListType iOutType, DataBuffer *out){
 +static int docListTrim(DocListType iType, const char *pData, int nData,
 +                       int iColumn, DocListType iOutType, DataBuffer *out){
    DLReader dlReader;
    DLWriter dlWriter;
 +  int rc;
  
    assert( iOutType<=iType );
  
 -  dlrInit(&dlReader, iType, pData, nData);
 +  rc = dlrInit(&dlReader, iType, pData, nData);
 +  if( rc!=SQLITE_OK ) return rc;
    dlwInit(&dlWriter, iOutType, out);
  
    while( !dlrAtEnd(&dlReader) ){
-@@ -1128,7 +1200,8 @@
+@@ -1152,7 +1200,8 @@ static void docListTrim(DocListType iType, const char *pData, int nData,
      PLWriter plWriter;
      int match = 0;
  
 -    plrInit(&plReader, &dlReader);
 +    rc = plrInit(&plReader, &dlReader);
 +    if( rc!=SQLITE_OK ) break;
  
      while( !plrAtEnd(&plReader) ){
        if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
-@@ -1139,7 +1212,11 @@
+@@ -1163,7 +1212,11 @@ static void docListTrim(DocListType iType, const char *pData, int nData,
          plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
                 plrStartOffset(&plReader), plrEndOffset(&plReader));
        }
 -      plrStep(&plReader);
 +      rc = plrStep(&plReader);
 +      if( rc!=SQLITE_OK ){
 +        plrDestroy(&plReader);
 +        goto err;
 +      }
      }
      if( match ){
        plwTerminate(&plWriter);
-@@ -1147,10 +1224,13 @@
+@@ -1171,10 +1224,13 @@ static void docListTrim(DocListType iType, const char *pData, int nData,
      }
  
      plrDestroy(&plReader);
 -    dlrStep(&dlReader);
 +    rc = dlrStep(&dlReader);
 +    if( rc!=SQLITE_OK ) break;
    }
 +err:
    dlwDestroy(&dlWriter);
    dlrDestroy(&dlReader);
 +  return rc;
  }
  
  /* Used by docListMerge() to keep doclists in the ascending order by
-@@ -1207,19 +1287,20 @@
+@@ -1231,19 +1287,20 @@ static void orderedDLReaderReorder(OrderedDLReader *p, int n){
  /* TODO(shess) nReaders must be <= MERGE_COUNT.  This should probably
  ** be fixed.
  */
 -static void docListMerge(DataBuffer *out,
 -                         DLReader *pReaders, int nReaders){
 +static int docListMerge(DataBuffer *out,
 +                        DLReader *pReaders, int nReaders){
    OrderedDLReader readers[MERGE_COUNT];
    DLWriter writer;
    int i, n;
    const char *pStart = 0;
    int nStart = 0;
    sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
 +  int rc = SQLITE_OK;
  
    assert( nReaders>0 );
    if( nReaders==1 ){
      dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
 -    return;
 +    return SQLITE_OK;
    }
  
    assert( nReaders<=MERGE_COUNT );
-@@ -1252,20 +1333,23 @@
+@@ -1276,20 +1333,23 @@ static void docListMerge(DataBuffer *out,
        nStart += dlrDocDataBytes(readers[0].pReader);
      }else{
        if( pStart!=0 ){
 -        dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
 +        rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
 +        if( rc!=SQLITE_OK ) goto err;
        }
        pStart = dlrDocData(readers[0].pReader);
        nStart = dlrDocDataBytes(readers[0].pReader);
        iFirstDocid = iDocid;
      }
      iLastDocid = iDocid;
 -    dlrStep(readers[0].pReader);
 +    rc = dlrStep(readers[0].pReader);
 +    if( rc!=SQLITE_OK ) goto err;
  
      /* Drop all of the older elements with the same docid. */
      for(i=1; i<nReaders &&
               !dlrAtEnd(readers[i].pReader) &&
               dlrDocid(readers[i].pReader)==iDocid; i++){
 -      dlrStep(readers[i].pReader);
 +      rc = dlrStep(readers[i].pReader);
 +      if( rc!=SQLITE_OK ) goto err;
      }
  
      /* Get the readers back into order. */
-@@ -1275,8 +1359,11 @@
+@@ -1299,8 +1359,11 @@ static void docListMerge(DataBuffer *out,
    }
  
    /* Copy over any remaining elements. */
 -  if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
 +  if( nStart>0 )
 +    rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
 +err:
    dlwDestroy(&writer);
 +  return rc;
  }
  
  /* Helper function for posListUnion().  Compares the current position
-@@ -1312,30 +1399,40 @@
+@@ -1336,30 +1399,40 @@ static int posListCmp(PLReader *pLeft, PLReader *pRight){
  ** work with any doclist type, though both inputs and the output
  ** should be the same type.
  */
 -static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
 +static int posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
    PLReader left, right;
    PLWriter writer;
 +  int rc;
  
    assert( dlrDocid(pLeft)==dlrDocid(pRight) );
@@ skipping 27 matching lines @@
        plwCopy(&writer, &left);
 -      plrStep(&left);
 -      plrStep(&right);
 +      rc = plrStep(&left);
 +      if( rc != SQLITE_OK ) break;
 +      rc = plrStep(&right);
 +      if( rc != SQLITE_OK ) break;
      }
    }
  
-@@ -1343,56 +1440,75 @@
+@@ -1367,56 +1440,75 @@ static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
    plwDestroy(&writer);
    plrDestroy(&left);
    plrDestroy(&right);
 +  return rc;
  }
  
  /* Write the union of doclists in pLeft and pRight to pOut.  For
  ** docids in common between the inputs, the union of the position
  ** lists is written.  Inputs and outputs are always type DL_DEFAULT.
  */
@@ skipping 71 matching lines @@
      }
    }
  
    dlrDestroy(&left);
    dlrDestroy(&right);
    dlwDestroy(&writer);
 +  return rc;
  }
  
  /* pLeft and pRight are DLReaders positioned to the same docid.
-@@ -1407,35 +1523,47 @@
+@@ -1431,35 +1523,47 @@ static void docListUnion(
  ** include the positions from pRight that are one more than a
  ** position in pLeft.  In other words:  pRight.iPos==pLeft.iPos+1.
  */
 -static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
 -                               DLWriter *pOut){
 +static int posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
 +                              DLWriter *pOut){
    PLReader left, right;
    PLWriter writer;
    int match = 0;
@@ skipping 37 matching lines @@
        plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
 -      plrStep(&left);
 -      plrStep(&right);
 +      rc = plrStep(&left);
 +      if( rc!=SQLITE_OK ) break;
 +      rc = plrStep(&right);
 +      if( rc!=SQLITE_OK ) break;
      }
    }
  
-@@ -1446,6 +1574,7 @@
+@@ -1470,6 +1574,7 @@ static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
  
    plrDestroy(&left);
    plrDestroy(&right);
 +  return rc;
  }
  
  /* We have two doclists with positions:  pLeft and pRight.
-@@ -1457,7 +1586,7 @@
+@@ -1481,7 +1586,7 @@ static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
  ** iType controls the type of data written to pOut.  If iType is
  ** DL_POSITIONS, the positions are those from pRight.
  */
 -static void docListPhraseMerge(
 +static int docListPhraseMerge(
    const char *pLeft, int nLeft,
    const char *pRight, int nRight,
    DocListType iType,
-@@ -1465,152 +1594,198 @@
+@@ -1489,152 +1594,198 @@ static void docListPhraseMerge(
  ){
    DLReader left, right;
    DLWriter writer;
 +  int rc;
  
 -  if( nLeft==0 || nRight==0 ) return;
 +  if( nLeft==0 || nRight==0 ) return SQLITE_OK;
  
    assert( iType!=DL_POSITIONS_OFFSETS );
  
@@ skipping 212 matching lines @@
    }
  
 +err:
    dlrDestroy(&left);
    dlrDestroy(&right);
    dlwDestroy(&writer);
 +  return rc;
  }
  
  static char *string_dup_n(const char *s, int n){
-@@ -1814,7 +1989,7 @@
-   /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
-   /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
-   /* SEGDIR_SELECT_LEVEL */
--  "select start_block, leaves_end_block, root from %_segdir "
-+  "select start_block, leaves_end_block, root, idx from %_segdir "
-   " where level = ? order by idx",
-   /* SEGDIR_SPAN */
-   "select min(start_block), max(end_block) from %_segdir "
-@@ -3413,7 +3588,8 @@
+@@ -3437,7 +3588,8 @@ static int fulltextNext(sqlite3_vtab_cursor *pCursor){
        return SQLITE_OK;
      }
      rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
 -    dlrStep(&c->reader);
 +    if( rc!=SQLITE_OK ) return rc;
 +    rc = dlrStep(&c->reader);
      if( rc!=SQLITE_OK ) return rc;
      /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
      rc = sqlite3_step(c->pStmt);
-@@ -3421,8 +3597,11 @@
-       c->eof = 0;
-       return SQLITE_OK;
-     }
--    /* an error occurred; abort */
--    return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
-+
-+    /* Corrupt if the index refers to missing document. */
-+    if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
-+
-+    return rc;
-   }
- }
- 
-@@ -3470,14 +3649,18 @@
+@@ -3497,14 +3649,18 @@ static int docListOfTerm(
        return rc;
      }
      dataBufferInit(&new, 0);
 -    docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
 -                       i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new);
 +    rc = docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
 +                            i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new);
      dataBufferDestroy(&left);
      dataBufferDestroy(&right);
 +    if( rc!=SQLITE_OK ){
 +      dataBufferDestroy(&new);
 +      return rc;
 +    }
      left = new;
    }
    *pResult = left;
 -  return SQLITE_OK;
 +  return rc;
  }
  
  /* Add a new term pTerm[0..nTerm-1] to the query *q.
-@@ -3544,6 +3727,7 @@
-   int firstIndex = pQuery->nTerms;
-   int iCol;
-   int nTerm = 1;
-+  int iEndLast = -1;
-   
-   int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
-   if( rc!=SQLITE_OK ) return rc;
-@@ -3568,6 +3752,20 @@
-       pQuery->nextIsOr = 1;
-       continue;
-     }
-+
-+    /*
-+     * The ICU tokenizer considers '*' a break character, so the code below
-+     * sets isPrefix correctly, but since that code doesn't eat the '*', the
-+     * ICU tokenizer returns it as the next token.  So eat it here until a
-+     * better solution presents itself.
-+     */
-+    if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' &&
-+        iEndLast==iBegin){
-+      pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
-+      continue;
-+    }
-+    iEndLast = iEnd;
-+    
-     queryAdd(pQuery, pToken, nToken);
-     if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
-       pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
-@@ -3707,18 +3905,30 @@
+@@ -3749,18 +3905,30 @@ static int fulltextQuery(
          return rc;
        }
        dataBufferInit(&new, 0);
 -      docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
 +      rc = docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
        dataBufferDestroy(&right);
        dataBufferDestroy(&or);
 +      if( rc!=SQLITE_OK ){
 +        if( i!=nNot ) dataBufferDestroy(&left);
 +        queryClear(pQuery);
@@ skipping 12 matching lines @@
        dataBufferDestroy(&right);
        dataBufferDestroy(&left);
 +      if( rc!=SQLITE_OK ){
 +        queryClear(pQuery);
 +        dataBufferDestroy(&new);
 +        return rc;
 +      }
        left = new;
      }
    }
-@@ -3738,9 +3948,15 @@
+@@ -3780,9 +3948,15 @@ static int fulltextQuery(
        return rc;
      }
      dataBufferInit(&new, 0);
 -    docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new);
 +    rc = docListExceptMerge(left.pData, left.nData,
 +                            right.pData, right.nData, &new);
      dataBufferDestroy(&right);
      dataBufferDestroy(&left);
 +    if( rc!=SQLITE_OK ){
 +      queryClear(pQuery);
 +      dataBufferDestroy(&new);
 +      return rc;
 +    }
      left = new;
    }
  
-@@ -3834,7 +4050,8 @@
+@@ -3876,7 +4050,8 @@ static int fulltextFilter(
        rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q);
        if( rc!=SQLITE_OK ) return rc;
        if( c->result.nData!=0 ){
 -        dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
 +        rc = dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
 +        if( rc!=SQLITE_OK ) return rc;
        }
        break;
      }
-@@ -4335,22 +4552,19 @@
+@@ -4377,22 +4552,19 @@ static void interiorReaderDestroy(InteriorReader *pReader){
    SCRAMBLE(pReader);
  }
  
 -/* TODO(shess) The assertions are great, but what if we're in NDEBUG
 -** and the blob is empty or otherwise contains suspect data?
 -*/
 -static void interiorReaderInit(const char *pData, int nData,
 -                               InteriorReader *pReader){
 +static int interiorReaderInit(const char *pData, int nData,
 +                              InteriorReader *pReader){
    int n, nTerm;
  
 -  /* Require at least the leading flag byte */
 +  /* These conditions are checked and met by the callers. */
    assert( nData>0 );
    assert( pData[0]!='\0' );
  
    CLEAR(pReader);
  
    /* Decode the base blockid, and set the cursor to the first term. */
 -  n = getVarint(pData+1, &pReader->iBlockid);
 -  assert( 1+n<=nData );
 +  n = getVarintSafe(pData+1, &pReader->iBlockid, nData-1);
 +  if( !n ) return SQLITE_CORRUPT_BKPT;
    pReader->pData = pData+1+n;
    pReader->nData = nData-(1+n);
  
-@@ -4361,17 +4575,18 @@
+@@ -4403,17 +4575,18 @@ static void interiorReaderInit(const char *pData, int nData,
    if( pReader->nData==0 ){
      dataBufferInit(&pReader->term, 0);
    }else{
 -    n = getVarint32(pReader->pData, &nTerm);
 +    n = getVarint32Safe(pReader->pData, &nTerm, pReader->nData);
 +    if( !n || nTerm<0 || nTerm>pReader->nData-n) return SQLITE_CORRUPT_BKPT;
      dataBufferInit(&pReader->term, nTerm);
      dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
 -    assert( n+nTerm<=pReader->nData );
      pReader->pData += n+nTerm;
      pReader->nData -= n+nTerm;
    }
 +  return SQLITE_OK;
  }
  
  static int interiorReaderAtEnd(InteriorReader *pReader){
 -  return pReader->term.nData==0;
 +  return pReader->term.nData<=0;
  }
  
  static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
-@@ -4388,7 +4603,7 @@
+@@ -4430,7 +4603,7 @@ static const char *interiorReaderTerm(InteriorReader *pReader){
  }
  
  /* Step forward to the next term in the node. */
 -static void interiorReaderStep(InteriorReader *pReader){
 +static int interiorReaderStep(InteriorReader *pReader){
    assert( !interiorReaderAtEnd(pReader) );
  
    /* If the last term has been read, signal eof, else construct the
-@@ -4399,18 +4614,26 @@
+@@ -4441,18 +4614,26 @@ static void interiorReaderStep(InteriorReader *pReader){
    }else{
      int n, nPrefix, nSuffix;
  
 -    n = getVarint32(pReader->pData, &nPrefix);
 -    n += getVarint32(pReader->pData+n, &nSuffix);
 +    n = getVarint32Safe(pReader->pData, &nPrefix, pReader->nData);
 +    if( !n ) return SQLITE_CORRUPT_BKPT;
 +    pReader->nData -= n;
 +    pReader->pData += n;
 +    n = getVarint32Safe(pReader->pData, &nSuffix, pReader->nData);
@@ skipping 12 matching lines @@
 -    pReader->pData += n+nSuffix;
 -    pReader->nData -= n+nSuffix;
 +    pReader->pData += nSuffix;
 +    pReader->nData -= nSuffix;
    }
    pReader->iBlockid++;
 +  return SQLITE_OK;
  }
  
  /* Compare the current term to pTerm[nTerm], returning strcmp-style
-@@ -4782,7 +5005,8 @@
+@@ -4824,7 +5005,8 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
    n = putVarint(c, nData);
    dataBufferAppend(&pWriter->data, c, n);
  
 -  docListMerge(&pWriter->data, pReaders, nReaders);
 +  rc = docListMerge(&pWriter->data, pReaders, nReaders);
 +  if( rc!= SQLITE_OK ) return rc;
    ASSERT_VALID_DOCLIST(DL_DEFAULT,
                         pWriter->data.pData+iDoclistData+n,
                         pWriter->data.nData-iDoclistData-n, NULL);
-@@ -4892,7 +5116,8 @@
+@@ -4934,7 +5116,8 @@ static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
    int rc;
    DLReader reader;
  
 -  dlrInit(&reader, DL_DEFAULT, pData, nData);
 +  rc = dlrInit(&reader, DL_DEFAULT, pData, nData);
 +  if( rc!=SQLITE_OK ) return rc;
    rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
    dlrDestroy(&reader);
  
-@@ -4937,38 +5162,40 @@
+@@ -4979,38 +5162,40 @@ static int leafReaderDataBytes(LeafReader *pReader){
  static const char *leafReaderData(LeafReader *pReader){
    int n, nData;
    assert( pReader->term.nData>0 );
 -  n = getVarint32(pReader->pData, &nData);
 +  n = getVarint32Safe(pReader->pData, &nData, pReader->nData);
 +  if( !n || nData>pReader->nData-n ) return NULL;
    return pReader->pData+n;
  }
  
 -static void leafReaderInit(const char *pData, int nData,
@@ skipping 28 matching lines @@
    assert( !leafReaderAtEnd(pReader) );
  
    /* Skip previous entry's data block. */
 -  n = getVarint32(pReader->pData, &nData);
 -  assert( n+nData<=pReader->nData );
 +  n = getVarint32Safe(pReader->pData, &nData, pReader->nData);
 +  if( !n || nData<0 || nData>pReader->nData-n ) return SQLITE_CORRUPT_BKPT;
    pReader->pData += n+nData;
    pReader->nData -= n+nData;
  
-@@ -4976,15 +5203,23 @@
+@@ -5018,15 +5203,23 @@ static void leafReaderStep(LeafReader *pReader){
      /* Construct the new term using a prefix from the old term plus a
      ** suffix from the leaf data.
      */
 -    n = getVarint32(pReader->pData, &nPrefix);
 -    n += getVarint32(pReader->pData+n, &nSuffix);
 -    assert( n+nSuffix<pReader->nData );
 +    n = getVarint32Safe(pReader->pData, &nPrefix, pReader->nData);
 +    if( !n ) return SQLITE_CORRUPT_BKPT;
 +    pReader->nData -= n;
 +    pReader->pData += n;
 +    n = getVarint32Safe(pReader->pData, &nSuffix, pReader->nData);
 +    if( !n ) return SQLITE_CORRUPT_BKPT;
 +    pReader->nData -= n;
 +    pReader->pData += n;
 +    if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT;
 +    if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT;
      pReader->term.nData = nPrefix;
 -    dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
 +    dataBufferAppend(&pReader->term, pReader->pData, nSuffix);
  
 -    pReader->pData += n+nSuffix;
 -    pReader->nData -= n+nSuffix;
 +    pReader->pData += nSuffix;
 +    pReader->nData -= nSuffix;
    }
 +  return SQLITE_OK;
  }
  
  /* strcmp-style comparison of pReader's current term against pTerm.
-@@ -5077,6 +5312,9 @@
- ** the leaf data was entirely contained in the root), or from the
- ** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
- */
-+/* TODO(shess): Figure out a means of indicating how many leaves are
-+** expected, for purposes of detecting corruption.
-+*/
- static int leavesReaderInit(fulltext_vtab *v,
-                             int idx,
-                             sqlite_int64 iStartBlockid,
-@@ -5088,32 +5326,67 @@
+@@ -5133,14 +5326,19 @@ static int leavesReaderInit(fulltext_vtab *v,
  
    dataBufferInit(&pReader->rootData, 0);
    if( iStartBlockid==0 ){
 +    int rc;
-+    /* Corrupt if this can't be a leaf node. */
-+    if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){
-+      return SQLITE_CORRUPT_BKPT;
-+    }
+     /* Corrupt if this can't be a leaf node. */
+     if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){
+       return SQLITE_CORRUPT_BKPT;
+     }
      /* Entire leaf level fit in root data. */
      dataBufferReplace(&pReader->rootData, pRootData, nRootData);
 -    leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
 -                   &pReader->leafReader);
 +    rc = leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
 +                        &pReader->leafReader);
 +    if( rc!=SQLITE_OK ){
 +      dataBufferDestroy(&pReader->rootData);
 +      return rc;
 +    }
    }else{
      sqlite3_stmt *s;
      int rc = sql_get_leaf_statement(v, idx, &s);
-     if( rc!=SQLITE_OK ) return rc;
+@@ -5174,7 +5372,7 @@ static int leavesReaderInit(fulltext_vtab *v,
+       if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
+         rc = SQLITE_CORRUPT_BKPT;
+       }else{
+-        leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
++        rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
+       }
+     }
  
-     rc = sqlite3_bind_int64(s, 1, iStartBlockid);
--    if( rc!=SQLITE_OK ) return rc;
-+    if( rc!=SQLITE_OK ) goto err;
- 
-     rc = sqlite3_bind_int64(s, 2, iEndBlockid);
--    if( rc!=SQLITE_OK ) return rc;
-+    if( rc!=SQLITE_OK ) goto err;
- 
-     rc = sqlite3_step(s);
-+
-+    /* Corrupt if interior node referenced missing leaf node. */
-     if( rc==SQLITE_DONE ){
--      pReader->eof = 1;
--      return SQLITE_OK;
-+      rc = SQLITE_CORRUPT_BKPT;
-+      goto err;
-+    }
-+
-+    if( rc!=SQLITE_ROW ) goto err;
-+    rc = SQLITE_OK;
-+
-+    /* Corrupt if leaf data isn't a blob. */
-+    if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
-+      rc = SQLITE_CORRUPT_BKPT;
-+    }else{
-+      const char *pLeafData = sqlite3_column_blob(s, 0);
-+      int nLeafData = sqlite3_column_bytes(s, 0);
-+
-+      /* Corrupt if this can't be a leaf node. */
-+      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
-+        rc = SQLITE_CORRUPT_BKPT;
-+      }else{
-+        rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
-+      }
-+    }
-+
-+ err:
-+    if( rc!=SQLITE_OK ){
-+      if( idx==-1 ){
-+        sqlite3_finalize(s);
-+      }else{
-+        sqlite3_reset(s);
-+      }
-+      return rc;
-     }
--    if( rc!=SQLITE_ROW ) return rc;
- 
-     pReader->pStmt = s;
--    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
--                   sqlite3_column_bytes(pReader->pStmt, 0),
--                   &pReader->leafReader);
-   }
-   return SQLITE_OK;
- }
-@@ -5122,11 +5395,12 @@
+@@ -5197,11 +5395,12 @@ static int leavesReaderInit(fulltext_vtab *v,
  ** end of the current leaf, step forward to the next leaf block.
  */
  static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
 +  int rc;
    assert( !leavesReaderAtEnd(pReader) );
 -  leafReaderStep(&pReader->leafReader);
 +  rc = leafReaderStep(&pReader->leafReader);
 +  if( rc!=SQLITE_OK ) return rc;
  
    if( leafReaderAtEnd(&pReader->leafReader) ){
 -    int rc;
      if( pReader->rootData.pData ){
        pReader->eof = 1;
        return SQLITE_OK;
-@@ -5136,10 +5410,25 @@
-       pReader->eof = 1;
-       return rc==SQLITE_DONE ? SQLITE_OK : rc;
-     }
--    leafReaderDestroy(&pReader->leafReader);
--    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
--                   sqlite3_column_bytes(pReader->pStmt, 0),
--                   &pReader->leafReader);
-+
-+    /* Corrupt if leaf data isn't a blob. */
-+    if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){
-+      return SQLITE_CORRUPT_BKPT;
-+    }else{
+@@ -5216,6 +5415,7 @@ static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
+     if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){
+       return SQLITE_CORRUPT_BKPT;
+     }else{
 +      LeafReader tmp;
-+      const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0);
-+      int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0);
-+
-+      /* Corrupt if this can't be a leaf node. */
-+      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
-+        return SQLITE_CORRUPT_BKPT;
-+      }
-+
+       const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0);
+       int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0);
+ 
+@@ -5224,8 +5424,10 @@ static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
+         return SQLITE_CORRUPT_BKPT;
+       }
+ 
 +      rc = leafReaderInit(pLeafData, nLeafData, &tmp);
 +      if( rc!=SQLITE_OK ) return rc;
-+      leafReaderDestroy(&pReader->leafReader);
+       leafReaderDestroy(&pReader->leafReader);
+-      leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
 +      pReader->leafReader = tmp;
-+    }
+     }
    }
    return SQLITE_OK;
- }
-@@ -5200,8 +5489,19 @@
-     sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
-     const char *pRootData = sqlite3_column_blob(s, 2);
-     int nRootData = sqlite3_column_bytes(s, 2);
-+    sqlite_int64 iIndex = sqlite3_column_int64(s, 3);
-+
-+    /* Corrupt if we get back different types than we stored. */
-+    /* Also corrupt if the index is not sequential starting at 0. */
-+    if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-+        sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+        sqlite3_column_type(s, 2)!=SQLITE_BLOB ||
-+        i!=iIndex ||
-+        i>=MERGE_COUNT ){
-+      rc = SQLITE_CORRUPT_BKPT;
-+      break;
-+    }
- 
--    assert( i<MERGE_COUNT );
-     rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
-                           &pReaders[i]);
-     if( rc!=SQLITE_OK ) break;
-@@ -5212,6 +5512,7 @@
-     while( i-->0 ){
-       leavesReaderDestroy(&pReaders[i]);
-     }
-+    sqlite3_reset(s);          /* So we don't leave a lock. */
-     return rc;
-   }
- 
-@@ -5235,13 +5536,26 @@
+@@ -5334,13 +5536,26 @@ static int leavesReadersMerge(fulltext_vtab *v,
    DLReader dlReaders[MERGE_COUNT];
    const char *pTerm = leavesReaderTerm(pReaders);
    int i, nTerm = leavesReaderTermBytes(pReaders);
 +  int rc;
  
    assert( nReaders<=MERGE_COUNT );
  
    for(i=0; i<nReaders; i++){
 -    dlrInit(&dlReaders[i], DL_DEFAULT,
 -            leavesReaderData(pReaders+i),
 -            leavesReaderDataBytes(pReaders+i));
 +    const char *pData = leavesReaderData(pReaders+i);
 +    if( pData==NULL ){
 +      rc = SQLITE_CORRUPT_BKPT;
 +      break;
 +    }
 +    rc = dlrInit(&dlReaders[i], DL_DEFAULT,
 +                 pData,
 +                 leavesReaderDataBytes(pReaders+i));
 +    if( rc!=SQLITE_OK ) break;
 +  }
 +  if( rc!=SQLITE_OK ){
-+    while( i-->0 ){ 
++    while( i-->0 ){
 +      dlrDestroy(&dlReaders[i]);
 +    }
 +    return rc;
    }
  
    return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
-@@ -5295,10 +5609,14 @@
-   memset(&lrs, '\0', sizeof(lrs));
-   rc = leavesReadersInit(v, iLevel, lrs, &i);
-   if( rc!=SQLITE_OK ) return rc;
--  assert( i==MERGE_COUNT );
- 
-   leafWriterInit(iLevel+1, idx, &writer);
- 
-+  if( i!=MERGE_COUNT ){
-+    rc = SQLITE_CORRUPT_BKPT;
-+    goto err;
-+  }
-+
-   /* Since leavesReaderReorder() pushes readers at eof to the end,
-   ** when the first reader is empty, all will be empty.
-   */
-@@ -5341,12 +5659,14 @@
+@@ -5444,12 +5659,14 @@ static int segmentMerge(fulltext_vtab *v, int iLevel){
  }
  
  /* Accumulate the union of *acc and *pData into *acc. */
 -static void docListAccumulateUnion(DataBuffer *acc,
 -                                   const char *pData, int nData) {
 +static int docListAccumulateUnion(DataBuffer *acc,
 +                                  const char *pData, int nData) {
    DataBuffer tmp = *acc;
 +  int rc;
    dataBufferInit(acc, tmp.nData+nData);
 -  docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
 +  rc = docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
    dataBufferDestroy(&tmp);
 +  return rc;
  }
  
  /* TODO(shess) It might be interesting to explore different merge
-@@ -5388,8 +5708,13 @@
+@@ -5491,8 +5708,13 @@ static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
      int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
      if( c>0 ) break;      /* Past any possible matches. */
      if( c==0 ){
 +      int iBuffer, nData;
        const char *pData = leavesReaderData(pReader);
 -      int iBuffer, nData = leavesReaderDataBytes(pReader);
 +      if( pData==NULL ){
 +        rc = SQLITE_CORRUPT_BKPT;
 +        break;
 +      }
 +      nData = leavesReaderDataBytes(pReader);
  
        /* Find the first empty buffer. */
        for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
-@@ -5435,11 +5760,13 @@
+@@ -5538,11 +5760,13 @@ static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
          ** with pData/nData.
          */
          dataBufferSwap(p, pAcc);
 -        docListAccumulateUnion(pAcc, pData, nData);
 +        rc = docListAccumulateUnion(pAcc, pData, nData);
 +        if( rc!=SQLITE_OK ) goto err;
  
          /* Accumulate remaining doclists into pAcc. */
          for(++p; p<pAcc; ++p){
 -          docListAccumulateUnion(pAcc, p->pData, p->nData);
 +          rc = docListAccumulateUnion(pAcc, p->pData, p->nData);
 +          if( rc!=SQLITE_OK ) goto err;
  
            /* dataBufferReset() could allow a large doclist to blow up
            ** our memory requirements.
-@@ -5464,13 +5791,15 @@
+@@ -5567,13 +5791,15 @@ static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
          if( out->nData==0 ){
            dataBufferSwap(out, &(pBuffers[iBuffer]));
          }else{
 -          docListAccumulateUnion(out, pBuffers[iBuffer].pData,
 -                                 pBuffers[iBuffer].nData);
 +          rc = docListAccumulateUnion(out, pBuffers[iBuffer].pData,
 +                                      pBuffers[iBuffer].nData);
 +          if( rc!=SQLITE_OK ) break;
          }
        }
      }
    }
  
 +err:
    while( nBuffers-- ){
      dataBufferDestroy(&(pBuffers[nBuffers]));
    }
-@@ -5529,20 +5858,26 @@
+@@ -5632,20 +5858,26 @@ static int loadSegmentLeaves(fulltext_vtab *v,
  ** node.  Consider whether breaking symmetry is worthwhile.  I suspect
  ** it is not worthwhile.
  */
 -static void getChildrenContaining(const char *pData, int nData,
 -                                  const char *pTerm, int nTerm, int isPrefix,
 -                                  sqlite_int64 *piStartChild,
 -                                  sqlite_int64 *piEndChild){
 +static int getChildrenContaining(const char *pData, int nData,
 +                                 const char *pTerm, int nTerm, int isPrefix,
 +                                 sqlite_int64 *piStartChild,
@@ skipping 12 matching lines @@
      if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
 -    interiorReaderStep(&reader);
 +    rc = interiorReaderStep(&reader);
 +    if( rc!=SQLITE_OK ){
 +      interiorReaderDestroy(&reader);
 +      return rc;
 +    }
    }
    *piStartChild = interiorReaderCurrentBlockid(&reader);
  
-@@ -5552,7 +5887,11 @@
+@@ -5655,7 +5887,11 @@ static void getChildrenContaining(const char *pData, int nData,
    */
    while( !interiorReaderAtEnd(&reader) ){
      if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
 -    interiorReaderStep(&reader);
 +    rc = interiorReaderStep(&reader);
 +    if( rc!=SQLITE_OK ){
 +      interiorReaderDestroy(&reader);
 +      return rc;
 +    }
    }
    *piEndChild = interiorReaderCurrentBlockid(&reader);
  
-@@ -5561,6 +5900,7 @@
+@@ -5664,6 +5900,7 @@ static void getChildrenContaining(const char *pData, int nData,
    /* Children must ascend, and if !prefix, both must be the same. */
    assert( *piEndChild>=*piStartChild );
    assert( isPrefix || *piStartChild==*piEndChild );
 +  return rc;
  }
  
  /* Read block at iBlockid and pass it with other params to
-@@ -5588,11 +5928,31 @@
-   if( rc!=SQLITE_OK ) return rc;
+@@ -5709,8 +5946,12 @@ static int loadAndGetChildrenContaining(
+       return SQLITE_CORRUPT_BKPT;
+     }
  
-   rc = sqlite3_step(s);
--  if( rc==SQLITE_DONE ) return SQLITE_ERROR;
-+  /* Corrupt if interior node references missing child node. */
-+  if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
-   if( rc!=SQLITE_ROW ) return rc;
- 
--  getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
--                        pTerm, nTerm, isPrefix, piStartChild, piEndChild);
-+  /* Corrupt if child node isn't a blob. */
-+  if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
-+    sqlite3_reset(s);         /* So we don't leave a lock. */
-+    return SQLITE_CORRUPT_BKPT;
-+  }else{
-+    const char *pData = sqlite3_column_blob(s, 0);
-+    int nData = sqlite3_column_bytes(s, 0);
-+
-+    /* Corrupt if child is not a valid interior node. */
-+    if( pData==NULL || nData<1 || pData[0]=='\0' ){
-+      sqlite3_reset(s);         /* So we don't leave a lock. */
-+      return SQLITE_CORRUPT_BKPT;
-+    }
-+
+-    getChildrenContaining(pData, nData, pTerm, nTerm,
+-                          isPrefix, piStartChild, piEndChild);
 +    rc = getChildrenContaining(pData, nData, pTerm, nTerm,
 +                               isPrefix, piStartChild, piEndChild);
 +    if( rc!=SQLITE_OK ){
 +      sqlite3_reset(s);
 +      return rc;
 +    }
-+  }
+   }
  
    /* We expect only one row.  We must execute another sqlite3_step()
-    * to complete the iteration; otherwise the table will remain
-@@ -5622,8 +5982,9 @@
+@@ -5741,8 +5982,9 @@ static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
      /* Process pData as an interior node, then loop down the tree
      ** until we find the set of leaf nodes to scan for the term.
      */
 -    getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
 -                          &iStartChild, &iEndChild);
 +    rc = getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
 +                               &iStartChild, &iEndChild);
 +    if( rc!=SQLITE_OK ) return rc;
      while( iStartChild>iLeavesEnd ){
        sqlite_int64 iNextStart, iNextEnd;
        rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
-@@ -5675,7 +6036,8 @@
-   DataBuffer result;
-   int rc;
- 
--  assert( nData>1 );
-+  /* Corrupt if segment root can't be valid. */
-+  if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT;
- 
-   /* This code should never be called with buffered updates. */
-   assert( v->nPendingData<0 );
-@@ -5692,16 +6054,21 @@
+@@ -5812,16 +6054,21 @@ static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
        DataBuffer merged;
        DLReader readers[2];
  
 -      dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
 -      dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
 -      dataBufferInit(&merged, out->nData+result.nData);
 -      docListMerge(&merged, readers, 2);
 -      dataBufferDestroy(out);
 -      *out = merged;
 -      dlrDestroy(&readers[0]);
 -      dlrDestroy(&readers[1]);
 +      rc = dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
 +      if( rc==SQLITE_OK ){
 +        rc = dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
 +        if( rc==SQLITE_OK ){
 +          dataBufferInit(&merged, out->nData+result.nData);
 +          rc = docListMerge(&merged, readers, 2);
 +          dataBufferDestroy(out);
 +          *out = merged;
 +          dlrDestroy(&readers[1]);
 +        }
 +        dlrDestroy(&readers[0]);
 +      }
      }
    }
 +
    dataBufferDestroy(&result);
    return rc;
  }
-@@ -5729,11 +6096,20 @@
-     const char *pData = sqlite3_column_blob(s, 2);
-     const int nData = sqlite3_column_bytes(s, 2);
-     const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
-+
-+    /* Corrupt if we get back different types than we stored. */
-+    if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+        sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-+      rc = SQLITE_CORRUPT_BKPT;
-+      goto err;
-+    }
-+
-     rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
-                      &doclist);
+@@ -5862,6 +6109,7 @@ static int termSelect(fulltext_vtab *v, int iColumn,
      if( rc!=SQLITE_OK ) goto err;
    }
    if( rc==SQLITE_DONE ){
 +    rc = SQLITE_OK;
      if( doclist.nData!=0 ){
        /* TODO(shess) The old term_select_all() code applied the column
        ** restrict as we merged segments, leading to smaller buffers.
-@@ -5741,13 +6117,13 @@
+@@ -5869,10 +6117,9 @@ static int termSelect(fulltext_vtab *v, int iColumn,
        ** system is checked in.
        */
        if( iColumn==v->nColumn) iColumn = -1;
 -      docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 -                  iColumn, iType, out);
 +      rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 +                       iColumn, iType, out);
      }
 -    rc = SQLITE_OK;
    }
  
   err:
-+  sqlite3_reset(s);         /* So we don't leave a lock. */
-   dataBufferDestroy(&doclist);
-   return rc;
- }
-@@ -6089,6 +6465,7 @@
+@@ -6218,6 +6465,7 @@ static int optimizeInternal(fulltext_vtab *v,
                              LeafWriter *pWriter){
    int i, rc = SQLITE_OK;
    DataBuffer doclist, merged, tmp;
 +  const char *pData;
  
    /* Order the readers. */
    i = nReaders;
-@@ -6109,14 +6486,21 @@
+@@ -6238,14 +6486,21 @@ static int optimizeInternal(fulltext_vtab *v,
        if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
      }
  
 +    pData = optLeavesReaderData(&readers[0]);
 +    if( pData==NULL ){
 +      rc = SQLITE_CORRUPT_BKPT;
 +      break;
 +    }
 +
      /* Special-case for no merge. */
      if( i==1 ){
        /* Trim deletions from the doclist. */
        dataBufferReset(&merged);
 -      docListTrim(DL_DEFAULT,
 -                  optLeavesReaderData(&readers[0]),
 -                  optLeavesReaderDataBytes(&readers[0]),
 -                  -1, DL_DEFAULT, &merged);
 +      rc = docListTrim(DL_DEFAULT,
 +                       pData,
 +                       optLeavesReaderDataBytes(&readers[0]),
 +                       -1, DL_DEFAULT, &merged);
 +      if( rc!= SQLITE_OK ) break;
      }else{
        DLReader dlReaders[MERGE_COUNT];
        int iReader, nReaders;
-@@ -6124,9 +6508,10 @@
+@@ -6253,9 +6508,10 @@ static int optimizeInternal(fulltext_vtab *v,
        /* Prime the pipeline with the first reader's doclist.  After
        ** one pass index 0 will reference the accumulated doclist.
        */
 -      dlrInit(&dlReaders[0], DL_DEFAULT,
 -              optLeavesReaderData(&readers[0]),
 -              optLeavesReaderDataBytes(&readers[0]));
 +      rc = dlrInit(&dlReaders[0], DL_DEFAULT,
 +                   pData,
 +                   optLeavesReaderDataBytes(&readers[0]));
 +      if( rc!=SQLITE_OK ) break;
        iReader = 1;
  
        assert( iReader<i );  /* Must execute the loop at least once. */
-@@ -6134,24 +6519,35 @@
+@@ -6263,24 +6519,35 @@ static int optimizeInternal(fulltext_vtab *v,
          /* Merge 16 inputs per pass. */
          for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
               iReader++, nReaders++ ){
 -          dlrInit(&dlReaders[nReaders], DL_DEFAULT,
 -                  optLeavesReaderData(&readers[iReader]),
 -                  optLeavesReaderDataBytes(&readers[iReader]));
 +          pData = optLeavesReaderData(&readers[iReader]);
 +          if( pData == NULL ){
 +            rc = SQLITE_CORRUPT_BKPT;
 +            break;
@@ skipping 24 matching lines @@
  
 +        if( rc!=SQLITE_OK ) goto err;
 +
          /* Accumulated doclist to reader 0 for next pass. */
 -        dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
 +        rc = dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
 +        if( rc!=SQLITE_OK ) goto err;
        }
  
        /* Destroy reader that was left in the pipeline. */
-@@ -6159,8 +6555,9 @@
+@@ -6288,8 +6555,9 @@ static int optimizeInternal(fulltext_vtab *v,
  
        /* Trim deletions from the doclist. */
        dataBufferReset(&merged);
 -      docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 -                  -1, DL_DEFAULT, &merged);
 +      rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 +                       -1, DL_DEFAULT, &merged);
 +      if( rc!=SQLITE_OK ) goto err;
      }
  
      /* Only pass doclists with hits (skip if all hits deleted). */
-@@ -6240,6 +6637,14 @@
-       const char *pRootData = sqlite3_column_blob(s, 2);
-       int nRootData = sqlite3_column_bytes(s, 2);
- 
-+      /* Corrupt if we get back different types than we stored. */
-+      if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-+          sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+          sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-+        rc = SQLITE_CORRUPT_BKPT;
-+        break;
-+      }
-+
-       assert( i<nReaders );
-       rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
-                             &readers[i].reader);
-@@ -6253,6 +6658,8 @@
-     if( rc==SQLITE_DONE ){
-       assert( i==nReaders );
-       rc = optimizeInternal(v, readers, nReaders, &writer);
-+    }else{
-+      sqlite3_reset(s);      /* So we don't leave a lock. */
-     }
- 
-     while( i-- > 0 ){
-@@ -6316,9 +6723,18 @@
-   const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
-   const char *pRootData = sqlite3_column_blob(s, 2);
-   const int nRootData = sqlite3_column_bytes(s, 2);
-+  int rc;
-   LeavesReader reader;
--  int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
--                            pRootData, nRootData, &reader);
-+
-+  /* Corrupt if we get back different types than we stored. */
-+  if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-+      sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+      sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-+    return SQLITE_CORRUPT_BKPT;
-+  }
-+
-+  rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
-+                        pRootData, nRootData, &reader);
-   if( rc!=SQLITE_OK ) return rc;
- 
-   while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
-@@ -6480,16 +6896,19 @@
+@@ -6628,16 +6896,19 @@ static void createDoclistResult(sqlite3_context *pContext,
                                  const char *pData, int nData){
    DataBuffer dump;
    DLReader dlReader;
 +  int rc;
  
    assert( pData!=NULL && nData>0 );
  
 +  rc = dlrInit(&dlReader, DL_DEFAULT, pData, nData);
-+  if( rc!=SQLITE_OK ) return rc;
++  if( rc!=SQLITE_OK ) return;
    dataBufferInit(&dump, 0);
 -  dlrInit(&dlReader, DL_DEFAULT, pData, nData);
 -  for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
 +  for( ; rc==SQLITE_OK && !dlrAtEnd(&dlReader); rc = dlrStep(&dlReader) ){
      char buf[256];
      PLReader plReader;
  
 -    plrInit(&plReader, &dlReader);
 +    rc = plrInit(&plReader, &dlReader);
 +    if( rc!=SQLITE_OK ) break;
      if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
        sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
        dataBufferAppend(&dump, buf, strlen(buf));
-@@ -6500,7 +6919,8 @@
+@@ -6648,7 +6919,8 @@ static void createDoclistResult(sqlite3_context *pContext,
                         dlrDocid(&dlReader), iColumn);
        dataBufferAppend(&dump, buf, strlen(buf));
  
 -      for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
 +      for( ; !plrAtEnd(&plReader); rc = plrStep(&plReader) ){
 +        if( rc!=SQLITE_OK ) break;
          if( plrColumn(&plReader)!=iColumn ){
            iColumn = plrColumn(&plReader);
            sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
-@@ -6521,6 +6941,7 @@
+@@ -6669,6 +6941,7 @@ static void createDoclistResult(sqlite3_context *pContext,
          dataBufferAppend(&dump, buf, strlen(buf));
        }
        plrDestroy(&plReader);
 +      if( rc!= SQLITE_OK ) break;
  
        assert( dump.nData>0 );
        dump.nData--;                     /* Overwrite trailing space. */
-@@ -6529,6 +6950,10 @@
+@@ -6677,6 +6950,10 @@ static void createDoclistResult(sqlite3_context *pContext,
      }
    }
    dlrDestroy(&dlReader);
 +  if( rc!=SQLITE_OK ){
 +    dataBufferDestroy(&dump);
-+    return rc;
++    return;
 +  }
  
    assert( dump.nData>0 );
    dump.nData--;                     /* Overwrite trailing space. */
-@@ -6540,6 +6965,7 @@
-   sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
-   dump.pData = NULL;
-   dump.nData = dump.nCapacity = 0;
-+  return SQLITE_OK;
- }
- 
- /* Implements dump_doclist() for use in inspecting the fts2 index from
-@@ -6822,7 +7248,11 @@
-   ** module with sqlite.
-   */
-   if( SQLITE_OK==rc 
-+#if GEARS_FTS2_CHANGES && !SQLITE_TEST
-+      /* fts2_tokenizer() disabled for security reasons. */
-+#else
-    && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer"))
-+#endif
-    && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
-    && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
-    && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
-diff -ru ext-orig/fts2/fts2_icu.c ext/fts2/fts2_icu.c
---- ext-orig/fts2/fts2_icu.c    2009-09-03 13:32:06.000000000 -0700
-+++ ext/fts2/fts2_icu.c 2009-09-18 14:39:41.000000000 -0700
-@@ -198,7 +198,7 @@
- 
-     while( iStart<iEnd ){
-       int iWhite = iStart;
--      U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
-+      U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
-       if( u_isspace(c) ){
-         iStart = iWhite;
-       }else{
-diff -ru ext-orig/fts2/fts2_tokenizer.c ext/fts2/fts2_tokenizer.c
---- ext-orig/fts2/fts2_tokenizer.c      2009-09-03 13:32:06.000000000 -0700
-+++ ext/fts2/fts2_tokenizer.c   2009-09-18 14:39:41.000000000 -0700
-@@ -28,11 +28,14 @@
- 
- #include "sqlite3.h"
- #include "sqlite3ext.h"
--SQLITE_EXTENSION_INIT1
-+#ifndef SQLITE_CORE
-+  SQLITE_EXTENSION_INIT1
-+#endif
- 
- #include "fts2_hash.h"
- #include "fts2_tokenizer.h"
- #include <assert.h>
-+#include <stddef.h>
- 
- /*
- ** Implementation of the SQL scalar function for accessing the underlying 
+-- 
+2.2.1
+