Fix the FTS3 patch.

third_party/sqlite/fts3.patch

-Index: ext/fts3/fts3_tokenizer.c
-===================================================================
---- ext/fts3/fts3_tokenizer.c   (revision 48758)
-+++ ext/fts3/fts3_tokenizer.c   (working copy)
-@@ -33,6 +33,7 @@
- #include "fts3_hash.h"
- #include "fts3_tokenizer.h"
- #include <assert.h>
-+#include <stddef.h>
- 
- /*
- ** Implementation of the SQL scalar function for accessing the underlying 
 Index: ext/fts3/fts3.c
 ===================================================================
---- ext/fts3/fts3.c     (revision 48758)
+--- ext/fts3/fts3.c     (revision 48811)
 +++ ext/fts3/fts3.c     (working copy)
 @@ -271,6 +271,7 @@
  ** deletions and duplications.  This would basically be a forced merge
  ** into a single segment.
  */
 +#define CHROMIUM_FTS3_CHANGES 1
  
  #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
  
 @@ -313,6 +314,16 @@
@@ skipping 1212 matching lines @@
  
 -    pReader->pData += n+nSuffix;
 -    pReader->nData -= n+nSuffix;
 +    pReader->pData += nSuffix;
 +    pReader->nData -= nSuffix;
    }
 +  return SQLITE_OK;
  }
  
  /* strcmp-style comparison of pReader's current term against pTerm.
-@@ -5222,32 +5423,65 @@
+@@ -5222,32 +5423,67 @@
  
    dataBufferInit(&pReader->rootData, 0);
    if( iStartBlockid==0 ){
 +    int rc;
 +    /* Corrupt if this can't be a leaf node. */
 +    if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){
 +      return SQLITE_CORRUPT_BKPT;
 +    }
      /* Entire leaf level fit in root data. */
      dataBufferReplace(&pReader->rootData, pRootData, nRootData);
@@ skipping 22 matching lines @@
 +
 +    /* Corrupt if interior node referenced missing leaf node. */
      if( rc==SQLITE_DONE ){
 -      pReader->eof = 1;
 -      return SQLITE_OK;
 +      rc = SQLITE_CORRUPT_BKPT;
 +      goto err;
      }
 -    if( rc!=SQLITE_ROW ) return rc;
  
--    pReader->pStmt = s;
--    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
--                   sqlite3_column_bytes(pReader->pStmt, 0),
--                   &pReader->leafReader);
 +    if( rc!=SQLITE_ROW ) goto err;
 +    rc = SQLITE_OK;
 +
 +    /* Corrupt if leaf data isn't a blob. */
 +    if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
 +      rc = SQLITE_CORRUPT_BKPT;
 +    }else{
 +      const char *pLeafData = sqlite3_column_blob(s, 0);
 +      int nLeafData = sqlite3_column_bytes(s, 0);
 +
 +      /* Corrupt if this can't be a leaf node. */
 +      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
 +        rc = SQLITE_CORRUPT_BKPT;
 +      }else{
 +        rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
 +      }
 +    }
 +
 + err:
 +    if( rc!=SQLITE_OK ){
 +      if( idx==-1 ){
 +        sqlite3_finalize(s);
 +      }else{
 +        sqlite3_reset(s);
 +      }
 +      return rc;
 +    }
++
+     pReader->pStmt = s;
+-    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
+-                   sqlite3_column_bytes(pReader->pStmt, 0),
+-                   &pReader->leafReader);
    }
    return SQLITE_OK;
  }
-@@ -5256,11 +5490,12 @@
+@@ -5256,11 +5492,12 @@
  ** end of the current leaf, step forward to the next leaf block.
  */
  static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
 +  int rc;
    assert( !leavesReaderAtEnd(pReader) );
 -  leafReaderStep(&pReader->leafReader);
 +  rc = leafReaderStep(&pReader->leafReader);
 +  if( rc!=SQLITE_OK ) return rc;
  
    if( leafReaderAtEnd(&pReader->leafReader) ){
 -    int rc;
      if( pReader->rootData.pData ){
        pReader->eof = 1;
        return SQLITE_OK;
-@@ -5270,10 +5505,25 @@
+@@ -5270,10 +5507,25 @@
        pReader->eof = 1;
        return rc==SQLITE_DONE ? SQLITE_OK : rc;
      }
 -    leafReaderDestroy(&pReader->leafReader);
 -    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
 -                   sqlite3_column_bytes(pReader->pStmt, 0),
 -                   &pReader->leafReader);
 +
 +    /* Corrupt if leaf data isn't a blob. */
 +    if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){
 +      return SQLITE_CORRUPT_BKPT;
 +    }else{
 +      LeafReader tmp;
 +      const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0);
 +      int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0);
 +
 +      /* Corrupt if this can't be a leaf node. */
 +      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
 +        return SQLITE_CORRUPT_BKPT;
 +      }
 +
 +      rc = leafReaderInit(pLeafData, nLeafData, &tmp);
 +      if( rc!=SQLITE_OK ) return rc;
 +      leafReaderDestroy(&pReader->leafReader);
 +      pReader->leafReader = tmp;
 +    }
    }
    return SQLITE_OK;
  }
-@@ -5334,8 +5584,19 @@
+@@ -5334,8 +5586,19 @@
      sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
      const char *pRootData = sqlite3_column_blob(s, 2);
      int nRootData = sqlite3_column_bytes(s, 2);
 +    sqlite_int64 iIndex = sqlite3_column_int64(s, 3);
  
 -    assert( i<MERGE_COUNT );
 +    /* Corrupt if we get back different types than we stored. */
 +    /* Also corrupt if the index is not sequential starting at 0. */
 +    if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
 +        sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 +        sqlite3_column_type(s, 2)!=SQLITE_BLOB ||
 +        i!=iIndex ||
 +        i>=MERGE_COUNT ){
 +      rc = SQLITE_CORRUPT_BKPT;
 +      break;
 +    }
 +
      rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
                            &pReaders[i]);
      if( rc!=SQLITE_OK ) break;
-@@ -5346,6 +5607,7 @@
+@@ -5346,6 +5609,7 @@
      while( i-->0 ){
        leavesReaderDestroy(&pReaders[i]);
      }
 +    sqlite3_reset(s);  /* So we don't leave a lock. */
      return rc;
    }
  
-@@ -5369,14 +5631,27 @@
+@@ -5369,14 +5633,27 @@
    DLReader dlReaders[MERGE_COUNT];
    const char *pTerm = leavesReaderTerm(pReaders);
    int i, nTerm = leavesReaderTermBytes(pReaders);
 +  int rc;
  
    assert( nReaders<=MERGE_COUNT );
  
    for(i=0; i<nReaders; i++){
 -    dlrInit(&dlReaders[i], DL_DEFAULT,
 -            leavesReaderData(pReaders+i),
@@ skipping 10 matching lines @@
    }
 +  if( rc!=SQLITE_OK ){
 +    while( i-->0 ){
 +      dlrDestroy(&dlReaders[i]);
 +    }
 +    return rc;
 +  }
  
    return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
  }
-@@ -5429,10 +5704,14 @@
+@@ -5429,10 +5706,14 @@
    memset(&lrs, '\0', sizeof(lrs));
    rc = leavesReadersInit(v, iLevel, lrs, &i);
    if( rc!=SQLITE_OK ) return rc;
 -  assert( i==MERGE_COUNT );
  
    leafWriterInit(iLevel+1, idx, &writer);
  
 +  if( i!=MERGE_COUNT ){
 +    rc = SQLITE_CORRUPT_BKPT;
 +    goto err;
 +  }
 +
    /* Since leavesReaderReorder() pushes readers at eof to the end,
    ** when the first reader is empty, all will be empty.
    */
-@@ -5475,12 +5754,14 @@
+@@ -5475,12 +5756,14 @@
  }
  
  /* Accumulate the union of *acc and *pData into *acc. */
 -static void docListAccumulateUnion(DataBuffer *acc,
 -                                   const char *pData, int nData) {
 +static int docListAccumulateUnion(DataBuffer *acc,
 +                                  const char *pData, int nData) {
    DataBuffer tmp = *acc;
 +  int rc;
    dataBufferInit(acc, tmp.nData+nData);
 -  docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
 +  rc = docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
    dataBufferDestroy(&tmp);
 +  return rc;
  }
  
  /* TODO(shess) It might be interesting to explore different merge
-@@ -5522,8 +5803,13 @@
+@@ -5522,8 +5805,13 @@
      int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
      if( c>0 ) break;      /* Past any possible matches. */
      if( c==0 ){
 +      int iBuffer, nData;
        const char *pData = leavesReaderData(pReader);
 -      int iBuffer, nData = leavesReaderDataBytes(pReader);
 +      if( pData==NULL ){
 +        rc = SQLITE_CORRUPT_BKPT;
 +        break;
 +      }
 +      nData = leavesReaderDataBytes(pReader);
  
        /* Find the first empty buffer. */
        for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
-@@ -5569,11 +5855,13 @@
+@@ -5569,11 +5857,13 @@
          ** with pData/nData.
          */
          dataBufferSwap(p, pAcc);
 -        docListAccumulateUnion(pAcc, pData, nData);
 +        rc = docListAccumulateUnion(pAcc, pData, nData);
 +        if( rc!=SQLITE_OK ) goto err;
  
          /* Accumulate remaining doclists into pAcc. */
          for(++p; p<pAcc; ++p){
 -          docListAccumulateUnion(pAcc, p->pData, p->nData);
 +          rc = docListAccumulateUnion(pAcc, p->pData, p->nData);
 +          if( rc!=SQLITE_OK ) goto err;
  
            /* dataBufferReset() could allow a large doclist to blow up
            ** our memory requirements.
-@@ -5598,13 +5886,15 @@
+@@ -5598,13 +5888,15 @@
          if( out->nData==0 ){
            dataBufferSwap(out, &(pBuffers[iBuffer]));
          }else{
 -          docListAccumulateUnion(out, pBuffers[iBuffer].pData,
 -                                 pBuffers[iBuffer].nData);
 +          rc = docListAccumulateUnion(out, pBuffers[iBuffer].pData,
 +                                      pBuffers[iBuffer].nData);
 +          if( rc!=SQLITE_OK ) break;
          }
        }
      }
    }
  
 +err:
    while( nBuffers-- ){
      dataBufferDestroy(&(pBuffers[nBuffers]));
    }
-@@ -5663,20 +5953,26 @@
+@@ -5663,20 +5955,26 @@
  ** node.  Consider whether breaking symmetry is worthwhile.  I suspect
  ** it is not worthwhile.
  */
 -static void getChildrenContaining(const char *pData, int nData,
 -                                  const char *pTerm, int nTerm, int isPrefix,
 -                                  sqlite_int64 *piStartChild,
 -                                  sqlite_int64 *piEndChild){
 +static int getChildrenContaining(const char *pData, int nData,
 +                                 const char *pTerm, int nTerm, int isPrefix,
 +                                 sqlite_int64 *piStartChild,
@@ skipping 12 matching lines @@
      if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
 -    interiorReaderStep(&reader);
 +    rc = interiorReaderStep(&reader);
 +    if( rc!=SQLITE_OK ){
 +      interiorReaderDestroy(&reader);
 +      return rc;
 +    }
    }
    *piStartChild = interiorReaderCurrentBlockid(&reader);
  
-@@ -5686,7 +5982,11 @@
+@@ -5686,7 +5984,11 @@
    */
    while( !interiorReaderAtEnd(&reader) ){
      if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
 -    interiorReaderStep(&reader);
 +    rc = interiorReaderStep(&reader);
 +    if( rc!=SQLITE_OK ){
 +      interiorReaderDestroy(&reader);
 +      return rc;
 +    }
    }
    *piEndChild = interiorReaderCurrentBlockid(&reader);
  
-@@ -5695,6 +5995,7 @@
+@@ -5695,6 +5997,7 @@
    /* Children must ascend, and if !prefix, both must be the same. */
    assert( *piEndChild>=*piStartChild );
    assert( isPrefix || *piStartChild==*piEndChild );
 +  return rc;
  }
  
  /* Read block at iBlockid and pass it with other params to
-@@ -5722,12 +6023,32 @@
+@@ -5722,12 +6025,32 @@
    if( rc!=SQLITE_OK ) return rc;
  
    rc = sqlite3_step(s);
 -  if( rc==SQLITE_DONE ) return SQLITE_ERROR;
 +  /* Corrupt if interior node references missing child node. */
 +  if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
    if( rc!=SQLITE_ROW ) return rc;
  
 -  getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
 -                        pTerm, nTerm, isPrefix, piStartChild, piEndChild);
@@ skipping 15 matching lines @@
 +                               isPrefix, piStartChild, piEndChild);
 +    if( rc!=SQLITE_OK ){
 +      sqlite3_reset(s);
 +      return rc;
 +    }
 +  }
 +
    /* We expect only one row.  We must execute another sqlite3_step()
     * to complete the iteration; otherwise the table will remain
     * locked. */
-@@ -5756,8 +6077,9 @@
+@@ -5756,8 +6079,9 @@
      /* Process pData as an interior node, then loop down the tree
      ** until we find the set of leaf nodes to scan for the term.
      */
 -    getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
 -                          &iStartChild, &iEndChild);
 +    rc = getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
 +                               &iStartChild, &iEndChild);
 +    if( rc!=SQLITE_OK ) return rc;
      while( iStartChild>iLeavesEnd ){
        sqlite_int64 iNextStart, iNextEnd;
        rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
-@@ -5809,7 +6131,8 @@
+@@ -5809,7 +6133,8 @@
    DataBuffer result;
    int rc;
  
 -  assert( nData>1 );
 +  /* Corrupt if segment root can't be valid. */
 +  if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT;
  
    /* This code should never be called with buffered updates. */
    assert( v->nPendingData<0 );
-@@ -5826,16 +6149,21 @@
+@@ -5826,16 +6151,21 @@
        DataBuffer merged;
        DLReader readers[2];
  
 -      dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
 -      dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
 -      dataBufferInit(&merged, out->nData+result.nData);
 -      docListMerge(&merged, readers, 2);
 -      dataBufferDestroy(out);
 -      *out = merged;
 -      dlrDestroy(&readers[0]);
 -      dlrDestroy(&readers[1]);
 +      rc = dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
 +      if( rc==SQLITE_OK ){
 +        rc = dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
 +        if( rc==SQLITE_OK ){
 +          dataBufferInit(&merged, out->nData+result.nData);
 +          rc = docListMerge(&merged, readers, 2);
 +          dataBufferDestroy(out);
 +          *out = merged;
 +          dlrDestroy(&readers[1]);
 +        }
 +        dlrDestroy(&readers[0]);
 +      }
      }
    }
 +
    dataBufferDestroy(&result);
    return rc;
  }
-@@ -5869,11 +6197,20 @@
+@@ -5869,11 +6199,20 @@
      const char *pData = sqlite3_column_blob(s, 2);
      const int nData = sqlite3_column_bytes(s, 2);
      const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
 +
 +    /* Corrupt if we get back different types than we stored. */
 +    if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 +        sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
 +      rc = SQLITE_CORRUPT_BKPT;
 +      goto err;
 +    }
 +
      rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
                       &doclist);
      if( rc!=SQLITE_OK ) goto err;
    }
    if( rc==SQLITE_DONE ){
 +    rc = SQLITE_OK;
      if( doclist.nData!=0 ){
        /* TODO(shess) The old term_select_all() code applied the column
        ** restrict as we merged segments, leading to smaller buffers.
-@@ -5881,13 +6218,13 @@
+@@ -5881,13 +6220,13 @@
        ** system is checked in.
        */
        if( iColumn==v->nColumn) iColumn = -1;
 -      docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 -                  iColumn, iType, out);
 +      rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 +                       iColumn, iType, out);
      }
 -    rc = SQLITE_OK;
    }
  
   err:
 +  sqlite3_reset(s);  /* So we don't leave a lock. */
    dataBufferDestroy(&doclist);
    return rc;
  }
-@@ -6250,6 +6587,7 @@
+@@ -6250,6 +6589,7 @@
                              LeafWriter *pWriter){
    int i, rc = SQLITE_OK;
    DataBuffer doclist, merged, tmp;
 +  const char *pData;
  
    /* Order the readers. */
    i = nReaders;
-@@ -6270,14 +6608,20 @@
+@@ -6270,14 +6610,20 @@
        if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
      }
  
 +    pData = optLeavesReaderData(&readers[0]);
 +    if( pData==NULL ){
 +      rc = SQLITE_CORRUPT_BKPT;
 +      break;
 +    }
 +
      /* Special-case for no merge. */
      if( i==1 ){
        /* Trim deletions from the doclist. */
        dataBufferReset(&merged);
 -      docListTrim(DL_DEFAULT,
 -                  optLeavesReaderData(&readers[0]),
 -                  optLeavesReaderDataBytes(&readers[0]),
 -                  -1, DL_DEFAULT, &merged);
 +      rc = docListTrim(DL_DEFAULT, pData,
 +                       optLeavesReaderDataBytes(&readers[0]),
 +                       -1, DL_DEFAULT, &merged);
 +      if( rc!=SQLITE_OK ) break;
      }else{
        DLReader dlReaders[MERGE_COUNT];
        int iReader, nReaders;
-@@ -6285,9 +6629,10 @@
+@@ -6285,9 +6631,10 @@
        /* Prime the pipeline with the first reader's doclist.  After
        ** one pass index 0 will reference the accumulated doclist.
        */
 -      dlrInit(&dlReaders[0], DL_DEFAULT,
 -              optLeavesReaderData(&readers[0]),
 -              optLeavesReaderDataBytes(&readers[0]));
 +      rc = dlrInit(&dlReaders[0], DL_DEFAULT,
 +                   pData,
 +                   optLeavesReaderDataBytes(&readers[0]));
 +      if( rc!=SQLITE_OK ) break;
        iReader = 1;
  
        assert( iReader<i );  /* Must execute the loop at least once. */
-@@ -6295,24 +6640,34 @@
+@@ -6295,24 +6642,34 @@
          /* Merge 16 inputs per pass. */
          for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
               iReader++, nReaders++ ){
 -          dlrInit(&dlReaders[nReaders], DL_DEFAULT,
 -                  optLeavesReaderData(&readers[iReader]),
 -                  optLeavesReaderDataBytes(&readers[iReader]));
 +          pData = optLeavesReaderData(&readers[iReader]);
 +          if( pData==NULL ){
 +            rc = SQLITE_CORRUPT_BKPT;
 +            break;
@@ skipping 23 matching lines @@
  
 +        if( rc!=SQLITE_OK ) goto err;
 +
          /* Accumulated doclist to reader 0 for next pass. */
 -        dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
 +        rc = dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
 +        if( rc!=SQLITE_OK ) goto err;
        }
  
        /* Destroy reader that was left in the pipeline. */
-@@ -6320,8 +6675,9 @@
+@@ -6320,8 +6677,9 @@
  
        /* Trim deletions from the doclist. */
        dataBufferReset(&merged);
 -      docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 -                  -1, DL_DEFAULT, &merged);
 +      rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
 +                       -1, DL_DEFAULT, &merged);
 +      if( rc!=SQLITE_OK ) goto err;
      }
  
      /* Only pass doclists with hits (skip if all hits deleted). */
-@@ -6401,6 +6757,14 @@
+@@ -6401,6 +6759,14 @@
        const char *pRootData = sqlite3_column_blob(s, 2);
        int nRootData = sqlite3_column_bytes(s, 2);
  
 +      /* Corrupt if we get back different types than we stored. */
 +      if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
 +          sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 +          sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
 +        rc = SQLITE_CORRUPT_BKPT;
 +        break;
 +      }
 +
        assert( i<nReaders );
        rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
                              &readers[i].reader);
-@@ -6414,6 +6778,8 @@
+@@ -6414,6 +6780,8 @@
      if( rc==SQLITE_DONE ){
        assert( i==nReaders );
        rc = optimizeInternal(v, readers, nReaders, &writer);
 +    }else{
 +      sqlite3_reset(s);  /* So we don't leave a lock. */
      }
  
      while( i-- > 0 ){
-@@ -6477,9 +6843,18 @@
+@@ -6477,9 +6845,18 @@
    const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
    const char *pRootData = sqlite3_column_blob(s, 2);
    const int nRootData = sqlite3_column_bytes(s, 2);
 +  int rc;
    LeavesReader reader;
 -  int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
 -                            pRootData, nRootData, &reader);
 +
 +  /* Corrupt if we get back different types than we stored. */
 +  if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
 +      sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 +      sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
 +    return SQLITE_CORRUPT_BKPT;
 +  }
 +
 +  rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
 +                        pRootData, nRootData, &reader);
    if( rc!=SQLITE_OK ) return rc;
  
    while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
-@@ -6641,16 +7016,19 @@
+@@ -6641,16 +7018,19 @@
                                  const char *pData, int nData){
    DataBuffer dump;
    DLReader dlReader;
 +  int rc;
  
    assert( pData!=NULL && nData>0 );
  
 +  rc = dlrInit(&dlReader, DL_DEFAULT, pData, nData);
 +  if( rc!=SQLITE_OK ) return rc;
    dataBufferInit(&dump, 0);
 -  dlrInit(&dlReader, DL_DEFAULT, pData, nData);
 -  for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
 +  for( ; rc==SQLITE_OK && !dlrAtEnd(&dlReader); rc = dlrStep(&dlReader) ){
      char buf[256];
      PLReader plReader;
  
 -    plrInit(&plReader, &dlReader);
 +    rc = plrInit(&plReader, &dlReader);
 +    if( rc!=SQLITE_OK ) break;
      if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
        sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
        dataBufferAppend(&dump, buf, strlen(buf));
-@@ -6661,7 +7039,8 @@
+@@ -6661,7 +7041,8 @@
                         dlrDocid(&dlReader), iColumn);
        dataBufferAppend(&dump, buf, strlen(buf));
  
 -      for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
 +      for( ; !plrAtEnd(&plReader); rc = plrStep(&plReader) ){
 +        if( rc!=SQLITE_OK ) break;
          if( plrColumn(&plReader)!=iColumn ){
            iColumn = plrColumn(&plReader);
            sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
-@@ -6682,6 +7061,7 @@
+@@ -6682,6 +7063,7 @@
          dataBufferAppend(&dump, buf, strlen(buf));
        }
        plrDestroy(&plReader);
 +      if( rc!= SQLITE_OK ) break;
  
        assert( dump.nData>0 );
        dump.nData--;                     /* Overwrite trailing space. */
-@@ -6690,6 +7070,10 @@
+@@ -6690,6 +7072,10 @@
      }
    }
    dlrDestroy(&dlReader);
 +  if( rc!=SQLITE_OK ){
 +    dataBufferDestroy(&dump);
 +    return rc;
 +  }
  
    assert( dump.nData>0 );
    dump.nData--;                     /* Overwrite trailing space. */
-@@ -6701,6 +7085,7 @@
+@@ -6701,6 +7087,7 @@
    sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
    dump.pData = NULL;
    dump.nData = dump.nCapacity = 0;
 +  return SQLITE_OK;
  }
  
  /* Implements dump_doclist() for use in inspecting the fts3 index from
-@@ -6987,7 +7372,11 @@
+@@ -6987,7 +7374,11 @@
    ** module with sqlite.
    */
    if( SQLITE_OK==rc 
 +#if CHROMIUM_FTS3_CHANGES && !SQLITE_TEST
 +      /* fts3_tokenizer() disabled for security reasons. */
 +#else
     && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
 +#endif
     && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
     && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
     && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
 Index: ext/fts3/fts3_icu.c
 ===================================================================
---- ext/fts3/fts3_icu.c (revision 48758)
+--- ext/fts3/fts3_icu.c (revision 48811)
 +++ ext/fts3/fts3_icu.c (working copy)
 @@ -198,7 +198,7 @@
  
      while( iStart<iEnd ){
        int iWhite = iStart;
 -      U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
 +      U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
        if( u_isspace(c) ){
          iStart = iWhite;
        }else{
+Index: ext/fts3/fts3_tokenizer.c
+===================================================================
+--- ext/fts3/fts3_tokenizer.c   (revision 48811)
++++ ext/fts3/fts3_tokenizer.c   (working copy)
+@@ -33,6 +33,7 @@
+ #include "fts3_hash.h"
+ #include "fts3_tokenizer.h"
+ #include <assert.h>
++#include <stddef.h>
+ 
+ /*
+ ** Implementation of the SQL scalar function for accessing the underlying