Move bundled copy of sqlite one level deeper to better separate it...

third_party/sqlite/ext/fts1/fulltext.c

Index: third_party/sqlite/ext/fts1/fulltext.c
===================================================================
--- third_party/sqlite/ext/fts1/fulltext.c	(revision 56608)
+++ third_party/sqlite/ext/fts1/fulltext.c	(working copy)
@@ -1,1496 +0,0 @@
-/* The author disclaims copyright to this source code.
- *
- * This is an SQLite module implementing full-text search.
- */
-
-#include <assert.h>
-#if !defined(__APPLE__)
-#include <malloc.h>
-#else
-#include <stdlib.h>
-#endif
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-
-#include "fulltext.h"
-#include "ft_hash.h"
-#include "tokenizer.h"
-#include "sqlite3.h"
-#include "sqlite3ext.h"
-SQLITE_EXTENSION_INIT1
-
-/* utility functions */
-
-/* We encode variable-length integers in little-endian order using seven bits
- * per byte as follows:
-**
-** KEY:
-**         A = 0xxxxxxx    7 bits of data and one flag bit
-**         B = 1xxxxxxx    7 bits of data and one flag bit
-**
-**  7 bits - A
-** 14 bits - BA
-** 21 bits - BBA
-** and so on.
-*/
-
-/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
-#define VARINT_MAX 10
-
-/* Write a 64-bit variable-length integer to memory starting at p[0].
- * The length of data written will be between 1 and VARINT_MAX bytes.
- * The number of bytes written is returned. */
-static int putVarint(char *p, sqlite_int64 v){
-  unsigned char *q = (unsigned char *) p;
-  sqlite_uint64 vu = v;
-  do{
-    *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
-    vu >>= 7;
-  }while( vu!=0 );
-  q[-1] &= 0x7f;  /* turn off high bit in final byte */
-  assert( q - (unsigned char *)p <= VARINT_MAX );
-  return (int) (q - (unsigned char *)p);
-}
-
-/* Read a 64-bit variable-length integer from memory starting at p[0].
- * Return the number of bytes read, or 0 on error.
- * The value is stored in *v. */
-static int getVarint(const char *p, sqlite_int64 *v){
-  const unsigned char *q = (const unsigned char *) p;
-  sqlite_uint64 x = 0, y = 1;
-  while( (*q & 0x80) == 0x80 ){
-    x += y * (*q++ & 0x7f);
-    y <<= 7;
-    if( q - (unsigned char *)p >= VARINT_MAX ){  /* bad data */
-      assert( 0 );
-      return 0;
-    }
-  }
-  x += y * (*q++);
-  *v = (sqlite_int64) x;
-  return (int) (q - (unsigned char *)p);
-}
-
-static int getVarint32(const char *p, int *pi){
- sqlite_int64 i;
- int ret = getVarint(p, &i);
- *pi = (int) i;
- assert( *pi==i );
- return ret;
-}
-
-/*** Document lists ***
- *
- * A document list holds a sorted list of varint-encoded document IDs.
- *
- * A doclist with type DL_POSITIONS_OFFSETS is stored like this:
- *
- * array {
- *   varint docid;
- *   array {
- *     varint position;     (delta from previous position plus 1, or 0 for end)
- *     varint startOffset;  (delta from previous startOffset)
- *     varint endOffset;    (delta from startOffset)
- *   }
- * }
- *
- * Here, array { X } means zero or more occurrences of X, adjacent in memory.
- *
- * A doclist with type DL_POSITIONS is like the above, but holds only docids
- * and positions without offset information.
- *
- * A doclist with type DL_DOCIDS is like the above, but holds only docids
- * without positions or offset information.
- *
- * On disk, every document list has positions and offsets, so we don't bother
- * to serialize a doclist's type.
- * 
- * We don't yet delta-encode document IDs; doing so will probably be a
- * modest win.
- *
- * NOTE(shess) I've thought of a slightly (1%) better offset encoding.
- * After the first offset, estimate the next offset by using the
- * current token position and the previous token position and offset,
- * offset to handle some variance.  So the estimate would be
- * (iPosition*w->iStartOffset/w->iPosition-64), which is delta-encoded
- * as normal.  Offsets more than 64 chars from the estimate are
- * encoded as the delta to the previous start offset + 128.  An
- * additional tiny increment can be gained by using the end offset of
- * the previous token to make the estimate a tiny bit more precise.
-*/
-
-typedef enum DocListType {
-  DL_DOCIDS,              /* docids only */
-  DL_POSITIONS,           /* docids + positions */
-  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
-} DocListType;
-
-typedef struct DocList {
-  char *pData;
-  int nData;
-  DocListType iType;
-  int iLastPos;       /* the last position written */
-  int iLastOffset;    /* the last start offset written */
-} DocList;
-
-/* Initialize a new DocList to hold the given data. */
-static void docListInit(DocList *d, DocListType iType,
-                        const char *pData, int nData){
-  d->nData = nData;
-  if( nData>0 ){
-    d->pData = malloc(nData);
-    memcpy(d->pData, pData, nData);
-  } else {
-    d->pData = NULL;
-  }
-  d->iType = iType;
-  d->iLastPos = 0;
-  d->iLastOffset = 0;
-}
-
-/* Create a new dynamically-allocated DocList. */
-static DocList *docListNew(DocListType iType){
-  DocList *d = (DocList *) malloc(sizeof(DocList));
-  docListInit(d, iType, 0, 0);
-  return d;
-}
-
-static void docListDestroy(DocList *d){
-  free(d->pData);
-#ifndef NDEBUG
-  memset(d, 0x55, sizeof(*d));
-#endif
-}
-
-static void docListDelete(DocList *d){
-  docListDestroy(d);
-  free(d);
-}
-
-static char *docListEnd(DocList *d){
-  return d->pData + d->nData;
-}
-
-/* Append a varint to a DocList's data. */
-static void appendVarint(DocList *d, sqlite_int64 i){
-  char c[VARINT_MAX];
-  int n = putVarint(c, i);
-  d->pData = realloc(d->pData, d->nData + n);
-  memcpy(d->pData + d->nData, c, n);
-  d->nData += n;
-}
-
-static void docListAddDocid(DocList *d, sqlite_int64 iDocid){
-  appendVarint(d, iDocid);
-  d->iLastPos = 0;
-}
-
-/* Add a position to the last position list in a doclist. */
-static void docListAddPos(DocList *d, int iPos){
-  assert( d->iType>=DL_POSITIONS );
-  appendVarint(d, iPos-d->iLastPos+1);
-  d->iLastPos = iPos;
-}
-
-static void docListAddPosOffset(DocList *d, int iPos,
-                                int iStartOffset, int iEndOffset){
-  assert( d->iType==DL_POSITIONS_OFFSETS );
-  docListAddPos(d, iPos);
-  appendVarint(d, iStartOffset-d->iLastOffset);
-  d->iLastOffset = iStartOffset;
-  appendVarint(d, iEndOffset-iStartOffset);
-}
-
-/* Terminate the last position list in the given doclist. */
-static void docListAddEndPos(DocList *d){
-  appendVarint(d, 0);
-}
-
-typedef struct DocListReader {
-  DocList *pDoclist;
-  char *p;
-  int iLastPos;    /* the last position read */
-} DocListReader;
-
-static void readerInit(DocListReader *r, DocList *pDoclist){
-  r->pDoclist = pDoclist;
-  if( pDoclist!=NULL ){
-    r->p = pDoclist->pData;
-  }
-  r->iLastPos = 0;
-}
-
-static int readerAtEnd(DocListReader *pReader){
-  return pReader->p >= docListEnd(pReader->pDoclist);
-}
-
-/* Peek at the next docid without advancing the read pointer. */
-static sqlite_int64 peekDocid(DocListReader *pReader){
-  sqlite_int64 ret;
-  assert( !readerAtEnd(pReader) );
-  getVarint(pReader->p, &ret);
-  return ret;
-}
-
-/* Read the next docid. */
-static sqlite_int64 readDocid(DocListReader *pReader){
-  sqlite_int64 ret;
-  assert( !readerAtEnd(pReader) );
-  pReader->p += getVarint(pReader->p, &ret);
-  pReader->iLastPos = 0;
-  return ret;
-}
-
-/* Read the next position from a position list.
- * Returns the position, or -1 at the end of the list. */
-static int readPosition(DocListReader *pReader){
-  int i;
-  int iType = pReader->pDoclist->iType;
-  assert( iType>=DL_POSITIONS );
-  assert( !readerAtEnd(pReader) );
-
-  pReader->p += getVarint32(pReader->p, &i);
-  if( i==0 ){
-    pReader->iLastPos = -1;
-    return -1;
-  }
-  pReader->iLastPos += ((int) i)-1;
-  if( iType>=DL_POSITIONS_OFFSETS ){
-    /* Skip over offsets, ignoring them for now. */
-    int iStart, iEnd;
-    pReader->p += getVarint32(pReader->p, &iStart);
-    pReader->p += getVarint32(pReader->p, &iEnd);
-  }
-  return pReader->iLastPos;
-}
-
-/* Skip past the end of a position list. */
-static void skipPositionList(DocListReader *pReader){
-  while( readPosition(pReader)!=-1 )
-    ;
-}
-
-/* Skip over a docid, including its position list if the doclist has
- * positions. */
-static void skipDocument(DocListReader *pReader){
-  readDocid(pReader);
-  if( pReader->pDoclist->iType >= DL_POSITIONS ){
-    skipPositionList(pReader);
-  }
-}
-
-static sqlite_int64 firstDocid(DocList *d){
-  DocListReader r;
-  readerInit(&r, d);
-  return readDocid(&r);
-}
-
-/* Doclist multi-tool.  Pass pUpdate==NULL to delete the indicated docid;
- * otherwise pUpdate, which must contain only the single docid [iDocid], is
- * inserted (if not present) or updated (if already present). */
-static int docListUpdate(DocList *d, sqlite_int64 iDocid, DocList *pUpdate){
-  int modified = 0;
-  DocListReader reader;
-  char *p;
-
-  if( pUpdate!=NULL ){
-    assert( d->iType==pUpdate->iType);
-    assert( iDocid==firstDocid(pUpdate) );
-  }
-
-  readerInit(&reader, d);
-  while( !readerAtEnd(&reader) && peekDocid(&reader)<iDocid ){
-    skipDocument(&reader);
-  }
-
-  p = reader.p;
-  /* Delete if there is a matching element. */
-  if( !readerAtEnd(&reader) && iDocid==peekDocid(&reader) ){
-    skipDocument(&reader);
-    memmove(p, reader.p, docListEnd(d) - reader.p);
-    d->nData -= (reader.p - p);
-    modified = 1;
-  }
-
-  /* Insert if indicated. */
-  if( pUpdate!=NULL ){
-    int iDoclist = p-d->pData;
-    docListAddEndPos(pUpdate);
-
-    d->pData = realloc(d->pData, d->nData+pUpdate->nData);
-    p = d->pData + iDoclist;
-
-    memmove(p+pUpdate->nData, p, docListEnd(d) - p);
-    memcpy(p, pUpdate->pData, pUpdate->nData);
-    d->nData += pUpdate->nData;
-    modified = 1;
-  }
-
-  return modified;
-}
-
-/* Split the second half of doclist d into a separate doclist d2.  Returns 1
- * if successful, or 0 if d contains a single document and hence can't be
- * split. */
-static int docListSplit(DocList *d, DocList *d2){
-  const char *pSplitPoint = d->pData + d->nData / 2;
-  DocListReader reader;
-
-  readerInit(&reader, d);
-  while( reader.p<pSplitPoint ){
-    skipDocument(&reader);
-  }
-  if( readerAtEnd(&reader) ) return 0;
-  docListInit(d2, d->iType, reader.p, docListEnd(d) - reader.p);
-  d->nData = reader.p - d->pData;
-  d->pData = realloc(d->pData, d->nData);
-  return 1;
-}
-
-/* A DocListMerge computes the AND of an in-memory DocList [in] and a chunked
- * on-disk doclist, resulting in another in-memory DocList [out].  [in]
- * and [out] may or may not store position information according to the
- * caller's wishes.  The on-disk doclist always comes with positions.
- *
- * The caller must read each chunk of the on-disk doclist in succession and
- * pass it to mergeBlock().
- *
- * If [in] has positions, then the merge output contains only documents with
- * matching positions in the two input doclists.  If [in] does not have
- * positions, then the merge output contains all documents common to the two
- * input doclists.
- *
- * If [in] is NULL, then the on-disk doclist is copied to [out] directly.
- *
- * A merge is performed using an integer [iOffset] provided by the caller.
- * [iOffset] is subtracted from each position in the on-disk doclist for the
- * purpose of position comparison; this is helpful in implementing phrase
- * searches.
- *
- * A DocListMerge is not yet able to propagate offsets through query
- * processing; we should add that capability soon.
-*/
-typedef struct DocListMerge {
-  DocListReader in;
-  DocList *pOut;
-  int iOffset;
-} DocListMerge;
-
-static void mergeInit(DocListMerge *m,
-                      DocList *pIn, int iOffset, DocList *pOut){
-  readerInit(&m->in, pIn);
-  m->pOut = pOut;
-  m->iOffset = iOffset;
-
-  /* can't handle offsets yet */
-  assert( pIn==NULL || pIn->iType <= DL_POSITIONS );
-  assert( pOut->iType <= DL_POSITIONS );
-}
-
-/* A helper function for mergeBlock(), below.  Merge the position lists
- * pointed to by m->in and pBlockReader.
- * If the merge matches, write [iDocid] to m->pOut; if m->pOut
- * has positions then write all matching positions as well. */
-static void mergePosList(DocListMerge *m, sqlite_int64 iDocid,
-                  DocListReader *pBlockReader){
-  int block_pos = readPosition(pBlockReader);
-  int in_pos = readPosition(&m->in);
-  int match = 0;
-  while( block_pos!=-1 || in_pos!=-1 ){
-    if( block_pos-m->iOffset==in_pos ){
-      if( !match ){
-        docListAddDocid(m->pOut, iDocid);
-        match = 1;
-      }
-      if( m->pOut->iType >= DL_POSITIONS ){
-        docListAddPos(m->pOut, in_pos);
-      }
-      block_pos = readPosition(pBlockReader);
-      in_pos = readPosition(&m->in);
-    } else if( in_pos==-1 || (block_pos!=-1 && block_pos-m->iOffset<in_pos) ){
-      block_pos = readPosition(pBlockReader);
-    } else {
-      in_pos = readPosition(&m->in);
-    }
-  }
-  if( m->pOut->iType >= DL_POSITIONS && match ){
-    docListAddEndPos(m->pOut);
-  }
-}
-
-/* Merge one block of an on-disk doclist into a DocListMerge. */
-static void mergeBlock(DocListMerge *m, DocList *pBlock){
-  DocListReader blockReader;
-  assert( pBlock->iType >= DL_POSITIONS );
-  readerInit(&blockReader, pBlock);
-  while( !readerAtEnd(&blockReader) ){
-    sqlite_int64 iDocid = readDocid(&blockReader);
-    if( m->in.pDoclist!=NULL ){
-      while( 1 ){
-        if( readerAtEnd(&m->in) ) return;  /* nothing more to merge */
-        if( peekDocid(&m->in)>=iDocid ) break;
-        skipDocument(&m->in);
-      }
-      if( peekDocid(&m->in)>iDocid ){  /* [pIn] has no match with iDocid */
-        skipPositionList(&blockReader);  /* skip this docid in the block */
-        continue;
-      }
-      readDocid(&m->in);
-    }
-    /* We have a document match. */
-    if( m->in.pDoclist==NULL || m->in.pDoclist->iType < DL_POSITIONS ){
-      /* We don't need to do a poslist merge. */
-      docListAddDocid(m->pOut, iDocid);
-      if( m->pOut->iType >= DL_POSITIONS ){
-        /* Copy all positions to the output doclist. */
-        while( 1 ){
-          int pos = readPosition(&blockReader);
-          if( pos==-1 ) break;
-          docListAddPos(m->pOut, pos);
-        }
-        docListAddEndPos(m->pOut);
-      } else skipPositionList(&blockReader);
-      continue;
-    }
-    mergePosList(m, iDocid, &blockReader);
-  }
-}
-
-static char *string_dup_n(const char *s, int n){
-  char *str = malloc(n + 1);
-  memcpy(str, s, n);
-  str[n] = '\0';
-  return str;
-}
-
-/* Duplicate a string; the caller must free() the returned string.
- * (We don't use strdup() since it's not part of the standard C library and
- * may not be available everywhere.) */
-static char *string_dup(const char *s){
-  return string_dup_n(s, strlen(s));
-}
-
-/* Format a string, replacing each occurrence of the % character with
- * zName.  This may be more convenient than sqlite_mprintf()
- * when one string is used repeatedly in a format string.
- * The caller must free() the returned string. */
-static char *string_format(const char *zFormat, const char *zName){
-  const char *p;
-  size_t len = 0;
-  size_t nName = strlen(zName);
-  char *result;
-  char *r;
-
-  /* first compute length needed */
-  for(p = zFormat ; *p ; ++p){
-    len += (*p=='%' ? nName : 1);
-  }
-  len += 1;  /* for null terminator */
-
-  r = result = malloc(len);
-  for(p = zFormat; *p; ++p){
-    if( *p=='%' ){
-      memcpy(r, zName, nName);
-      r += nName;
-    } else {
-      *r++ = *p;
-    }
-  }
-  *r++ = '\0';
-  assert( r == result + len );
-  return result;
-}
-
-static int sql_exec(sqlite3 *db, const char *zName, const char *zFormat){
-  char *zCommand = string_format(zFormat, zName);
-  int rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
-  free(zCommand);
-  return rc;
-}
-
-static int sql_prepare(sqlite3 *db, const char *zName, sqlite3_stmt **ppStmt,
-                const char *zFormat){
-  char *zCommand = string_format(zFormat, zName);
-  int rc = sqlite3_prepare(db, zCommand, -1, ppStmt, NULL);
-  free(zCommand);
-  return rc;
-}
-
-/* end utility functions */
-
-#define QUERY_GENERIC 0
-#define QUERY_FULLTEXT 1
-
-#define CHUNK_MAX 1024
-
-typedef enum fulltext_statement {
-  CONTENT_INSERT_STMT,
-  CONTENT_SELECT_STMT,
-  CONTENT_DELETE_STMT,
-
-  TERM_SELECT_STMT,
-  TERM_CHUNK_SELECT_STMT,
-  TERM_INSERT_STMT,
-  TERM_UPDATE_STMT,
-  TERM_DELETE_STMT,
-
-  MAX_STMT                     /* Always at end! */
-} fulltext_statement;
-
-/* These must exactly match the enum above. */
-/* TODO(adam): Is there some risk that a statement (in particular,
-** pTermSelectStmt) will be used in two cursors at once, e.g.  if a
-** query joins a virtual table to itself?  If so perhaps we should
-** move some of these to the cursor object.
-*/
-static const char *fulltext_zStatement[MAX_STMT] = {
-  /* CONTENT_INSERT */ "insert into %_content (rowid, content) values (?, ?)",
-  /* CONTENT_SELECT */ "select content from %_content where rowid = ?",
-  /* CONTENT_DELETE */ "delete from %_content where rowid = ?",
-
-  /* TERM_SELECT */
-  "select rowid, doclist from %_term where term = ? and first = ?",
-  /* TERM_CHUNK_SELECT */
-  "select max(first) from %_term where term = ? and first <= ?",
-  /* TERM_INSERT */
-  "insert into %_term (term, first, doclist) values (?, ?, ?)",
-  /* TERM_UPDATE */ "update %_term set doclist = ? where rowid = ?",
-  /* TERM_DELETE */ "delete from %_term where rowid = ?",
-};
-
-typedef struct fulltext_vtab {
-  sqlite3_vtab base;
-  sqlite3 *db;
-  const char *zName;               /* virtual table name */
-  sqlite3_tokenizer *pTokenizer;   /* tokenizer for inserts and queries */
-
-  /* Precompiled statements which we keep as long as the table is
-  ** open.
-  */
-  sqlite3_stmt *pFulltextStatements[MAX_STMT];
-} fulltext_vtab;
-
-typedef struct fulltext_cursor {
-  sqlite3_vtab_cursor base;
-  int iCursorType;  /* QUERY_GENERIC or QUERY_FULLTEXT */
-
-  sqlite3_stmt *pStmt;
-
-  int eof;
-
-  /* The following is used only when iCursorType == QUERY_FULLTEXT. */
-  DocListReader result;
-} fulltext_cursor;
-
-static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
-  return (fulltext_vtab *) c->base.pVtab;
-}
-
-static sqlite3_module fulltextModule;   /* forward declaration */
-
-/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
-** If the indicated statement has never been prepared, it is prepared
-** and cached, otherwise the cached version is reset.
-*/
-static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
-                             sqlite3_stmt **ppStmt){
-  assert( iStmt<MAX_STMT );
-  if( v->pFulltextStatements[iStmt]==NULL ){
-    int rc = sql_prepare(v->db, v->zName, &v->pFulltextStatements[iStmt],
-                         fulltext_zStatement[iStmt]);
-    if( rc!=SQLITE_OK ) return rc;
-  } else {
-    int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-
-  *ppStmt = v->pFulltextStatements[iStmt];
-  return SQLITE_OK;
-}
-
-/* Step the indicated statement, handling errors SQLITE_BUSY (by
-** retrying) and SQLITE_SCHEMA (by re-preparing and transferring
-** bindings to the new statement).
-** TODO(adam): We should extend this function so that it can work with
-** statements declared locally, not only globally cached statements.
-*/
-static int sql_step_statement(fulltext_vtab *v, fulltext_statement iStmt,
-                              sqlite3_stmt **ppStmt){
-  int rc;
-  sqlite3_stmt *s = *ppStmt;
-  assert( iStmt<MAX_STMT );
-  assert( s==v->pFulltextStatements[iStmt] );
-
-  while( (rc=sqlite3_step(s))!=SQLITE_DONE && rc!=SQLITE_ROW ){
-    sqlite3_stmt *pNewStmt;
-
-    if( rc==SQLITE_BUSY ) continue;
-    if( rc!=SQLITE_ERROR ) return rc;
-
-    rc = sqlite3_reset(s);
-    if( rc!=SQLITE_SCHEMA ) return SQLITE_ERROR;
-
-    v->pFulltextStatements[iStmt] = NULL;   /* Still in s */
-    rc = sql_get_statement(v, iStmt, &pNewStmt);
-    if( rc!=SQLITE_OK ) goto err;
-    *ppStmt = pNewStmt;
-
-    rc = sqlite3_transfer_bindings(s, pNewStmt);
-    if( rc!=SQLITE_OK ) goto err;
-
-    rc = sqlite3_finalize(s);
-    if( rc!=SQLITE_OK ) return rc;
-    s = pNewStmt;
-  }
-  return rc;
-
- err:
-  sqlite3_finalize(s);
-  return rc;
-}
-
-/* Like sql_step_statement(), but convert SQLITE_DONE to SQLITE_OK.
-** Useful for statements like UPDATE, where we expect no results.
-*/
-static int sql_single_step_statement(fulltext_vtab *v,
-                                     fulltext_statement iStmt,
-                                     sqlite3_stmt **ppStmt){
-  int rc = sql_step_statement(v, iStmt, ppStmt);
-  return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
-}
-
-/* insert into %_content (rowid, content) values ([rowid], [zContent]) */
-static int content_insert(fulltext_vtab *v, sqlite3_value *rowid,
-                          const char *zContent, int nContent){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_value(s, 1, rowid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_text(s, 2, zContent, nContent, SQLITE_STATIC);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step_statement(v, CONTENT_INSERT_STMT, &s);
-}
-
-/* select content from %_content where rowid = [iRow]
- * The caller must delete the returned string. */
-static int content_select(fulltext_vtab *v, sqlite_int64 iRow,
-                          char **pzContent){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, iRow);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sql_step_statement(v, CONTENT_SELECT_STMT, &s);
-  if( rc!=SQLITE_ROW ) return rc;
-
-  *pzContent = string_dup((const char *)sqlite3_column_text(s, 0));
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_DONE ) return SQLITE_OK;
-
-  free(*pzContent);
-  return rc;
-}
-
-/* delete from %_content where rowid = [iRow ] */
-static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, iRow);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step_statement(v, CONTENT_DELETE_STMT, &s);
-}
-
-/* select rowid, doclist from %_term where term = [zTerm] and first = [iFirst]
- * If found, returns SQLITE_OK; the caller must free the returned doclist.
- * If no rows found, returns SQLITE_ERROR. */
-static int term_select(fulltext_vtab *v, const char *zTerm, int nTerm,
-                       sqlite_int64 iFirst,
-                       sqlite_int64 *rowid,
-                       DocList *out){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, TERM_SELECT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_TRANSIENT);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 2, iFirst);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sql_step_statement(v, TERM_SELECT_STMT, &s);
-  if( rc!=SQLITE_ROW ) return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
-
-  *rowid = sqlite3_column_int64(s, 0);
-  docListInit(out, DL_POSITIONS_OFFSETS,
-              sqlite3_column_blob(s, 1), sqlite3_column_bytes(s, 1));
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  rc = sqlite3_step(s);
-  return rc==SQLITE_DONE ? SQLITE_OK : rc;
-}
-
-/* select max(first) from %_term where term = [zTerm] and first <= [iFirst]
- * If found, returns SQLITE_ROW and result in *piResult; if the query returns
- * NULL (meaning no row found) returns SQLITE_DONE.
- */
-static int term_chunk_select(fulltext_vtab *v, const char *zTerm, int nTerm,
-                           sqlite_int64 iFirst, sqlite_int64 *piResult){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, TERM_CHUNK_SELECT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_STATIC);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 2, iFirst);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sql_step_statement(v, TERM_CHUNK_SELECT_STMT, &s);
-  if( rc!=SQLITE_ROW ) return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
-
-  switch( sqlite3_column_type(s, 0) ){
-    case SQLITE_NULL:
-      rc = SQLITE_DONE;
-      break;
-    case SQLITE_INTEGER:
-     *piResult = sqlite3_column_int64(s, 0);
-     break;
-    default:
-      return SQLITE_ERROR;
-  }
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  if( sqlite3_step(s) != SQLITE_DONE ) return SQLITE_ERROR;
-  return rc;
-}
-
-/* insert into %_term (term, first, doclist)
-               values ([zTerm], [iFirst], [doclist]) */
-static int term_insert(fulltext_vtab *v, const char *zTerm, int nTerm,
-                       sqlite_int64 iFirst, DocList *doclist){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, TERM_INSERT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_STATIC);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 2, iFirst);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_blob(s, 3, doclist->pData, doclist->nData, SQLITE_STATIC);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step_statement(v, TERM_INSERT_STMT, &s);
-}
-
-/* update %_term set doclist = [doclist] where rowid = [rowid] */
-static int term_update(fulltext_vtab *v, sqlite_int64 rowid,
-                       DocList *doclist){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, TERM_UPDATE_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_blob(s, 1, doclist->pData, doclist->nData,
-                         SQLITE_STATIC);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 2, rowid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step_statement(v, TERM_UPDATE_STMT, &s);
-}
-
-static int term_delete(fulltext_vtab *v, sqlite_int64 rowid){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, TERM_DELETE_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, rowid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step_statement(v, TERM_DELETE_STMT, &s);
-}
-
-static void fulltext_vtab_destroy(fulltext_vtab *v){
-  int iStmt;
-
-  for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
-    if( v->pFulltextStatements[iStmt]!=NULL ){
-      sqlite3_finalize(v->pFulltextStatements[iStmt]);
-      v->pFulltextStatements[iStmt] = NULL;
-    }
-  }
-
-  if( v->pTokenizer!=NULL ){
-    v->pTokenizer->pModule->xDestroy(v->pTokenizer);
-    v->pTokenizer = NULL;
-  }
-
-  free((void *) v->zName);
-  free(v);
-}
-
-/* Current interface:
-** argv[0] - module name
-** argv[1] - database name
-** argv[2] - table name
-** argv[3] - tokenizer name (optional, a sensible default is provided)
-** argv[4..] - passed to tokenizer (optional based on tokenizer)
-**/
-static int fulltextConnect(sqlite3 *db, void *pAux, int argc, char **argv,
-                           sqlite3_vtab **ppVTab){
-  int rc;
-  fulltext_vtab *v;
-  sqlite3_tokenizer_module *m = NULL;
-
-  assert( argc>=3 );
-  v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab));
-  /* sqlite will initialize v->base */
-  v->db = db;
-  v->zName = string_dup(argv[2]);
-  v->pTokenizer = NULL;
-
-  if( argc==3 ){
-    get_simple_tokenizer_module(&m);
-  } else {
-    /* TODO(shess) For now, add new tokenizers as else if clauses. */
-    if( !strcmp(argv[3], "simple") ){
-      get_simple_tokenizer_module(&m);
-    } else {
-      assert( "unrecognized tokenizer"==NULL );
-    }
-  }
-
-  /* TODO(shess) Since tokenization impacts the index, the parameters
-  ** to the tokenizer need to be identical when a persistent virtual
-  ** table is re-created.  One solution would be a meta-table to track
-  ** such information in the database.  Then we could verify that the
-  ** information is identical on subsequent creates.
-  */
-  /* TODO(shess) Why isn't argv already (const char **)? */
-  rc = m->xCreate(argc-3, (const char **) (argv+3), &v->pTokenizer);
-  if( rc!=SQLITE_OK ) return rc;
-  v->pTokenizer->pModule = m;
-
-  /* TODO: verify the existence of backing tables foo_content, foo_term */
-
-  rc = sqlite3_declare_vtab(db, "create table x(content text)");
-  if( rc!=SQLITE_OK ) return rc;
-
-  memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
-
-  *ppVTab = &v->base;
-  return SQLITE_OK;
-}
-
-static int fulltextCreate(sqlite3 *db, void *pAux, int argc, char **argv,
-                          sqlite3_vtab **ppVTab){
-  int rc;
-  assert( argc>=3 );
-
-  /* The %_content table holds the text of each full-text item, with
-  ** the rowid used as the docid.
-  **
-  ** The %_term table maps each term to a document list blob
-  ** containing elements sorted by ascending docid, each element
-  ** encoded as:
-  **
-  **   docid varint-encoded
-  **   token count varint-encoded
-  **   "count" token elements (poslist):
-  **     position varint-encoded as delta from previous position
-  **     start offset varint-encoded as delta from previous start offset
-  **     end offset varint-encoded as delta from start offset
-  **
-  ** Additionally, doclist blobs can be chunked into multiple rows,
-  ** using "first" to order the blobs.  "first" is simply the first
-  ** docid in the blob.
-  */
-  /*
-  ** NOTE(shess) That last sentence is incorrect in the face of
-  ** deletion, which can leave a doclist that doesn't contain the
-  ** first from that row.  I _believe_ this does not matter to the
-  ** operation of the system, but it might be reasonable to update
-  ** appropriately in case this assumption becomes more important.
-  */
-  rc = sql_exec(db, argv[2],
-    "create table %_content(content text);"
-    "create table %_term(term text, first integer, doclist blob);"
-    "create index %_index on %_term(term, first)");
-  if( rc!=SQLITE_OK ) return rc;
-
-  return fulltextConnect(db, pAux, argc, argv, ppVTab);
-}
-
-/* Decide how to handle an SQL query.
- * At the moment, MATCH queries can include implicit boolean ANDs; we
- * haven't implemented phrase searches or OR yet. */
-static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
-  int i;
-
-  for(i=0; i<pInfo->nConstraint; ++i){
-    const struct sqlite3_index_constraint *pConstraint;
-    pConstraint = &pInfo->aConstraint[i];
-    if( pConstraint->iColumn==0 &&
-        pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH &&
-        pConstraint->usable ){   /* a full-text search */
-      pInfo->aConstraintUsage[i].argvIndex = 1;
-      pInfo->aConstraintUsage[i].omit = 1;
-      pInfo->idxNum = QUERY_FULLTEXT;
-      pInfo->estimatedCost = 1.0;   /* an arbitrary value for now */
-      return SQLITE_OK;
-    }
-  }
-  pInfo->idxNum = QUERY_GENERIC;
-  return SQLITE_OK;
-}
-
-static int fulltextDisconnect(sqlite3_vtab *pVTab){
-  fulltext_vtab_destroy((fulltext_vtab *)pVTab);
-  return SQLITE_OK;
-}
-
-static int fulltextDestroy(sqlite3_vtab *pVTab){
-  fulltext_vtab *v = (fulltext_vtab *)pVTab;
-
-  int rc = sql_exec(v->db, v->zName,
-                    "drop table %_content; drop table %_term");
-  if( rc!=SQLITE_OK ) return rc;
-
-  fulltext_vtab_destroy((fulltext_vtab *)pVTab);
-  return SQLITE_OK;
-}
-
-static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
-  fulltext_cursor *c;
-
-  c = (fulltext_cursor *) calloc(sizeof(fulltext_cursor), 1);
-  /* sqlite will initialize c->base */
-  *ppCursor = &c->base;
-
-  return SQLITE_OK;
-}
-
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  sqlite3_finalize(c->pStmt);
-  if( c->result.pDoclist!=NULL ){
-    docListDelete(c->result.pDoclist);
-  }
-  free(c);
-  return SQLITE_OK;
-}
-
-static int fulltextNext(sqlite3_vtab_cursor *pCursor){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  sqlite_int64 iDocid;
-  int rc;
-
-  switch( c->iCursorType ){
-    case QUERY_GENERIC:
-      /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
-      rc = sqlite3_step(c->pStmt);
-      switch( rc ){
-        case SQLITE_ROW:
-          c->eof = 0;
-          return SQLITE_OK;
-        case SQLITE_DONE:
-          c->eof = 1;
-          return SQLITE_OK;
-        default:
-          c->eof = 1;
-          return rc;
-      }
-    case QUERY_FULLTEXT:
-      rc = sqlite3_reset(c->pStmt);
-      if( rc!=SQLITE_OK ) return rc;
-
-      if( readerAtEnd(&c->result)){
-        c->eof = 1;
-        return SQLITE_OK;
-      }
-      iDocid = readDocid(&c->result);
-      rc = sqlite3_bind_int64(c->pStmt, 1, iDocid);
-      if( rc!=SQLITE_OK ) return rc;
-      /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
-      rc = sqlite3_step(c->pStmt);
-      if( rc==SQLITE_ROW ){   /* the case we expect */
-        c->eof = 0;
-        return SQLITE_OK;
-      }
-      /* an error occurred; abort */
-      return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
-    default:
-      assert( 0 );
-      return SQLITE_ERROR;  /* not reached */
-  }
-}
-
-static int term_select_doclist(fulltext_vtab *v, const char *pTerm, int nTerm,
-                               sqlite3_stmt **ppStmt){
-  int rc;
-  if( *ppStmt ){
-    rc = sqlite3_reset(*ppStmt);
-  } else {
-    rc = sql_prepare(v->db, v->zName, ppStmt,
-      "select doclist from %_term where term = ? order by first");
-  }
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_text(*ppStmt, 1, pTerm, nTerm, SQLITE_TRANSIENT);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sqlite3_step(*ppStmt);   /* TODO(adamd): handle schema error */
-}
-
-/* Read the posting list for [zTerm]; AND it with the doclist [in] to
- * produce the doclist [out], using the given offset [iOffset] for phrase
- * matching.
- * (*pSelect) is used to hold an SQLite statement used inside this function;
- * the caller should initialize *pSelect to NULL before the first call.
- */
-static int query_merge(fulltext_vtab *v, sqlite3_stmt **pSelect,
-                       const char *zTerm,
-                       DocList *pIn, int iOffset, DocList *out){
-  int rc;
-  DocListMerge merge;
-
-  if( pIn!=NULL && !pIn->nData ){
-    /* If [pIn] is already empty, there's no point in reading the
-     * posting list to AND it in; return immediately. */
-      return SQLITE_OK;
-  }
-
-  rc = term_select_doclist(v, zTerm, -1, pSelect);
-  if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
-
-  mergeInit(&merge, pIn, iOffset, out);
-  while( rc==SQLITE_ROW ){
-    DocList block;
-    docListInit(&block, DL_POSITIONS_OFFSETS,
-                sqlite3_column_blob(*pSelect, 0),
-                sqlite3_column_bytes(*pSelect, 0));
-    mergeBlock(&merge, &block);
-    docListDestroy(&block);
-
-    rc = sqlite3_step(*pSelect);
-    if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
-      return rc;
-    }
-  }
-  
-  return SQLITE_OK;
-}
-
-typedef struct QueryTerm {
-  int is_phrase;    /* true if this term begins a new phrase */
-  const char *zTerm;
-} QueryTerm;
-
-/* A parsed query.
- *
- * As an example, parsing the query ["four score" years "new nation"] will
- * yield a Query with 5 terms:
- *   "four",   is_phrase = 1
- *   "score",  is_phrase = 0
- *   "years",  is_phrase = 1
- *   "new",    is_phrase = 1
- *   "nation", is_phrase = 0
- */
-typedef struct Query {
-  int nTerms;
-  QueryTerm *pTerm;
-} Query;
-
-static void query_add(Query *q, int is_phrase, const char *zTerm){
-  QueryTerm *t;
-  ++q->nTerms;
-  q->pTerm = realloc(q->pTerm, q->nTerms * sizeof(q->pTerm[0]));
-  t = &q->pTerm[q->nTerms - 1];
-  t->is_phrase = is_phrase;
-  t->zTerm = zTerm;
-}
-    
-static void query_free(Query *q){
-  int i;
-  for(i = 0; i < q->nTerms; ++i){
-    free((void *) q->pTerm[i].zTerm);
-  }
-  free(q->pTerm);
-}
-
-static int tokenize_segment(sqlite3_tokenizer *pTokenizer,
-                            const char *zQuery, int in_phrase,
-                            Query *pQuery){
-  sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
-  sqlite3_tokenizer_cursor *pCursor;
-  int is_first = 1;
-  
-  int rc = pModule->xOpen(pTokenizer, zQuery, -1, &pCursor);
-  if( rc!=SQLITE_OK ) return rc;
-  pCursor->pTokenizer = pTokenizer;
-
-  while( 1 ){
-    const char *zToken;
-    int nToken, iStartOffset, iEndOffset, dummy_pos;
-
-    rc = pModule->xNext(pCursor,
-                        &zToken, &nToken,
-                        &iStartOffset, &iEndOffset,
-                        &dummy_pos);
-    if( rc!=SQLITE_OK ) break;
-    query_add(pQuery, !in_phrase || is_first, string_dup_n(zToken, nToken));
-    is_first = 0;
-  }
-
-  return pModule->xClose(pCursor);
-}
-
-/* Parse a query string, yielding a Query object. */
-static int parse_query(fulltext_vtab *v, const char *zQuery, Query *pQuery){
-  char *zQuery1 = string_dup(zQuery);
-  int in_phrase = 0;
-  char *s = zQuery1;
-  pQuery->nTerms = 0;
-  pQuery->pTerm = NULL;
-
-  while( *s ){
-    char *t = s;
-    while( *t ){
-      if( *t=='"' ){
-        *t++ = '\0';
-        break;
-      }
-      ++t;
-    }
-    if( *s ){
-      tokenize_segment(v->pTokenizer, s, in_phrase, pQuery);
-    }
-    s = t;
-    in_phrase = !in_phrase;
-  }
-  
-  free(zQuery1);
-  return SQLITE_OK;
-}
-
-/* Perform a full-text query; return a list of documents in [pResult]. */
-static int fulltext_query(fulltext_vtab *v, const char *zQuery,
-                          DocList **pResult){
-  Query q;
-  int phrase_start = -1;
-  int i;
-  sqlite3_stmt *pSelect = NULL;
-  DocList *d = NULL;
-
-  int rc = parse_query(v, zQuery, &q);
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* Merge terms. */
-  for(i = 0 ; i < q.nTerms ; ++i){
-    /* In each merge step, we need to generate positions whenever we're
-     * processing a phrase which hasn't ended yet. */
-    int need_positions = i<q.nTerms-1 && !q.pTerm[i+1].is_phrase;
-    DocList *next = docListNew(need_positions ? DL_POSITIONS : DL_DOCIDS);
-    if( q.pTerm[i].is_phrase ){
-      phrase_start = i;
-    }
-    rc = query_merge(v, &pSelect, q.pTerm[i].zTerm, d, i - phrase_start, next);
-    if( rc!=SQLITE_OK ) break;
-    if( d!=NULL ){
-      docListDelete(d);
-    }
-    d = next;
-  }
-
-  sqlite3_finalize(pSelect);
-  query_free(&q);
-  *pResult = d;
-  return rc;
-}
-
-static int fulltextFilter(sqlite3_vtab_cursor *pCursor,
-                          int idxNum, const char *idxStr,
-                          int argc, sqlite3_value **argv){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  fulltext_vtab *v = cursor_vtab(c);
-  int rc;
-  const char *zStatement;
-
-  c->iCursorType = idxNum;
-  switch( idxNum ){
-    case QUERY_GENERIC:
-      zStatement = "select rowid, content from %_content";
-      break;
-
-    case QUERY_FULLTEXT:   /* full-text search */
-    {
-      const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
-      DocList *pResult;
-      assert( argc==1 );
-      rc = fulltext_query(v, zQuery, &pResult);
-      if( rc!=SQLITE_OK ) return rc;
-      readerInit(&c->result, pResult);
-      zStatement = "select rowid, content from %_content where rowid = ?";
-      break;
-    }
-
-    default:
-      assert( 0 );
-  }
-
-  rc = sql_prepare(v->db, v->zName, &c->pStmt, zStatement);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return fulltextNext(pCursor);
-}
-
-static int fulltextEof(sqlite3_vtab_cursor *pCursor){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  return c->eof;
-}
-
-static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
-                          sqlite3_context *pContext, int idxCol){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  const char *s;
-
-  assert( idxCol==0 );
-  s = (const char *) sqlite3_column_text(c->pStmt, 1);
-  sqlite3_result_text(pContext, s, -1, SQLITE_TRANSIENT);
-
-  return SQLITE_OK;
-}
-
-static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-
-  *pRowid = sqlite3_column_int64(c->pStmt, 0);
-  return SQLITE_OK;
-}
-
-/* Build a hash table containing all terms in zText. */
-static int build_terms(Hash *terms, sqlite3_tokenizer *pTokenizer,
-                       const char *zText, sqlite_int64 iDocid){
-  sqlite3_tokenizer_cursor *pCursor;
-  const char *pToken;
-  int nTokenBytes;
-  int iStartOffset, iEndOffset, iPosition;
-
-  int rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
-  if( rc!=SQLITE_OK ) return rc;
-
-  pCursor->pTokenizer = pTokenizer;
-  HashInit(terms, HASH_STRING, 1);
-  while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
-                                               &pToken, &nTokenBytes,
-                                               &iStartOffset, &iEndOffset,
-                                               &iPosition) ){
-    DocList *p;
-
-    /* Positions can't be negative; we use -1 as a terminator internally. */
-    if( iPosition<0 ) {
-      rc = SQLITE_ERROR;  
-      goto err;
-    }
-
-    p = HashFind(terms, pToken, nTokenBytes);
-    if( p==NULL ){
-      p = docListNew(DL_POSITIONS_OFFSETS);
-      docListAddDocid(p, iDocid);
-      HashInsert(terms, pToken, nTokenBytes, p);
-    }
-    docListAddPosOffset(p, iPosition, iStartOffset, iEndOffset);
-  }
-
-err:
-  /* TODO(shess) Check return?  Should this be able to cause errors at
-  ** this point?  Actually, same question about sqlite3_finalize(),
-  ** though one could argue that failure there means that the data is
-  ** not durable.  *ponder*
-  */
-  pTokenizer->pModule->xClose(pCursor);
-  return rc;
-}
-/* Update the %_terms table to map the term [zTerm] to the given rowid. */
-static int index_insert_term(fulltext_vtab *v, const char *zTerm, int nTerm,
-                             sqlite_int64 iDocid, DocList *p){
-  sqlite_int64 iFirst;
-  sqlite_int64 iIndexRow;
-  DocList doclist;
-
-  int rc = term_chunk_select(v, zTerm, nTerm, iDocid, &iFirst);
-  if( rc==SQLITE_DONE ){
-    docListInit(&doclist, DL_POSITIONS_OFFSETS, 0, 0);
-    if( docListUpdate(&doclist, iDocid, p) ){
-      rc = term_insert(v, zTerm, nTerm, iDocid, &doclist);
-      docListDestroy(&doclist);
-      return rc;
-    }
-    return SQLITE_OK;
-  }
-  if( rc!=SQLITE_ROW ) return SQLITE_ERROR;
-
-  /* This word is in the index; add this document ID to its blob. */
-
-  rc = term_select(v, zTerm, nTerm, iFirst, &iIndexRow, &doclist);
-  if( rc!=SQLITE_OK ) return rc;
-
-  if( docListUpdate(&doclist, iDocid, p) ){
-    /* If the blob is too big, split it in half. */
-    if( doclist.nData>CHUNK_MAX ){
-      DocList half;
-      if( docListSplit(&doclist, &half) ){
-        rc = term_insert(v, zTerm, nTerm, firstDocid(&half), &half);
-        docListDestroy(&half);
-        if( rc!=SQLITE_OK ) goto err;
-      }
-    }
-    rc = term_update(v, iIndexRow, &doclist);
-  }
-
-err:
-  docListDestroy(&doclist);
-  return rc;
-}
-
-/* Insert a row into the full-text index; set *piRowid to be the ID of the
- * new row. */
-static int index_insert(fulltext_vtab *v,
-                        sqlite3_value *pRequestRowid, const char *zText,
-                        sqlite_int64 *piRowid){
-  Hash terms;  /* maps term string -> PosList */
-  HashElem *e;
-
-  int rc = content_insert(v, pRequestRowid, zText, -1);
-  if( rc!=SQLITE_OK ) return rc;
-  *piRowid = sqlite3_last_insert_rowid(v->db);
-
-  if( !zText ) return SQLITE_OK;   /* nothing to index */
-
-  rc = build_terms(&terms, v->pTokenizer, zText, *piRowid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  for(e=HashFirst(&terms); e; e=HashNext(e)){
-    DocList *p = HashData(e);
-    rc = index_insert_term(v, HashKey(e), HashKeysize(e), *piRowid, p);
-    if( rc!=SQLITE_OK ) break;
-  }
-
-  for(e=HashFirst(&terms); e; e=HashNext(e)){
-    DocList *p = HashData(e);
-    docListDelete(p);
-  }
-  HashClear(&terms);
-  return rc;
-}
-
-static int index_delete_term(fulltext_vtab *v, const char *zTerm, int nTerm,
-                             sqlite_int64 iDocid){
-  sqlite_int64 iFirst;
-  sqlite_int64 iIndexRow;
-  DocList doclist;
-
-  int rc = term_chunk_select(v, zTerm, nTerm, iDocid, &iFirst);
-  if( rc!=SQLITE_ROW ) return SQLITE_ERROR;
-
-  rc = term_select(v, zTerm, nTerm, iFirst, &iIndexRow, &doclist);
-  if( rc!=SQLITE_OK ) return rc;
-
-  if( docListUpdate(&doclist, iDocid, NULL) ){
-    if( doclist.nData>0 ){
-      rc = term_update(v, iIndexRow, &doclist);
-    } else {  /* empty posting list */
-      rc = term_delete(v, iIndexRow);
-    }
-  }
-  docListDestroy(&doclist);
-  return rc;
-}
-
-/* Delete a row from the full-text index. */
-static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
-  char *zText;
-  Hash terms;
-  HashElem *e;
-
-  int rc = content_select(v, iRow, &zText);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = build_terms(&terms, v->pTokenizer, zText, iRow);
-  free(zText);
-  if( rc!=SQLITE_OK ) return rc;
-
-  for(e=HashFirst(&terms); e; e=HashNext(e)){
-    rc = index_delete_term(v, HashKey(e), HashKeysize(e), iRow);
-    if( rc!=SQLITE_OK ) break;
-  }
-  for(e=HashFirst(&terms); e; e=HashNext(e)){
-    DocList *p = HashData(e);
-    docListDelete(p);
-  }
-  HashClear(&terms);
-
-  return content_delete(v, iRow);
-}
-
-static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
-                   sqlite_int64 *pRowid){
-  fulltext_vtab *v = (fulltext_vtab *) pVtab;
-
-  if( nArg<2 ){
-    return index_delete(v, sqlite3_value_int64(ppArg[0]));
-  }
-
-  if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
-    return SQLITE_ERROR;   /* an update; not yet supported */
-  }
-
-  assert( nArg==3 );    /* ppArg[1] = rowid, ppArg[2] = content */
-  return index_insert(v, ppArg[1],
-                      (const char *)sqlite3_value_text(ppArg[2]), pRowid);
-}
-
-static sqlite3_module fulltextModule = {
-  0,
-  fulltextCreate,
-  fulltextConnect,
-  fulltextBestIndex,
-  fulltextDisconnect,
-  fulltextDestroy,
-  fulltextOpen,
-  fulltextClose,
-  fulltextFilter,
-  fulltextNext,
-  fulltextEof,
-  fulltextColumn,
-  fulltextRowid,
-  fulltextUpdate
-};
-
-int fulltext_init(sqlite3 *db){
- return sqlite3_create_module(db, "fulltext", &fulltextModule, 0);
-}
-
-#if !SQLITE_CORE
-int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
-                           const sqlite3_api_routines *pApi){
- SQLITE_EXTENSION_INIT2(pApi)
- return fulltext_init(db);
-}
-#endif