| Index: third_party/sqlite/sqlite-src-3080704/ext/fts2/fts2.c
|
| diff --git a/third_party/sqlite/sqlite-src-3080704/ext/fts2/fts2.c b/third_party/sqlite/sqlite-src-3080704/ext/fts2/fts2.c
|
| new file mode 100644
|
| index 0000000000000000000000000000000000000000..0405fb7b1e56f9aefccebcc274131b973a16b768
|
| --- /dev/null
|
| +++ b/third_party/sqlite/sqlite-src-3080704/ext/fts2/fts2.c
|
| @@ -0,0 +1,6860 @@
|
| +/* fts2 has a design flaw which can lead to database corruption (see
|
| +** below). It is recommended not to use it any longer, instead use
|
| +** fts3 (or higher). If you believe that your use of fts2 is safe,
|
| +** add -DSQLITE_ENABLE_BROKEN_FTS2=1 to your CFLAGS.
|
| +*/
|
| +#if (!defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)) \
|
| + && !defined(SQLITE_ENABLE_BROKEN_FTS2)
|
| +#error fts2 has a design flaw and has been deprecated.
|
| +#endif
|
| +/* The flaw is that fts2 uses the content table's unaliased rowid as
|
| +** the unique docid. fts2 embeds the rowid in the index it builds,
|
| +** and expects the rowid to not change. The SQLite VACUUM operation
|
| +** will renumber such rowids, thereby breaking fts2. If you are using
|
| +** fts2 in a system which has disabled VACUUM, then you can continue
|
| +** to use it safely. Note that PRAGMA auto_vacuum does NOT disable
|
| +** VACUUM, though systems using auto_vacuum are unlikely to invoke
|
| +** VACUUM.
|
| +**
|
| +** Unlike fts1, which is safe across VACUUM if you never delete
|
| +** documents, fts2 has a second exposure to this flaw, in the segments
|
| +** table. So fts2 should be considered unsafe across VACUUM in all
|
| +** cases.
|
| +*/
|
| +
|
| +/*
|
| +** 2006 Oct 10
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** This is an SQLite module implementing full-text search.
|
| +*/
|
| +
|
| +/*
|
| +** The code in this file is only compiled if:
|
| +**
|
| +** * The FTS2 module is being built as an extension
|
| +** (in which case SQLITE_CORE is not defined), or
|
| +**
|
| +** * The FTS2 module is being built into the core of
|
| +** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
|
| +*/
|
| +
|
| +/* TODO(shess) Consider exporting this comment to an HTML file or the
|
| +** wiki.
|
| +*/
|
| +/* The full-text index is stored in a series of b+tree (-like)
|
| +** structures called segments which map terms to doclists. The
|
| +** structures are like b+trees in layout, but are constructed from the
|
| +** bottom up in optimal fashion and are not updatable. Since trees
|
| +** are built from the bottom up, things will be described from the
|
| +** bottom up.
|
| +**
|
| +**
|
| +**** Varints ****
|
| +** The basic unit of encoding is a variable-length integer called a
|
| +** varint. We encode variable-length integers in little-endian order
|
| +** using seven bits * per byte as follows:
|
| +**
|
| +** KEY:
|
| +** A = 0xxxxxxx 7 bits of data and one flag bit
|
| +** B = 1xxxxxxx 7 bits of data and one flag bit
|
| +**
|
| +** 7 bits - A
|
| +** 14 bits - BA
|
| +** 21 bits - BBA
|
| +** and so on.
|
| +**
|
| +** This is identical to how sqlite encodes varints (see util.c).
|
| +**
|
| +**
|
| +**** Document lists ****
|
| +** A doclist (document list) holds a docid-sorted list of hits for a
|
| +** given term. Doclists hold docids, and can optionally associate
|
| +** token positions and offsets with docids.
|
| +**
|
| +** A DL_POSITIONS_OFFSETS doclist is stored like this:
|
| +**
|
| +** array {
|
| +** varint docid;
|
| +** array { (position list for column 0)
|
| +** varint position; (delta from previous position plus POS_BASE)
|
| +** varint startOffset; (delta from previous startOffset)
|
| +** varint endOffset; (delta from startOffset)
|
| +** }
|
| +** array {
|
| +** varint POS_COLUMN; (marks start of position list for new column)
|
| +** varint column; (index of new column)
|
| +** array {
|
| +** varint position; (delta from previous position plus POS_BASE)
|
| +** varint startOffset;(delta from previous startOffset)
|
| +** varint endOffset; (delta from startOffset)
|
| +** }
|
| +** }
|
| +** varint POS_END; (marks end of positions for this document.
|
| +** }
|
| +**
|
| +** Here, array { X } means zero or more occurrences of X, adjacent in
|
| +** memory. A "position" is an index of a token in the token stream
|
| +** generated by the tokenizer, while an "offset" is a byte offset,
|
| +** both based at 0. Note that POS_END and POS_COLUMN occur in the
|
| +** same logical place as the position element, and act as sentinals
|
| +** ending a position list array.
|
| +**
|
| +** A DL_POSITIONS doclist omits the startOffset and endOffset
|
| +** information. A DL_DOCIDS doclist omits both the position and
|
| +** offset information, becoming an array of varint-encoded docids.
|
| +**
|
| +** On-disk data is stored as type DL_DEFAULT, so we don't serialize
|
| +** the type. Due to how deletion is implemented in the segmentation
|
| +** system, on-disk doclists MUST store at least positions.
|
| +**
|
| +**
|
| +**** Segment leaf nodes ****
|
| +** Segment leaf nodes store terms and doclists, ordered by term. Leaf
|
| +** nodes are written using LeafWriter, and read using LeafReader (to
|
| +** iterate through a single leaf node's data) and LeavesReader (to
|
| +** iterate through a segment's entire leaf layer). Leaf nodes have
|
| +** the format:
|
| +**
|
| +** varint iHeight; (height from leaf level, always 0)
|
| +** varint nTerm; (length of first term)
|
| +** char pTerm[nTerm]; (content of first term)
|
| +** varint nDoclist; (length of term's associated doclist)
|
| +** char pDoclist[nDoclist]; (content of doclist)
|
| +** array {
|
| +** (further terms are delta-encoded)
|
| +** varint nPrefix; (length of prefix shared with previous term)
|
| +** varint nSuffix; (length of unshared suffix)
|
| +** char pTermSuffix[nSuffix];(unshared suffix of next term)
|
| +** varint nDoclist; (length of term's associated doclist)
|
| +** char pDoclist[nDoclist]; (content of doclist)
|
| +** }
|
| +**
|
| +** Here, array { X } means zero or more occurrences of X, adjacent in
|
| +** memory.
|
| +**
|
| +** Leaf nodes are broken into blocks which are stored contiguously in
|
| +** the %_segments table in sorted order. This means that when the end
|
| +** of a node is reached, the next term is in the node with the next
|
| +** greater node id.
|
| +**
|
| +** New data is spilled to a new leaf node when the current node
|
| +** exceeds LEAF_MAX bytes (default 2048). New data which itself is
|
| +** larger than STANDALONE_MIN (default 1024) is placed in a standalone
|
| +** node (a leaf node with a single term and doclist). The goal of
|
| +** these settings is to pack together groups of small doclists while
|
| +** making it efficient to directly access large doclists. The
|
| +** assumption is that large doclists represent terms which are more
|
| +** likely to be query targets.
|
| +**
|
| +** TODO(shess) It may be useful for blocking decisions to be more
|
| +** dynamic. For instance, it may make more sense to have a 2.5k leaf
|
| +** node rather than splitting into 2k and .5k nodes. My intuition is
|
| +** that this might extend through 2x or 4x the pagesize.
|
| +**
|
| +**
|
| +**** Segment interior nodes ****
|
| +** Segment interior nodes store blockids for subtree nodes and terms
|
| +** to describe what data is stored by the each subtree. Interior
|
| +** nodes are written using InteriorWriter, and read using
|
| +** InteriorReader. InteriorWriters are created as needed when
|
| +** SegmentWriter creates new leaf nodes, or when an interior node
|
| +** itself grows too big and must be split. The format of interior
|
| +** nodes:
|
| +**
|
| +** varint iHeight; (height from leaf level, always >0)
|
| +** varint iBlockid; (block id of node's leftmost subtree)
|
| +** optional {
|
| +** varint nTerm; (length of first term)
|
| +** char pTerm[nTerm]; (content of first term)
|
| +** array {
|
| +** (further terms are delta-encoded)
|
| +** varint nPrefix; (length of shared prefix with previous term)
|
| +** varint nSuffix; (length of unshared suffix)
|
| +** char pTermSuffix[nSuffix]; (unshared suffix of next term)
|
| +** }
|
| +** }
|
| +**
|
| +** Here, optional { X } means an optional element, while array { X }
|
| +** means zero or more occurrences of X, adjacent in memory.
|
| +**
|
| +** An interior node encodes n terms separating n+1 subtrees. The
|
| +** subtree blocks are contiguous, so only the first subtree's blockid
|
| +** is encoded. The subtree at iBlockid will contain all terms less
|
| +** than the first term encoded (or all terms if no term is encoded).
|
| +** Otherwise, for terms greater than or equal to pTerm[i] but less
|
| +** than pTerm[i+1], the subtree for that term will be rooted at
|
| +** iBlockid+i. Interior nodes only store enough term data to
|
| +** distinguish adjacent children (if the rightmost term of the left
|
| +** child is "something", and the leftmost term of the right child is
|
| +** "wicked", only "w" is stored).
|
| +**
|
| +** New data is spilled to a new interior node at the same height when
|
| +** the current node exceeds INTERIOR_MAX bytes (default 2048).
|
| +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing
|
| +** interior nodes and making the tree too skinny. The interior nodes
|
| +** at a given height are naturally tracked by interior nodes at
|
| +** height+1, and so on.
|
| +**
|
| +**
|
| +**** Segment directory ****
|
| +** The segment directory in table %_segdir stores meta-information for
|
| +** merging and deleting segments, and also the root node of the
|
| +** segment's tree.
|
| +**
|
| +** The root node is the top node of the segment's tree after encoding
|
| +** the entire segment, restricted to ROOT_MAX bytes (default 1024).
|
| +** This could be either a leaf node or an interior node. If the top
|
| +** node requires more than ROOT_MAX bytes, it is flushed to %_segments
|
| +** and a new root interior node is generated (which should always fit
|
| +** within ROOT_MAX because it only needs space for 2 varints, the
|
| +** height and the blockid of the previous root).
|
| +**
|
| +** The meta-information in the segment directory is:
|
| +** level - segment level (see below)
|
| +** idx - index within level
|
| +** - (level,idx uniquely identify a segment)
|
| +** start_block - first leaf node
|
| +** leaves_end_block - last leaf node
|
| +** end_block - last block (including interior nodes)
|
| +** root - contents of root node
|
| +**
|
| +** If the root node is a leaf node, then start_block,
|
| +** leaves_end_block, and end_block are all 0.
|
| +**
|
| +**
|
| +**** Segment merging ****
|
| +** To amortize update costs, segments are groups into levels and
|
| +** merged in matches. Each increase in level represents exponentially
|
| +** more documents.
|
| +**
|
| +** New documents (actually, document updates) are tokenized and
|
| +** written individually (using LeafWriter) to a level 0 segment, with
|
| +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all
|
| +** level 0 segments are merged into a single level 1 segment. Level 1
|
| +** is populated like level 0, and eventually MERGE_COUNT level 1
|
| +** segments are merged to a single level 2 segment (representing
|
| +** MERGE_COUNT^2 updates), and so on.
|
| +**
|
| +** A segment merge traverses all segments at a given level in
|
| +** parallel, performing a straightforward sorted merge. Since segment
|
| +** leaf nodes are written in to the %_segments table in order, this
|
| +** merge traverses the underlying sqlite disk structures efficiently.
|
| +** After the merge, all segment blocks from the merged level are
|
| +** deleted.
|
| +**
|
| +** MERGE_COUNT controls how often we merge segments. 16 seems to be
|
| +** somewhat of a sweet spot for insertion performance. 32 and 64 show
|
| +** very similar performance numbers to 16 on insertion, though they're
|
| +** a tiny bit slower (perhaps due to more overhead in merge-time
|
| +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than
|
| +** 16, 2 about 66% slower than 16.
|
| +**
|
| +** At query time, high MERGE_COUNT increases the number of segments
|
| +** which need to be scanned and merged. For instance, with 100k docs
|
| +** inserted:
|
| +**
|
| +** MERGE_COUNT segments
|
| +** 16 25
|
| +** 8 12
|
| +** 4 10
|
| +** 2 6
|
| +**
|
| +** This appears to have only a moderate impact on queries for very
|
| +** frequent terms (which are somewhat dominated by segment merge
|
| +** costs), and infrequent and non-existent terms still seem to be fast
|
| +** even with many segments.
|
| +**
|
| +** TODO(shess) That said, it would be nice to have a better query-side
|
| +** argument for MERGE_COUNT of 16. Also, it is possible/likely that
|
| +** optimizations to things like doclist merging will swing the sweet
|
| +** spot around.
|
| +**
|
| +**
|
| +**
|
| +**** Handling of deletions and updates ****
|
| +** Since we're using a segmented structure, with no docid-oriented
|
| +** index into the term index, we clearly cannot simply update the term
|
| +** index when a document is deleted or updated. For deletions, we
|
| +** write an empty doclist (varint(docid) varint(POS_END)), for updates
|
| +** we simply write the new doclist. Segment merges overwrite older
|
| +** data for a particular docid with newer data, so deletes or updates
|
| +** will eventually overtake the earlier data and knock it out. The
|
| +** query logic likewise merges doclists so that newer data knocks out
|
| +** older data.
|
| +**
|
| +** TODO(shess) Provide a VACUUM type operation to clear out all
|
| +** deletions and duplications. This would basically be a forced merge
|
| +** into a single segment.
|
| +*/
|
| +
|
| +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
|
| +
|
| +#if defined(SQLITE_ENABLE_FTS2) && !defined(SQLITE_CORE)
|
| +# define SQLITE_CORE 1
|
| +#endif
|
| +
|
| +#include <assert.h>
|
| +#include <stdlib.h>
|
| +#include <stdio.h>
|
| +#include <string.h>
|
| +#include "fts2.h"
|
| +#include "fts2_hash.h"
|
| +#include "fts2_tokenizer.h"
|
| +#include "sqlite3.h"
|
| +#include "sqlite3ext.h"
|
| +SQLITE_EXTENSION_INIT1
|
| +
|
| +
|
| +/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
|
| +** would be nice to order the file better, perhaps something along the
|
| +** lines of:
|
| +**
|
| +** - utility functions
|
| +** - table setup functions
|
| +** - table update functions
|
| +** - table query functions
|
| +**
|
| +** Put the query functions last because they're likely to reference
|
| +** typedefs or functions from the table update section.
|
| +*/
|
| +
|
| +#if 0
|
| +# define TRACE(A) printf A; fflush(stdout)
|
| +#else
|
| +# define TRACE(A)
|
| +#endif
|
| +
|
| +/* It is not safe to call isspace(), tolower(), or isalnum() on
|
| +** hi-bit-set characters. This is the same solution used in the
|
| +** tokenizer.
|
| +*/
|
| +/* TODO(shess) The snippet-generation code should be using the
|
| +** tokenizer-generated tokens rather than doing its own local
|
| +** tokenization.
|
| +*/
|
| +/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
|
| +static int safe_isspace(char c){
|
| + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
|
| +}
|
| +static int safe_tolower(char c){
|
| + return (c>='A' && c<='Z') ? (c - 'A' + 'a') : c;
|
| +}
|
| +static int safe_isalnum(char c){
|
| + return (c>='0' && c<='9') || (c>='A' && c<='Z') || (c>='a' && c<='z');
|
| +}
|
| +
|
| +typedef enum DocListType {
|
| + DL_DOCIDS, /* docids only */
|
| + DL_POSITIONS, /* docids + positions */
|
| + DL_POSITIONS_OFFSETS /* docids + positions + offsets */
|
| +} DocListType;
|
| +
|
| +/*
|
| +** By default, only positions and not offsets are stored in the doclists.
|
| +** To change this so that offsets are stored too, compile with
|
| +**
|
| +** -DDL_DEFAULT=DL_POSITIONS_OFFSETS
|
| +**
|
| +** If DL_DEFAULT is set to DL_DOCIDS, your table can only be inserted
|
| +** into (no deletes or updates).
|
| +*/
|
| +#ifndef DL_DEFAULT
|
| +# define DL_DEFAULT DL_POSITIONS
|
| +#endif
|
| +
|
| +enum {
|
| + POS_END = 0, /* end of this position list */
|
| + POS_COLUMN, /* followed by new column number */
|
| + POS_BASE
|
| +};
|
| +
|
| +/* MERGE_COUNT controls how often we merge segments (see comment at
|
| +** top of file).
|
| +*/
|
| +#define MERGE_COUNT 16
|
| +
|
| +/* utility functions */
|
| +
|
| +/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
|
| +** record to prevent errors of the form:
|
| +**
|
| +** my_function(SomeType *b){
|
| +** memset(b, '\0', sizeof(b)); // sizeof(b)!=sizeof(*b)
|
| +** }
|
| +*/
|
| +/* TODO(shess) Obvious candidates for a header file. */
|
| +#define CLEAR(b) memset(b, '\0', sizeof(*(b)))
|
| +
|
| +#ifndef NDEBUG
|
| +# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
|
| +#else
|
| +# define SCRAMBLE(b)
|
| +#endif
|
| +
|
| +/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
|
| +#define VARINT_MAX 10
|
| +
|
| +/* Write a 64-bit variable-length integer to memory starting at p[0].
|
| + * The length of data written will be between 1 and VARINT_MAX bytes.
|
| + * The number of bytes written is returned. */
|
| +static int putVarint(char *p, sqlite_int64 v){
|
| + unsigned char *q = (unsigned char *) p;
|
| + sqlite_uint64 vu = v;
|
| + do{
|
| + *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
|
| + vu >>= 7;
|
| + }while( vu!=0 );
|
| + q[-1] &= 0x7f; /* turn off high bit in final byte */
|
| + assert( q - (unsigned char *)p <= VARINT_MAX );
|
| + return (int) (q - (unsigned char *)p);
|
| +}
|
| +
|
| +/* Read a 64-bit variable-length integer from memory starting at p[0].
|
| + * Return the number of bytes read, or 0 on error.
|
| + * The value is stored in *v. */
|
| +static int getVarint(const char *p, sqlite_int64 *v){
|
| + const unsigned char *q = (const unsigned char *) p;
|
| + sqlite_uint64 x = 0, y = 1;
|
| + while( (*q & 0x80) == 0x80 ){
|
| + x += y * (*q++ & 0x7f);
|
| + y <<= 7;
|
| + if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
|
| + assert( 0 );
|
| + return 0;
|
| + }
|
| + }
|
| + x += y * (*q++);
|
| + *v = (sqlite_int64) x;
|
| + return (int) (q - (unsigned char *)p);
|
| +}
|
| +
|
| +static int getVarint32(const char *p, int *pi){
|
| + sqlite_int64 i;
|
| + int ret = getVarint(p, &i);
|
| + *pi = (int) i;
|
| + assert( *pi==i );
|
| + return ret;
|
| +}
|
| +
|
| +/*******************************************************************/
|
| +/* DataBuffer is used to collect data into a buffer in piecemeal
|
| +** fashion. It implements the usual distinction between amount of
|
| +** data currently stored (nData) and buffer capacity (nCapacity).
|
| +**
|
| +** dataBufferInit - create a buffer with given initial capacity.
|
| +** dataBufferReset - forget buffer's data, retaining capacity.
|
| +** dataBufferDestroy - free buffer's data.
|
| +** dataBufferSwap - swap contents of two buffers.
|
| +** dataBufferExpand - expand capacity without adding data.
|
| +** dataBufferAppend - append data.
|
| +** dataBufferAppend2 - append two pieces of data at once.
|
| +** dataBufferReplace - replace buffer's data.
|
| +*/
|
| +typedef struct DataBuffer {
|
| + char *pData; /* Pointer to malloc'ed buffer. */
|
| + int nCapacity; /* Size of pData buffer. */
|
| + int nData; /* End of data loaded into pData. */
|
| +} DataBuffer;
|
| +
|
| +static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
|
| + assert( nCapacity>=0 );
|
| + pBuffer->nData = 0;
|
| + pBuffer->nCapacity = nCapacity;
|
| + pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
|
| +}
|
| +static void dataBufferReset(DataBuffer *pBuffer){
|
| + pBuffer->nData = 0;
|
| +}
|
| +static void dataBufferDestroy(DataBuffer *pBuffer){
|
| + if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
|
| + SCRAMBLE(pBuffer);
|
| +}
|
| +static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){
|
| + DataBuffer tmp = *pBuffer1;
|
| + *pBuffer1 = *pBuffer2;
|
| + *pBuffer2 = tmp;
|
| +}
|
| +static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
|
| + assert( nAddCapacity>0 );
|
| + /* TODO(shess) Consider expanding more aggressively. Note that the
|
| + ** underlying malloc implementation may take care of such things for
|
| + ** us already.
|
| + */
|
| + if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
|
| + pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
|
| + pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
|
| + }
|
| +}
|
| +static void dataBufferAppend(DataBuffer *pBuffer,
|
| + const char *pSource, int nSource){
|
| + assert( nSource>0 && pSource!=NULL );
|
| + dataBufferExpand(pBuffer, nSource);
|
| + memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
|
| + pBuffer->nData += nSource;
|
| +}
|
| +static void dataBufferAppend2(DataBuffer *pBuffer,
|
| + const char *pSource1, int nSource1,
|
| + const char *pSource2, int nSource2){
|
| + assert( nSource1>0 && pSource1!=NULL );
|
| + assert( nSource2>0 && pSource2!=NULL );
|
| + dataBufferExpand(pBuffer, nSource1+nSource2);
|
| + memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
|
| + memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
|
| + pBuffer->nData += nSource1+nSource2;
|
| +}
|
| +static void dataBufferReplace(DataBuffer *pBuffer,
|
| + const char *pSource, int nSource){
|
| + dataBufferReset(pBuffer);
|
| + dataBufferAppend(pBuffer, pSource, nSource);
|
| +}
|
| +
|
| +/* StringBuffer is a null-terminated version of DataBuffer. */
|
| +typedef struct StringBuffer {
|
| + DataBuffer b; /* Includes null terminator. */
|
| +} StringBuffer;
|
| +
|
| +static void initStringBuffer(StringBuffer *sb){
|
| + dataBufferInit(&sb->b, 100);
|
| + dataBufferReplace(&sb->b, "", 1);
|
| +}
|
| +static int stringBufferLength(StringBuffer *sb){
|
| + return sb->b.nData-1;
|
| +}
|
| +static char *stringBufferData(StringBuffer *sb){
|
| + return sb->b.pData;
|
| +}
|
| +static void stringBufferDestroy(StringBuffer *sb){
|
| + dataBufferDestroy(&sb->b);
|
| +}
|
| +
|
| +static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
|
| + assert( sb->b.nData>0 );
|
| + if( nFrom>0 ){
|
| + sb->b.nData--;
|
| + dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
|
| + }
|
| +}
|
| +static void append(StringBuffer *sb, const char *zFrom){
|
| + nappend(sb, zFrom, strlen(zFrom));
|
| +}
|
| +
|
| +/* Append a list of strings separated by commas. */
|
| +static void appendList(StringBuffer *sb, int nString, char **azString){
|
| + int i;
|
| + for(i=0; i<nString; ++i){
|
| + if( i>0 ) append(sb, ", ");
|
| + append(sb, azString[i]);
|
| + }
|
| +}
|
| +
|
| +static int endsInWhiteSpace(StringBuffer *p){
|
| + return stringBufferLength(p)>0 &&
|
| + safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
|
| +}
|
| +
|
| +/* If the StringBuffer ends in something other than white space, add a
|
| +** single space character to the end.
|
| +*/
|
| +static void appendWhiteSpace(StringBuffer *p){
|
| + if( stringBufferLength(p)==0 ) return;
|
| + if( !endsInWhiteSpace(p) ) append(p, " ");
|
| +}
|
| +
|
| +/* Remove white space from the end of the StringBuffer */
|
| +static void trimWhiteSpace(StringBuffer *p){
|
| + while( endsInWhiteSpace(p) ){
|
| + p->b.pData[--p->b.nData-1] = '\0';
|
| + }
|
| +}
|
| +
|
| +/*******************************************************************/
|
| +/* DLReader is used to read document elements from a doclist. The
|
| +** current docid is cached, so dlrDocid() is fast. DLReader does not
|
| +** own the doclist buffer.
|
| +**
|
| +** dlrAtEnd - true if there's no more data to read.
|
| +** dlrDocid - docid of current document.
|
| +** dlrDocData - doclist data for current document (including docid).
|
| +** dlrDocDataBytes - length of same.
|
| +** dlrAllDataBytes - length of all remaining data.
|
| +** dlrPosData - position data for current document.
|
| +** dlrPosDataLen - length of pos data for current document (incl POS_END).
|
| +** dlrStep - step to current document.
|
| +** dlrInit - initial for doclist of given type against given data.
|
| +** dlrDestroy - clean up.
|
| +**
|
| +** Expected usage is something like:
|
| +**
|
| +** DLReader reader;
|
| +** dlrInit(&reader, pData, nData);
|
| +** while( !dlrAtEnd(&reader) ){
|
| +** // calls to dlrDocid() and kin.
|
| +** dlrStep(&reader);
|
| +** }
|
| +** dlrDestroy(&reader);
|
| +*/
|
| +typedef struct DLReader {
|
| + DocListType iType;
|
| + const char *pData;
|
| + int nData;
|
| +
|
| + sqlite_int64 iDocid;
|
| + int nElement;
|
| +} DLReader;
|
| +
|
| +static int dlrAtEnd(DLReader *pReader){
|
| + assert( pReader->nData>=0 );
|
| + return pReader->nData==0;
|
| +}
|
| +static sqlite_int64 dlrDocid(DLReader *pReader){
|
| + assert( !dlrAtEnd(pReader) );
|
| + return pReader->iDocid;
|
| +}
|
| +static const char *dlrDocData(DLReader *pReader){
|
| + assert( !dlrAtEnd(pReader) );
|
| + return pReader->pData;
|
| +}
|
| +static int dlrDocDataBytes(DLReader *pReader){
|
| + assert( !dlrAtEnd(pReader) );
|
| + return pReader->nElement;
|
| +}
|
| +static int dlrAllDataBytes(DLReader *pReader){
|
| + assert( !dlrAtEnd(pReader) );
|
| + return pReader->nData;
|
| +}
|
| +/* TODO(shess) Consider adding a field to track iDocid varint length
|
| +** to make these two functions faster. This might matter (a tiny bit)
|
| +** for queries.
|
| +*/
|
| +static const char *dlrPosData(DLReader *pReader){
|
| + sqlite_int64 iDummy;
|
| + int n = getVarint(pReader->pData, &iDummy);
|
| + assert( !dlrAtEnd(pReader) );
|
| + return pReader->pData+n;
|
| +}
|
| +static int dlrPosDataLen(DLReader *pReader){
|
| + sqlite_int64 iDummy;
|
| + int n = getVarint(pReader->pData, &iDummy);
|
| + assert( !dlrAtEnd(pReader) );
|
| + return pReader->nElement-n;
|
| +}
|
| +static void dlrStep(DLReader *pReader){
|
| + assert( !dlrAtEnd(pReader) );
|
| +
|
| + /* Skip past current doclist element. */
|
| + assert( pReader->nElement<=pReader->nData );
|
| + pReader->pData += pReader->nElement;
|
| + pReader->nData -= pReader->nElement;
|
| +
|
| + /* If there is more data, read the next doclist element. */
|
| + if( pReader->nData!=0 ){
|
| + sqlite_int64 iDocidDelta;
|
| + int iDummy, n = getVarint(pReader->pData, &iDocidDelta);
|
| + pReader->iDocid += iDocidDelta;
|
| + if( pReader->iType>=DL_POSITIONS ){
|
| + assert( n<pReader->nData );
|
| + while( 1 ){
|
| + n += getVarint32(pReader->pData+n, &iDummy);
|
| + assert( n<=pReader->nData );
|
| + if( iDummy==POS_END ) break;
|
| + if( iDummy==POS_COLUMN ){
|
| + n += getVarint32(pReader->pData+n, &iDummy);
|
| + assert( n<pReader->nData );
|
| + }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
|
| + n += getVarint32(pReader->pData+n, &iDummy);
|
| + n += getVarint32(pReader->pData+n, &iDummy);
|
| + assert( n<pReader->nData );
|
| + }
|
| + }
|
| + }
|
| + pReader->nElement = n;
|
| + assert( pReader->nElement<=pReader->nData );
|
| + }
|
| +}
|
| +static void dlrInit(DLReader *pReader, DocListType iType,
|
| + const char *pData, int nData){
|
| + assert( pData!=NULL && nData!=0 );
|
| + pReader->iType = iType;
|
| + pReader->pData = pData;
|
| + pReader->nData = nData;
|
| + pReader->nElement = 0;
|
| + pReader->iDocid = 0;
|
| +
|
| + /* Load the first element's data. There must be a first element. */
|
| + dlrStep(pReader);
|
| +}
|
| +static void dlrDestroy(DLReader *pReader){
|
| + SCRAMBLE(pReader);
|
| +}
|
| +
|
| +#ifndef NDEBUG
|
| +/* Verify that the doclist can be validly decoded. Also returns the
|
| +** last docid found because it is convenient in other assertions for
|
| +** DLWriter.
|
| +*/
|
| +static void docListValidate(DocListType iType, const char *pData, int nData,
|
| + sqlite_int64 *pLastDocid){
|
| + sqlite_int64 iPrevDocid = 0;
|
| + assert( nData>0 );
|
| + assert( pData!=0 );
|
| + assert( pData+nData>pData );
|
| + while( nData!=0 ){
|
| + sqlite_int64 iDocidDelta;
|
| + int n = getVarint(pData, &iDocidDelta);
|
| + iPrevDocid += iDocidDelta;
|
| + if( iType>DL_DOCIDS ){
|
| + int iDummy;
|
| + while( 1 ){
|
| + n += getVarint32(pData+n, &iDummy);
|
| + if( iDummy==POS_END ) break;
|
| + if( iDummy==POS_COLUMN ){
|
| + n += getVarint32(pData+n, &iDummy);
|
| + }else if( iType>DL_POSITIONS ){
|
| + n += getVarint32(pData+n, &iDummy);
|
| + n += getVarint32(pData+n, &iDummy);
|
| + }
|
| + assert( n<=nData );
|
| + }
|
| + }
|
| + assert( n<=nData );
|
| + pData += n;
|
| + nData -= n;
|
| + }
|
| + if( pLastDocid ) *pLastDocid = iPrevDocid;
|
| +}
|
| +#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
|
| +#else
|
| +#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
|
| +#endif
|
| +
|
| +/*******************************************************************/
|
| +/* DLWriter is used to write doclist data to a DataBuffer. DLWriter
|
| +** always appends to the buffer and does not own it.
|
| +**
|
| +** dlwInit - initialize to write a given type doclistto a buffer.
|
| +** dlwDestroy - clear the writer's memory. Does not free buffer.
|
| +** dlwAppend - append raw doclist data to buffer.
|
| +** dlwCopy - copy next doclist from reader to writer.
|
| +** dlwAdd - construct doclist element and append to buffer.
|
| +** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
|
| +*/
|
| +typedef struct DLWriter {
|
| + DocListType iType;
|
| + DataBuffer *b;
|
| + sqlite_int64 iPrevDocid;
|
| +#ifndef NDEBUG
|
| + int has_iPrevDocid;
|
| +#endif
|
| +} DLWriter;
|
| +
|
| +static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){
|
| + pWriter->b = b;
|
| + pWriter->iType = iType;
|
| + pWriter->iPrevDocid = 0;
|
| +#ifndef NDEBUG
|
| + pWriter->has_iPrevDocid = 0;
|
| +#endif
|
| +}
|
| +static void dlwDestroy(DLWriter *pWriter){
|
| + SCRAMBLE(pWriter);
|
| +}
|
| +/* iFirstDocid is the first docid in the doclist in pData. It is
|
| +** needed because pData may point within a larger doclist, in which
|
| +** case the first item would be delta-encoded.
|
| +**
|
| +** iLastDocid is the final docid in the doclist in pData. It is
|
| +** needed to create the new iPrevDocid for future delta-encoding. The
|
| +** code could decode the passed doclist to recreate iLastDocid, but
|
| +** the only current user (docListMerge) already has decoded this
|
| +** information.
|
| +*/
|
| +/* TODO(shess) This has become just a helper for docListMerge.
|
| +** Consider a refactor to make this cleaner.
|
| +*/
|
| +static void dlwAppend(DLWriter *pWriter,
|
| + const char *pData, int nData,
|
| + sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
|
| + sqlite_int64 iDocid = 0;
|
| + char c[VARINT_MAX];
|
| + int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
|
| +#ifndef NDEBUG
|
| + sqlite_int64 iLastDocidDelta;
|
| +#endif
|
| +
|
| + /* Recode the initial docid as delta from iPrevDocid. */
|
| + nFirstOld = getVarint(pData, &iDocid);
|
| + assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
|
| + nFirstNew = putVarint(c, iFirstDocid-pWriter->iPrevDocid);
|
| +
|
| + /* Verify that the incoming doclist is valid AND that it ends with
|
| + ** the expected docid. This is essential because we'll trust this
|
| + ** docid in future delta-encoding.
|
| + */
|
| + ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
|
| + assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
|
| +
|
| + /* Append recoded initial docid and everything else. Rest of docids
|
| + ** should have been delta-encoded from previous initial docid.
|
| + */
|
| + if( nFirstOld<nData ){
|
| + dataBufferAppend2(pWriter->b, c, nFirstNew,
|
| + pData+nFirstOld, nData-nFirstOld);
|
| + }else{
|
| + dataBufferAppend(pWriter->b, c, nFirstNew);
|
| + }
|
| + pWriter->iPrevDocid = iLastDocid;
|
| +}
|
| +static void dlwCopy(DLWriter *pWriter, DLReader *pReader){
|
| + dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
|
| + dlrDocid(pReader), dlrDocid(pReader));
|
| +}
|
| +static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
|
| + char c[VARINT_MAX];
|
| + int n = putVarint(c, iDocid-pWriter->iPrevDocid);
|
| +
|
| + /* Docids must ascend. */
|
| + assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid );
|
| + assert( pWriter->iType==DL_DOCIDS );
|
| +
|
| + dataBufferAppend(pWriter->b, c, n);
|
| + pWriter->iPrevDocid = iDocid;
|
| +#ifndef NDEBUG
|
| + pWriter->has_iPrevDocid = 1;
|
| +#endif
|
| +}
|
| +
|
| +/*******************************************************************/
|
| +/* PLReader is used to read data from a document's position list. As
|
| +** the caller steps through the list, data is cached so that varints
|
| +** only need to be decoded once.
|
| +**
|
| +** plrInit, plrDestroy - create/destroy a reader.
|
| +** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
|
| +** plrAtEnd - at end of stream, only call plrDestroy once true.
|
| +** plrStep - step to the next element.
|
| +*/
|
| +typedef struct PLReader {
|
| + /* These refer to the next position's data. nData will reach 0 when
|
| + ** reading the last position, so plrStep() signals EOF by setting
|
| + ** pData to NULL.
|
| + */
|
| + const char *pData;
|
| + int nData;
|
| +
|
| + DocListType iType;
|
| + int iColumn; /* the last column read */
|
| + int iPosition; /* the last position read */
|
| + int iStartOffset; /* the last start offset read */
|
| + int iEndOffset; /* the last end offset read */
|
| +} PLReader;
|
| +
|
| +static int plrAtEnd(PLReader *pReader){
|
| + return pReader->pData==NULL;
|
| +}
|
| +static int plrColumn(PLReader *pReader){
|
| + assert( !plrAtEnd(pReader) );
|
| + return pReader->iColumn;
|
| +}
|
| +static int plrPosition(PLReader *pReader){
|
| + assert( !plrAtEnd(pReader) );
|
| + return pReader->iPosition;
|
| +}
|
| +static int plrStartOffset(PLReader *pReader){
|
| + assert( !plrAtEnd(pReader) );
|
| + return pReader->iStartOffset;
|
| +}
|
| +static int plrEndOffset(PLReader *pReader){
|
| + assert( !plrAtEnd(pReader) );
|
| + return pReader->iEndOffset;
|
| +}
|
| +static void plrStep(PLReader *pReader){
|
| + int i, n;
|
| +
|
| + assert( !plrAtEnd(pReader) );
|
| +
|
| + if( pReader->nData==0 ){
|
| + pReader->pData = NULL;
|
| + return;
|
| + }
|
| +
|
| + n = getVarint32(pReader->pData, &i);
|
| + if( i==POS_COLUMN ){
|
| + n += getVarint32(pReader->pData+n, &pReader->iColumn);
|
| + pReader->iPosition = 0;
|
| + pReader->iStartOffset = 0;
|
| + n += getVarint32(pReader->pData+n, &i);
|
| + }
|
| + /* Should never see adjacent column changes. */
|
| + assert( i!=POS_COLUMN );
|
| +
|
| + if( i==POS_END ){
|
| + pReader->nData = 0;
|
| + pReader->pData = NULL;
|
| + return;
|
| + }
|
| +
|
| + pReader->iPosition += i-POS_BASE;
|
| + if( pReader->iType==DL_POSITIONS_OFFSETS ){
|
| + n += getVarint32(pReader->pData+n, &i);
|
| + pReader->iStartOffset += i;
|
| + n += getVarint32(pReader->pData+n, &i);
|
| + pReader->iEndOffset = pReader->iStartOffset+i;
|
| + }
|
| + assert( n<=pReader->nData );
|
| + pReader->pData += n;
|
| + pReader->nData -= n;
|
| +}
|
| +
|
| +static void plrInit(PLReader *pReader, DLReader *pDLReader){
|
| + pReader->pData = dlrPosData(pDLReader);
|
| + pReader->nData = dlrPosDataLen(pDLReader);
|
| + pReader->iType = pDLReader->iType;
|
| + pReader->iColumn = 0;
|
| + pReader->iPosition = 0;
|
| + pReader->iStartOffset = 0;
|
| + pReader->iEndOffset = 0;
|
| + plrStep(pReader);
|
| +}
|
| +static void plrDestroy(PLReader *pReader){
|
| + SCRAMBLE(pReader);
|
| +}
|
| +
|
| +/*******************************************************************/
|
| +/* PLWriter is used in constructing a document's position list. As a
|
| +** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
|
| +** PLWriter writes to the associated DLWriter's buffer.
|
| +**
|
| +** plwInit - init for writing a document's poslist.
|
| +** plwDestroy - clear a writer.
|
| +** plwAdd - append position and offset information.
|
| +** plwCopy - copy next position's data from reader to writer.
|
| +** plwTerminate - add any necessary doclist terminator.
|
| +**
|
| +** Calling plwAdd() after plwTerminate() may result in a corrupt
|
| +** doclist.
|
| +*/
|
| +/* TODO(shess) Until we've written the second item, we can cache the
|
| +** first item's information. Then we'd have three states:
|
| +**
|
| +** - initialized with docid, no positions.
|
| +** - docid and one position.
|
| +** - docid and multiple positions.
|
| +**
|
| +** Only the last state needs to actually write to dlw->b, which would
|
| +** be an improvement in the DLCollector case.
|
| +*/
|
| +typedef struct PLWriter {
|
| + DLWriter *dlw;
|
| +
|
| + int iColumn; /* the last column written */
|
| + int iPos; /* the last position written */
|
| + int iOffset; /* the last start offset written */
|
| +} PLWriter;
|
| +
|
| +/* TODO(shess) In the case where the parent is reading these values
|
| +** from a PLReader, we could optimize to a copy if that PLReader has
|
| +** the same type as pWriter.
|
| +*/
|
| +static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
|
| + int iStartOffset, int iEndOffset){
|
| + /* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
|
| + ** iStartOffsetDelta, and iEndOffsetDelta.
|
| + */
|
| + char c[5*VARINT_MAX];
|
| + int n = 0;
|
| +
|
| + /* Ban plwAdd() after plwTerminate(). */
|
| + assert( pWriter->iPos!=-1 );
|
| +
|
| + if( pWriter->dlw->iType==DL_DOCIDS ) return;
|
| +
|
| + if( iColumn!=pWriter->iColumn ){
|
| + n += putVarint(c+n, POS_COLUMN);
|
| + n += putVarint(c+n, iColumn);
|
| + pWriter->iColumn = iColumn;
|
| + pWriter->iPos = 0;
|
| + pWriter->iOffset = 0;
|
| + }
|
| + assert( iPos>=pWriter->iPos );
|
| + n += putVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
|
| + pWriter->iPos = iPos;
|
| + if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
|
| + assert( iStartOffset>=pWriter->iOffset );
|
| + n += putVarint(c+n, iStartOffset-pWriter->iOffset);
|
| + pWriter->iOffset = iStartOffset;
|
| + assert( iEndOffset>=iStartOffset );
|
| + n += putVarint(c+n, iEndOffset-iStartOffset);
|
| + }
|
| + dataBufferAppend(pWriter->dlw->b, c, n);
|
| +}
|
| +static void plwCopy(PLWriter *pWriter, PLReader *pReader){
|
| + plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
|
| + plrStartOffset(pReader), plrEndOffset(pReader));
|
| +}
|
| +static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){
|
| + char c[VARINT_MAX];
|
| + int n;
|
| +
|
| + pWriter->dlw = dlw;
|
| +
|
| + /* Docids must ascend. */
|
| + assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid );
|
| + n = putVarint(c, iDocid-pWriter->dlw->iPrevDocid);
|
| + dataBufferAppend(pWriter->dlw->b, c, n);
|
| + pWriter->dlw->iPrevDocid = iDocid;
|
| +#ifndef NDEBUG
|
| + pWriter->dlw->has_iPrevDocid = 1;
|
| +#endif
|
| +
|
| + pWriter->iColumn = 0;
|
| + pWriter->iPos = 0;
|
| + pWriter->iOffset = 0;
|
| +}
|
| +/* TODO(shess) Should plwDestroy() also terminate the doclist? But
|
| +** then plwDestroy() would no longer be just a destructor, it would
|
| +** also be doing work, which isn't consistent with the overall idiom.
|
| +** Another option would be for plwAdd() to always append any necessary
|
| +** terminator, so that the output is always correct. But that would
|
| +** add incremental work to the common case with the only benefit being
|
| +** API elegance. Punt for now.
|
| +*/
|
| +static void plwTerminate(PLWriter *pWriter){
|
| + if( pWriter->dlw->iType>DL_DOCIDS ){
|
| + char c[VARINT_MAX];
|
| + int n = putVarint(c, POS_END);
|
| + dataBufferAppend(pWriter->dlw->b, c, n);
|
| + }
|
| +#ifndef NDEBUG
|
| + /* Mark as terminated for assert in plwAdd(). */
|
| + pWriter->iPos = -1;
|
| +#endif
|
| +}
|
| +static void plwDestroy(PLWriter *pWriter){
|
| + SCRAMBLE(pWriter);
|
| +}
|
| +
|
| +/*******************************************************************/
|
| +/* DLCollector wraps PLWriter and DLWriter to provide a
|
| +** dynamically-allocated doclist area to use during tokenization.
|
| +**
|
| +** dlcNew - malloc up and initialize a collector.
|
| +** dlcDelete - destroy a collector and all contained items.
|
| +** dlcAddPos - append position and offset information.
|
| +** dlcAddDoclist - add the collected doclist to the given buffer.
|
| +** dlcNext - terminate the current document and open another.
|
| +*/
|
| +typedef struct DLCollector {
|
| + DataBuffer b;
|
| + DLWriter dlw;
|
| + PLWriter plw;
|
| +} DLCollector;
|
| +
|
| +/* TODO(shess) This could also be done by calling plwTerminate() and
|
| +** dataBufferAppend(). I tried that, expecting nominal performance
|
| +** differences, but it seemed to pretty reliably be worth 1% to code
|
| +** it this way. I suspect it is the incremental malloc overhead (some
|
| +** percentage of the plwTerminate() calls will cause a realloc), so
|
| +** this might be worth revisiting if the DataBuffer implementation
|
| +** changes.
|
| +*/
|
| +static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){
|
| + if( pCollector->dlw.iType>DL_DOCIDS ){
|
| + char c[VARINT_MAX];
|
| + int n = putVarint(c, POS_END);
|
| + dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
|
| + }else{
|
| + dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
|
| + }
|
| +}
|
| +static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
|
| + plwTerminate(&pCollector->plw);
|
| + plwDestroy(&pCollector->plw);
|
| + plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
|
| +}
|
| +static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
|
| + int iStartOffset, int iEndOffset){
|
| + plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
|
| +}
|
| +
|
| +static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
|
| + DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
|
| + dataBufferInit(&pCollector->b, 0);
|
| + dlwInit(&pCollector->dlw, iType, &pCollector->b);
|
| + plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
|
| + return pCollector;
|
| +}
|
| +static void dlcDelete(DLCollector *pCollector){
|
| + plwDestroy(&pCollector->plw);
|
| + dlwDestroy(&pCollector->dlw);
|
| + dataBufferDestroy(&pCollector->b);
|
| + SCRAMBLE(pCollector);
|
| + sqlite3_free(pCollector);
|
| +}
|
| +
|
| +
|
| +/* Copy the doclist data of iType in pData/nData into *out, trimming
|
| +** unnecessary data as we go. Only columns matching iColumn are
|
| +** copied, all columns copied if iColumn is -1. Elements with no
|
| +** matching columns are dropped. The output is an iOutType doclist.
|
| +*/
|
| +/* NOTE(shess) This code is only valid after all doclists are merged.
|
| +** If this is run before merges, then doclist items which represent
|
| +** deletion will be trimmed, and will thus not effect a deletion
|
| +** during the merge.
|
| +*/
|
| +static void docListTrim(DocListType iType, const char *pData, int nData,
|
| + int iColumn, DocListType iOutType, DataBuffer *out){
|
| + DLReader dlReader;
|
| + DLWriter dlWriter;
|
| +
|
| + assert( iOutType<=iType );
|
| +
|
| + dlrInit(&dlReader, iType, pData, nData);
|
| + dlwInit(&dlWriter, iOutType, out);
|
| +
|
| + while( !dlrAtEnd(&dlReader) ){
|
| + PLReader plReader;
|
| + PLWriter plWriter;
|
| + int match = 0;
|
| +
|
| + plrInit(&plReader, &dlReader);
|
| +
|
| + while( !plrAtEnd(&plReader) ){
|
| + if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
|
| + if( !match ){
|
| + plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
|
| + match = 1;
|
| + }
|
| + plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
|
| + plrStartOffset(&plReader), plrEndOffset(&plReader));
|
| + }
|
| + plrStep(&plReader);
|
| + }
|
| + if( match ){
|
| + plwTerminate(&plWriter);
|
| + plwDestroy(&plWriter);
|
| + }
|
| +
|
| + plrDestroy(&plReader);
|
| + dlrStep(&dlReader);
|
| + }
|
| + dlwDestroy(&dlWriter);
|
| + dlrDestroy(&dlReader);
|
| +}
|
| +
|
| +/* Used by docListMerge() to keep doclists in the ascending order by
|
| +** docid, then ascending order by age (so the newest comes first).
|
| +*/
|
| +typedef struct OrderedDLReader {
|
| + DLReader *pReader;
|
| +
|
| + /* TODO(shess) If we assume that docListMerge pReaders is ordered by
|
| + ** age (which we do), then we could use pReader comparisons to break
|
| + ** ties.
|
| + */
|
| + int idx;
|
| +} OrderedDLReader;
|
| +
|
| +/* Order eof to end, then by docid asc, idx desc. */
|
| +static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){
|
| + if( dlrAtEnd(r1->pReader) ){
|
| + if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
|
| + return 1; /* Only r1 atEnd(). */
|
| + }
|
| + if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
|
| +
|
| + if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
|
| + if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
|
| +
|
| + /* Descending on idx. */
|
| + return r2->idx-r1->idx;
|
| +}
|
| +
|
| +/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
|
| +** p[1..n-1] is already sorted.
|
| +*/
|
| +/* TODO(shess) Is this frequent enough to warrant a binary search?
|
| +** Before implementing that, instrument the code to check. In most
|
| +** current usage, I expect that p[0] will be less than p[1] a very
|
| +** high proportion of the time.
|
| +*/
|
| +static void orderedDLReaderReorder(OrderedDLReader *p, int n){
|
| + while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
|
| + OrderedDLReader tmp = p[0];
|
| + p[0] = p[1];
|
| + p[1] = tmp;
|
| + n--;
|
| + p++;
|
| + }
|
| +}
|
| +
|
| +/* Given an array of doclist readers, merge their doclist elements
|
| +** into out in sorted order (by docid), dropping elements from older
|
| +** readers when there is a duplicate docid. pReaders is assumed to be
|
| +** ordered by age, oldest first.
|
| +*/
|
| +/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
|
| +** be fixed.
|
| +*/
|
| +static void docListMerge(DataBuffer *out,
|
| + DLReader *pReaders, int nReaders){
|
| + OrderedDLReader readers[MERGE_COUNT];
|
| + DLWriter writer;
|
| + int i, n;
|
| + const char *pStart = 0;
|
| + int nStart = 0;
|
| + sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
|
| +
|
| + assert( nReaders>0 );
|
| + if( nReaders==1 ){
|
| + dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
|
| + return;
|
| + }
|
| +
|
| + assert( nReaders<=MERGE_COUNT );
|
| + n = 0;
|
| + for(i=0; i<nReaders; i++){
|
| + assert( pReaders[i].iType==pReaders[0].iType );
|
| + readers[i].pReader = pReaders+i;
|
| + readers[i].idx = i;
|
| + n += dlrAllDataBytes(&pReaders[i]);
|
| + }
|
| + /* Conservatively size output to sum of inputs. Output should end
|
| + ** up strictly smaller than input.
|
| + */
|
| + dataBufferExpand(out, n);
|
| +
|
| + /* Get the readers into sorted order. */
|
| + while( i-->0 ){
|
| + orderedDLReaderReorder(readers+i, nReaders-i);
|
| + }
|
| +
|
| + dlwInit(&writer, pReaders[0].iType, out);
|
| + while( !dlrAtEnd(readers[0].pReader) ){
|
| + sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
|
| +
|
| + /* If this is a continuation of the current buffer to copy, extend
|
| + ** that buffer. memcpy() seems to be more efficient if it has a
|
| + ** lots of data to copy.
|
| + */
|
| + if( dlrDocData(readers[0].pReader)==pStart+nStart ){
|
| + nStart += dlrDocDataBytes(readers[0].pReader);
|
| + }else{
|
| + if( pStart!=0 ){
|
| + dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
|
| + }
|
| + pStart = dlrDocData(readers[0].pReader);
|
| + nStart = dlrDocDataBytes(readers[0].pReader);
|
| + iFirstDocid = iDocid;
|
| + }
|
| + iLastDocid = iDocid;
|
| + dlrStep(readers[0].pReader);
|
| +
|
| + /* Drop all of the older elements with the same docid. */
|
| + for(i=1; i<nReaders &&
|
| + !dlrAtEnd(readers[i].pReader) &&
|
| + dlrDocid(readers[i].pReader)==iDocid; i++){
|
| + dlrStep(readers[i].pReader);
|
| + }
|
| +
|
| + /* Get the readers back into order. */
|
| + while( i-->0 ){
|
| + orderedDLReaderReorder(readers+i, nReaders-i);
|
| + }
|
| + }
|
| +
|
| + /* Copy over any remaining elements. */
|
| + if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
|
| + dlwDestroy(&writer);
|
| +}
|
| +
|
| +/* Helper function for posListUnion(). Compares the current position
|
| +** between left and right, returning as standard C idiom of <0 if
|
| +** left<right, >0 if left>right, and 0 if left==right. "End" always
|
| +** compares greater.
|
| +*/
|
| +static int posListCmp(PLReader *pLeft, PLReader *pRight){
|
| + assert( pLeft->iType==pRight->iType );
|
| + if( pLeft->iType==DL_DOCIDS ) return 0;
|
| +
|
| + if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
|
| + if( plrAtEnd(pRight) ) return -1;
|
| +
|
| + if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
|
| + if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
|
| +
|
| + if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
|
| + if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
|
| + if( pLeft->iType==DL_POSITIONS ) return 0;
|
| +
|
| + if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
|
| + if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
|
| +
|
| + if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
|
| + if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
|
| +
|
| + return 0;
|
| +}
|
| +
|
| +/* Write the union of position lists in pLeft and pRight to pOut.
|
| +** "Union" in this case meaning "All unique position tuples". Should
|
| +** work with any doclist type, though both inputs and the output
|
| +** should be the same type.
|
| +*/
|
| +static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
|
| + PLReader left, right;
|
| + PLWriter writer;
|
| +
|
| + assert( dlrDocid(pLeft)==dlrDocid(pRight) );
|
| + assert( pLeft->iType==pRight->iType );
|
| + assert( pLeft->iType==pOut->iType );
|
| +
|
| + plrInit(&left, pLeft);
|
| + plrInit(&right, pRight);
|
| + plwInit(&writer, pOut, dlrDocid(pLeft));
|
| +
|
| + while( !plrAtEnd(&left) || !plrAtEnd(&right) ){
|
| + int c = posListCmp(&left, &right);
|
| + if( c<0 ){
|
| + plwCopy(&writer, &left);
|
| + plrStep(&left);
|
| + }else if( c>0 ){
|
| + plwCopy(&writer, &right);
|
| + plrStep(&right);
|
| + }else{
|
| + plwCopy(&writer, &left);
|
| + plrStep(&left);
|
| + plrStep(&right);
|
| + }
|
| + }
|
| +
|
| + plwTerminate(&writer);
|
| + plwDestroy(&writer);
|
| + plrDestroy(&left);
|
| + plrDestroy(&right);
|
| +}
|
| +
|
| +/* Write the union of doclists in pLeft and pRight to pOut. For
|
| +** docids in common between the inputs, the union of the position
|
| +** lists is written. Inputs and outputs are always type DL_DEFAULT.
|
| +*/
|
| +static void docListUnion(
|
| + const char *pLeft, int nLeft,
|
| + const char *pRight, int nRight,
|
| + DataBuffer *pOut /* Write the combined doclist here */
|
| +){
|
| + DLReader left, right;
|
| + DLWriter writer;
|
| +
|
| + if( nLeft==0 ){
|
| + if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
|
| + return;
|
| + }
|
| + if( nRight==0 ){
|
| + dataBufferAppend(pOut, pLeft, nLeft);
|
| + return;
|
| + }
|
| +
|
| + dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
|
| + dlrInit(&right, DL_DEFAULT, pRight, nRight);
|
| + dlwInit(&writer, DL_DEFAULT, pOut);
|
| +
|
| + while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
|
| + if( dlrAtEnd(&right) ){
|
| + dlwCopy(&writer, &left);
|
| + dlrStep(&left);
|
| + }else if( dlrAtEnd(&left) ){
|
| + dlwCopy(&writer, &right);
|
| + dlrStep(&right);
|
| + }else if( dlrDocid(&left)<dlrDocid(&right) ){
|
| + dlwCopy(&writer, &left);
|
| + dlrStep(&left);
|
| + }else if( dlrDocid(&left)>dlrDocid(&right) ){
|
| + dlwCopy(&writer, &right);
|
| + dlrStep(&right);
|
| + }else{
|
| + posListUnion(&left, &right, &writer);
|
| + dlrStep(&left);
|
| + dlrStep(&right);
|
| + }
|
| + }
|
| +
|
| + dlrDestroy(&left);
|
| + dlrDestroy(&right);
|
| + dlwDestroy(&writer);
|
| +}
|
| +
|
| +/* pLeft and pRight are DLReaders positioned to the same docid.
|
| +**
|
| +** If there are no instances in pLeft or pRight where the position
|
| +** of pLeft is one less than the position of pRight, then this
|
| +** routine adds nothing to pOut.
|
| +**
|
| +** If there are one or more instances where positions from pLeft
|
| +** are exactly one less than positions from pRight, then add a new
|
| +** document record to pOut. If pOut wants to hold positions, then
|
| +** include the positions from pRight that are one more than a
|
| +** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1.
|
| +*/
|
| +static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
|
| + DLWriter *pOut){
|
| + PLReader left, right;
|
| + PLWriter writer;
|
| + int match = 0;
|
| +
|
| + assert( dlrDocid(pLeft)==dlrDocid(pRight) );
|
| + assert( pOut->iType!=DL_POSITIONS_OFFSETS );
|
| +
|
| + plrInit(&left, pLeft);
|
| + plrInit(&right, pRight);
|
| +
|
| + while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
|
| + if( plrColumn(&left)<plrColumn(&right) ){
|
| + plrStep(&left);
|
| + }else if( plrColumn(&left)>plrColumn(&right) ){
|
| + plrStep(&right);
|
| + }else if( plrPosition(&left)+1<plrPosition(&right) ){
|
| + plrStep(&left);
|
| + }else if( plrPosition(&left)+1>plrPosition(&right) ){
|
| + plrStep(&right);
|
| + }else{
|
| + if( !match ){
|
| + plwInit(&writer, pOut, dlrDocid(pLeft));
|
| + match = 1;
|
| + }
|
| + plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
|
| + plrStep(&left);
|
| + plrStep(&right);
|
| + }
|
| + }
|
| +
|
| + if( match ){
|
| + plwTerminate(&writer);
|
| + plwDestroy(&writer);
|
| + }
|
| +
|
| + plrDestroy(&left);
|
| + plrDestroy(&right);
|
| +}
|
| +
|
| +/* We have two doclists with positions: pLeft and pRight.
|
| +** Write the phrase intersection of these two doclists into pOut.
|
| +**
|
| +** A phrase intersection means that two documents only match
|
| +** if pLeft.iPos+1==pRight.iPos.
|
| +**
|
| +** iType controls the type of data written to pOut. If iType is
|
| +** DL_POSITIONS, the positions are those from pRight.
|
| +*/
|
| +static void docListPhraseMerge(
|
| + const char *pLeft, int nLeft,
|
| + const char *pRight, int nRight,
|
| + DocListType iType,
|
| + DataBuffer *pOut /* Write the combined doclist here */
|
| +){
|
| + DLReader left, right;
|
| + DLWriter writer;
|
| +
|
| + if( nLeft==0 || nRight==0 ) return;
|
| +
|
| + assert( iType!=DL_POSITIONS_OFFSETS );
|
| +
|
| + dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
|
| + dlrInit(&right, DL_POSITIONS, pRight, nRight);
|
| + dlwInit(&writer, iType, pOut);
|
| +
|
| + while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
|
| + if( dlrDocid(&left)<dlrDocid(&right) ){
|
| + dlrStep(&left);
|
| + }else if( dlrDocid(&right)<dlrDocid(&left) ){
|
| + dlrStep(&right);
|
| + }else{
|
| + posListPhraseMerge(&left, &right, &writer);
|
| + dlrStep(&left);
|
| + dlrStep(&right);
|
| + }
|
| + }
|
| +
|
| + dlrDestroy(&left);
|
| + dlrDestroy(&right);
|
| + dlwDestroy(&writer);
|
| +}
|
| +
|
| +/* We have two DL_DOCIDS doclists: pLeft and pRight.
|
| +** Write the intersection of these two doclists into pOut as a
|
| +** DL_DOCIDS doclist.
|
| +*/
|
| +static void docListAndMerge(
|
| + const char *pLeft, int nLeft,
|
| + const char *pRight, int nRight,
|
| + DataBuffer *pOut /* Write the combined doclist here */
|
| +){
|
| + DLReader left, right;
|
| + DLWriter writer;
|
| +
|
| + if( nLeft==0 || nRight==0 ) return;
|
| +
|
| + dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
|
| + dlrInit(&right, DL_DOCIDS, pRight, nRight);
|
| + dlwInit(&writer, DL_DOCIDS, pOut);
|
| +
|
| + while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
|
| + if( dlrDocid(&left)<dlrDocid(&right) ){
|
| + dlrStep(&left);
|
| + }else if( dlrDocid(&right)<dlrDocid(&left) ){
|
| + dlrStep(&right);
|
| + }else{
|
| + dlwAdd(&writer, dlrDocid(&left));
|
| + dlrStep(&left);
|
| + dlrStep(&right);
|
| + }
|
| + }
|
| +
|
| + dlrDestroy(&left);
|
| + dlrDestroy(&right);
|
| + dlwDestroy(&writer);
|
| +}
|
| +
|
| +/* We have two DL_DOCIDS doclists: pLeft and pRight.
|
| +** Write the union of these two doclists into pOut as a
|
| +** DL_DOCIDS doclist.
|
| +*/
|
| +static void docListOrMerge(
|
| + const char *pLeft, int nLeft,
|
| + const char *pRight, int nRight,
|
| + DataBuffer *pOut /* Write the combined doclist here */
|
| +){
|
| + DLReader left, right;
|
| + DLWriter writer;
|
| +
|
| + if( nLeft==0 ){
|
| + if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
|
| + return;
|
| + }
|
| + if( nRight==0 ){
|
| + dataBufferAppend(pOut, pLeft, nLeft);
|
| + return;
|
| + }
|
| +
|
| + dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
|
| + dlrInit(&right, DL_DOCIDS, pRight, nRight);
|
| + dlwInit(&writer, DL_DOCIDS, pOut);
|
| +
|
| + while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
|
| + if( dlrAtEnd(&right) ){
|
| + dlwAdd(&writer, dlrDocid(&left));
|
| + dlrStep(&left);
|
| + }else if( dlrAtEnd(&left) ){
|
| + dlwAdd(&writer, dlrDocid(&right));
|
| + dlrStep(&right);
|
| + }else if( dlrDocid(&left)<dlrDocid(&right) ){
|
| + dlwAdd(&writer, dlrDocid(&left));
|
| + dlrStep(&left);
|
| + }else if( dlrDocid(&right)<dlrDocid(&left) ){
|
| + dlwAdd(&writer, dlrDocid(&right));
|
| + dlrStep(&right);
|
| + }else{
|
| + dlwAdd(&writer, dlrDocid(&left));
|
| + dlrStep(&left);
|
| + dlrStep(&right);
|
| + }
|
| + }
|
| +
|
| + dlrDestroy(&left);
|
| + dlrDestroy(&right);
|
| + dlwDestroy(&writer);
|
| +}
|
| +
|
| +/* We have two DL_DOCIDS doclists: pLeft and pRight.
|
| +** Write into pOut as DL_DOCIDS doclist containing all documents that
|
| +** occur in pLeft but not in pRight.
|
| +*/
|
| +static void docListExceptMerge(
|
| + const char *pLeft, int nLeft,
|
| + const char *pRight, int nRight,
|
| + DataBuffer *pOut /* Write the combined doclist here */
|
| +){
|
| + DLReader left, right;
|
| + DLWriter writer;
|
| +
|
| + if( nLeft==0 ) return;
|
| + if( nRight==0 ){
|
| + dataBufferAppend(pOut, pLeft, nLeft);
|
| + return;
|
| + }
|
| +
|
| + dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
|
| + dlrInit(&right, DL_DOCIDS, pRight, nRight);
|
| + dlwInit(&writer, DL_DOCIDS, pOut);
|
| +
|
| + while( !dlrAtEnd(&left) ){
|
| + while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
|
| + dlrStep(&right);
|
| + }
|
| + if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
|
| + dlwAdd(&writer, dlrDocid(&left));
|
| + }
|
| + dlrStep(&left);
|
| + }
|
| +
|
| + dlrDestroy(&left);
|
| + dlrDestroy(&right);
|
| + dlwDestroy(&writer);
|
| +}
|
| +
|
| +static char *string_dup_n(const char *s, int n){
|
| + char *str = sqlite3_malloc(n + 1);
|
| + memcpy(str, s, n);
|
| + str[n] = '\0';
|
| + return str;
|
| +}
|
| +
|
| +/* Duplicate a string; the caller must free() the returned string.
|
| + * (We don't use strdup() since it is not part of the standard C library and
|
| + * may not be available everywhere.) */
|
| +static char *string_dup(const char *s){
|
| + return string_dup_n(s, strlen(s));
|
| +}
|
| +
|
| +/* Format a string, replacing each occurrence of the % character with
|
| + * zDb.zName. This may be more convenient than sqlite_mprintf()
|
| + * when one string is used repeatedly in a format string.
|
| + * The caller must free() the returned string. */
|
| +static char *string_format(const char *zFormat,
|
| + const char *zDb, const char *zName){
|
| + const char *p;
|
| + size_t len = 0;
|
| + size_t nDb = strlen(zDb);
|
| + size_t nName = strlen(zName);
|
| + size_t nFullTableName = nDb+1+nName;
|
| + char *result;
|
| + char *r;
|
| +
|
| + /* first compute length needed */
|
| + for(p = zFormat ; *p ; ++p){
|
| + len += (*p=='%' ? nFullTableName : 1);
|
| + }
|
| + len += 1; /* for null terminator */
|
| +
|
| + r = result = sqlite3_malloc(len);
|
| + for(p = zFormat; *p; ++p){
|
| + if( *p=='%' ){
|
| + memcpy(r, zDb, nDb);
|
| + r += nDb;
|
| + *r++ = '.';
|
| + memcpy(r, zName, nName);
|
| + r += nName;
|
| + } else {
|
| + *r++ = *p;
|
| + }
|
| + }
|
| + *r++ = '\0';
|
| + assert( r == result + len );
|
| + return result;
|
| +}
|
| +
|
| +static int sql_exec(sqlite3 *db, const char *zDb, const char *zName,
|
| + const char *zFormat){
|
| + char *zCommand = string_format(zFormat, zDb, zName);
|
| + int rc;
|
| + TRACE(("FTS2 sql: %s\n", zCommand));
|
| + rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
|
| + sqlite3_free(zCommand);
|
| + return rc;
|
| +}
|
| +
|
| +static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName,
|
| + sqlite3_stmt **ppStmt, const char *zFormat){
|
| + char *zCommand = string_format(zFormat, zDb, zName);
|
| + int rc;
|
| + TRACE(("FTS2 prepare: %s\n", zCommand));
|
| + rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
|
| + sqlite3_free(zCommand);
|
| + return rc;
|
| +}
|
| +
|
| +/* end utility functions */
|
| +
|
| +/* Forward reference */
|
| +typedef struct fulltext_vtab fulltext_vtab;
|
| +
|
| +/* A single term in a query is represented by an instances of
|
| +** the following structure.
|
| +*/
|
| +typedef struct QueryTerm {
|
| + short int nPhrase; /* How many following terms are part of the same phrase */
|
| + short int iPhrase; /* This is the i-th term of a phrase. */
|
| + short int iColumn; /* Column of the index that must match this term */
|
| + signed char isOr; /* this term is preceded by "OR" */
|
| + signed char isNot; /* this term is preceded by "-" */
|
| + signed char isPrefix; /* this term is followed by "*" */
|
| + char *pTerm; /* text of the term. '\000' terminated. malloced */
|
| + int nTerm; /* Number of bytes in pTerm[] */
|
| +} QueryTerm;
|
| +
|
| +
|
| +/* A query string is parsed into a Query structure.
|
| + *
|
| + * We could, in theory, allow query strings to be complicated
|
| + * nested expressions with precedence determined by parentheses.
|
| + * But none of the major search engines do this. (Perhaps the
|
| + * feeling is that an parenthesized expression is two complex of
|
| + * an idea for the average user to grasp.) Taking our lead from
|
| + * the major search engines, we will allow queries to be a list
|
| + * of terms (with an implied AND operator) or phrases in double-quotes,
|
| + * with a single optional "-" before each non-phrase term to designate
|
| + * negation and an optional OR connector.
|
| + *
|
| + * OR binds more tightly than the implied AND, which is what the
|
| + * major search engines seem to do. So, for example:
|
| + *
|
| + * [one two OR three] ==> one AND (two OR three)
|
| + * [one OR two three] ==> (one OR two) AND three
|
| + *
|
| + * A "-" before a term matches all entries that lack that term.
|
| + * The "-" must occur immediately before the term with in intervening
|
| + * space. This is how the search engines do it.
|
| + *
|
| + * A NOT term cannot be the right-hand operand of an OR. If this
|
| + * occurs in the query string, the NOT is ignored:
|
| + *
|
| + * [one OR -two] ==> one OR two
|
| + *
|
| + */
|
| +typedef struct Query {
|
| + fulltext_vtab *pFts; /* The full text index */
|
| + int nTerms; /* Number of terms in the query */
|
| + QueryTerm *pTerms; /* Array of terms. Space obtained from malloc() */
|
| + int nextIsOr; /* Set the isOr flag on the next inserted term */
|
| + int nextColumn; /* Next word parsed must be in this column */
|
| + int dfltColumn; /* The default column */
|
| +} Query;
|
| +
|
| +
|
| +/*
|
| +** An instance of the following structure keeps track of generated
|
| +** matching-word offset information and snippets.
|
| +*/
|
| +typedef struct Snippet {
|
| + int nMatch; /* Total number of matches */
|
| + int nAlloc; /* Space allocated for aMatch[] */
|
| + struct snippetMatch { /* One entry for each matching term */
|
| + char snStatus; /* Status flag for use while constructing snippets */
|
| + short int iCol; /* The column that contains the match */
|
| + short int iTerm; /* The index in Query.pTerms[] of the matching term */
|
| + short int nByte; /* Number of bytes in the term */
|
| + int iStart; /* The offset to the first character of the term */
|
| + } *aMatch; /* Points to space obtained from malloc */
|
| + char *zOffset; /* Text rendering of aMatch[] */
|
| + int nOffset; /* strlen(zOffset) */
|
| + char *zSnippet; /* Snippet text */
|
| + int nSnippet; /* strlen(zSnippet) */
|
| +} Snippet;
|
| +
|
| +
|
| +typedef enum QueryType {
|
| + QUERY_GENERIC, /* table scan */
|
| + QUERY_ROWID, /* lookup by rowid */
|
| + QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
|
| +} QueryType;
|
| +
|
| +typedef enum fulltext_statement {
|
| + CONTENT_INSERT_STMT,
|
| + CONTENT_SELECT_STMT,
|
| + CONTENT_UPDATE_STMT,
|
| + CONTENT_DELETE_STMT,
|
| + CONTENT_EXISTS_STMT,
|
| +
|
| + BLOCK_INSERT_STMT,
|
| + BLOCK_SELECT_STMT,
|
| + BLOCK_DELETE_STMT,
|
| + BLOCK_DELETE_ALL_STMT,
|
| +
|
| + SEGDIR_MAX_INDEX_STMT,
|
| + SEGDIR_SET_STMT,
|
| + SEGDIR_SELECT_LEVEL_STMT,
|
| + SEGDIR_SPAN_STMT,
|
| + SEGDIR_DELETE_STMT,
|
| + SEGDIR_SELECT_SEGMENT_STMT,
|
| + SEGDIR_SELECT_ALL_STMT,
|
| + SEGDIR_DELETE_ALL_STMT,
|
| + SEGDIR_COUNT_STMT,
|
| +
|
| + MAX_STMT /* Always at end! */
|
| +} fulltext_statement;
|
| +
|
| +/* These must exactly match the enum above. */
|
| +/* TODO(shess): Is there some risk that a statement will be used in two
|
| +** cursors at once, e.g. if a query joins a virtual table to itself?
|
| +** If so perhaps we should move some of these to the cursor object.
|
| +*/
|
| +static const char *const fulltext_zStatement[MAX_STMT] = {
|
| + /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */
|
| + /* CONTENT_SELECT */ "select * from %_content where rowid = ?",
|
| + /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */
|
| + /* CONTENT_DELETE */ "delete from %_content where rowid = ?",
|
| + /* CONTENT_EXISTS */ "select rowid from %_content limit 1",
|
| +
|
| + /* BLOCK_INSERT */ "insert into %_segments values (?)",
|
| + /* BLOCK_SELECT */ "select block from %_segments where rowid = ?",
|
| + /* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?",
|
| + /* BLOCK_DELETE_ALL */ "delete from %_segments",
|
| +
|
| + /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
|
| + /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
|
| + /* SEGDIR_SELECT_LEVEL */
|
| + "select start_block, leaves_end_block, root from %_segdir "
|
| + " where level = ? order by idx",
|
| + /* SEGDIR_SPAN */
|
| + "select min(start_block), max(end_block) from %_segdir "
|
| + " where level = ? and start_block <> 0",
|
| + /* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
|
| +
|
| + /* NOTE(shess): The first three results of the following two
|
| + ** statements must match.
|
| + */
|
| + /* SEGDIR_SELECT_SEGMENT */
|
| + "select start_block, leaves_end_block, root from %_segdir "
|
| + " where level = ? and idx = ?",
|
| + /* SEGDIR_SELECT_ALL */
|
| + "select start_block, leaves_end_block, root from %_segdir "
|
| + " order by level desc, idx asc",
|
| + /* SEGDIR_DELETE_ALL */ "delete from %_segdir",
|
| + /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir",
|
| +};
|
| +
|
| +/*
|
| +** A connection to a fulltext index is an instance of the following
|
| +** structure. The xCreate and xConnect methods create an instance
|
| +** of this structure and xDestroy and xDisconnect free that instance.
|
| +** All other methods receive a pointer to the structure as one of their
|
| +** arguments.
|
| +*/
|
| +struct fulltext_vtab {
|
| + sqlite3_vtab base; /* Base class used by SQLite core */
|
| + sqlite3 *db; /* The database connection */
|
| + const char *zDb; /* logical database name */
|
| + const char *zName; /* virtual table name */
|
| + int nColumn; /* number of columns in virtual table */
|
| + char **azColumn; /* column names. malloced */
|
| + char **azContentColumn; /* column names in content table; malloced */
|
| + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
|
| +
|
| + /* Precompiled statements which we keep as long as the table is
|
| + ** open.
|
| + */
|
| + sqlite3_stmt *pFulltextStatements[MAX_STMT];
|
| +
|
| + /* Precompiled statements used for segment merges. We run a
|
| + ** separate select across the leaf level of each tree being merged.
|
| + */
|
| + sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
|
| + /* The statement used to prepare pLeafSelectStmts. */
|
| +#define LEAF_SELECT \
|
| + "select block from %_segments where rowid between ? and ? order by rowid"
|
| +
|
| + /* These buffer pending index updates during transactions.
|
| + ** nPendingData estimates the memory size of the pending data. It
|
| + ** doesn't include the hash-bucket overhead, nor any malloc
|
| + ** overhead. When nPendingData exceeds kPendingThreshold, the
|
| + ** buffer is flushed even before the transaction closes.
|
| + ** pendingTerms stores the data, and is only valid when nPendingData
|
| + ** is >=0 (nPendingData<0 means pendingTerms has not been
|
| + ** initialized). iPrevDocid is the last docid written, used to make
|
| + ** certain we're inserting in sorted order.
|
| + */
|
| + int nPendingData;
|
| +#define kPendingThreshold (1*1024*1024)
|
| + sqlite_int64 iPrevDocid;
|
| + fts2Hash pendingTerms;
|
| +};
|
| +
|
| +/*
|
| +** When the core wants to do a query, it create a cursor using a
|
| +** call to xOpen. This structure is an instance of a cursor. It
|
| +** is destroyed by xClose.
|
| +*/
|
| +typedef struct fulltext_cursor {
|
| + sqlite3_vtab_cursor base; /* Base class used by SQLite core */
|
| + QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
|
| + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
|
| + int eof; /* True if at End Of Results */
|
| + Query q; /* Parsed query string */
|
| + Snippet snippet; /* Cached snippet for the current row */
|
| + int iColumn; /* Column being searched */
|
| + DataBuffer result; /* Doclist results from fulltextQuery */
|
| + DLReader reader; /* Result reader if result not empty */
|
| +} fulltext_cursor;
|
| +
|
| +static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
|
| + return (fulltext_vtab *) c->base.pVtab;
|
| +}
|
| +
|
| +static const sqlite3_module fts2Module; /* forward declaration */
|
| +
|
| +/* Return a dynamically generated statement of the form
|
| + * insert into %_content (rowid, ...) values (?, ...)
|
| + */
|
| +static const char *contentInsertStatement(fulltext_vtab *v){
|
| + StringBuffer sb;
|
| + int i;
|
| +
|
| + initStringBuffer(&sb);
|
| + append(&sb, "insert into %_content (rowid, ");
|
| + appendList(&sb, v->nColumn, v->azContentColumn);
|
| + append(&sb, ") values (?");
|
| + for(i=0; i<v->nColumn; ++i)
|
| + append(&sb, ", ?");
|
| + append(&sb, ")");
|
| + return stringBufferData(&sb);
|
| +}
|
| +
|
| +/* Return a dynamically generated statement of the form
|
| + * update %_content set [col_0] = ?, [col_1] = ?, ...
|
| + * where rowid = ?
|
| + */
|
| +static const char *contentUpdateStatement(fulltext_vtab *v){
|
| + StringBuffer sb;
|
| + int i;
|
| +
|
| + initStringBuffer(&sb);
|
| + append(&sb, "update %_content set ");
|
| + for(i=0; i<v->nColumn; ++i) {
|
| + if( i>0 ){
|
| + append(&sb, ", ");
|
| + }
|
| + append(&sb, v->azContentColumn[i]);
|
| + append(&sb, " = ?");
|
| + }
|
| + append(&sb, " where rowid = ?");
|
| + return stringBufferData(&sb);
|
| +}
|
| +
|
| +/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
|
| +** If the indicated statement has never been prepared, it is prepared
|
| +** and cached, otherwise the cached version is reset.
|
| +*/
|
| +static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
|
| + sqlite3_stmt **ppStmt){
|
| + assert( iStmt<MAX_STMT );
|
| + if( v->pFulltextStatements[iStmt]==NULL ){
|
| + const char *zStmt;
|
| + int rc;
|
| + switch( iStmt ){
|
| + case CONTENT_INSERT_STMT:
|
| + zStmt = contentInsertStatement(v); break;
|
| + case CONTENT_UPDATE_STMT:
|
| + zStmt = contentUpdateStatement(v); break;
|
| + default:
|
| + zStmt = fulltext_zStatement[iStmt];
|
| + }
|
| + rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
|
| + zStmt);
|
| + if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + } else {
|
| + int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| +
|
| + *ppStmt = v->pFulltextStatements[iStmt];
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
|
| +** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
|
| +** where we expect no results.
|
| +*/
|
| +static int sql_single_step(sqlite3_stmt *s){
|
| + int rc = sqlite3_step(s);
|
| + return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
|
| +}
|
| +
|
| +/* Like sql_get_statement(), but for special replicated LEAF_SELECT
|
| +** statements. idx -1 is a special case for an uncached version of
|
| +** the statement (used in the optimize implementation).
|
| +*/
|
| +/* TODO(shess) Write version for generic statements and then share
|
| +** that between the cached-statement functions.
|
| +*/
|
| +static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
|
| + sqlite3_stmt **ppStmt){
|
| + assert( idx>=-1 && idx<MERGE_COUNT );
|
| + if( idx==-1 ){
|
| + return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
|
| + }else if( v->pLeafSelectStmts[idx]==NULL ){
|
| + int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
|
| + LEAF_SELECT);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }else{
|
| + int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| +
|
| + *ppStmt = v->pLeafSelectStmts[idx];
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* insert into %_content (rowid, ...) values ([rowid], [pValues]) */
|
| +static int content_insert(fulltext_vtab *v, sqlite3_value *rowid,
|
| + sqlite3_value **pValues){
|
| + sqlite3_stmt *s;
|
| + int i;
|
| + int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_value(s, 1, rowid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + for(i=0; i<v->nColumn; ++i){
|
| + rc = sqlite3_bind_value(s, 2+i, pValues[i]);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| +
|
| + return sql_single_step(s);
|
| +}
|
| +
|
| +/* update %_content set col0 = pValues[0], col1 = pValues[1], ...
|
| + * where rowid = [iRowid] */
|
| +static int content_update(fulltext_vtab *v, sqlite3_value **pValues,
|
| + sqlite_int64 iRowid){
|
| + sqlite3_stmt *s;
|
| + int i;
|
| + int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + for(i=0; i<v->nColumn; ++i){
|
| + rc = sqlite3_bind_value(s, 1+i, pValues[i]);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| +
|
| + rc = sqlite3_bind_int64(s, 1+v->nColumn, iRowid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return sql_single_step(s);
|
| +}
|
| +
|
| +static void freeStringArray(int nString, const char **pString){
|
| + int i;
|
| +
|
| + for (i=0 ; i < nString ; ++i) {
|
| + if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
|
| + }
|
| + sqlite3_free((void *) pString);
|
| +}
|
| +
|
| +/* select * from %_content where rowid = [iRow]
|
| + * The caller must delete the returned array and all strings in it.
|
| + * null fields will be NULL in the returned array.
|
| + *
|
| + * TODO: Perhaps we should return pointer/length strings here for consistency
|
| + * with other code which uses pointer/length. */
|
| +static int content_select(fulltext_vtab *v, sqlite_int64 iRow,
|
| + const char ***pValues){
|
| + sqlite3_stmt *s;
|
| + const char **values;
|
| + int i;
|
| + int rc;
|
| +
|
| + *pValues = NULL;
|
| +
|
| + rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 1, iRow);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + if( rc!=SQLITE_ROW ) return rc;
|
| +
|
| + values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *));
|
| + for(i=0; i<v->nColumn; ++i){
|
| + if( sqlite3_column_type(s, i)==SQLITE_NULL ){
|
| + values[i] = NULL;
|
| + }else{
|
| + values[i] = string_dup((char*)sqlite3_column_text(s, i));
|
| + }
|
| + }
|
| +
|
| + /* We expect only one row. We must execute another sqlite3_step()
|
| + * to complete the iteration; otherwise the table will remain locked. */
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_DONE ){
|
| + *pValues = values;
|
| + return SQLITE_OK;
|
| + }
|
| +
|
| + freeStringArray(v->nColumn, values);
|
| + return rc;
|
| +}
|
| +
|
| +/* delete from %_content where rowid = [iRow ] */
|
| +static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 1, iRow);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return sql_single_step(s);
|
| +}
|
| +
|
| +/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
|
| +** no rows exist, and any error in case of failure.
|
| +*/
|
| +static int content_exists(fulltext_vtab *v){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + if( rc!=SQLITE_ROW ) return rc;
|
| +
|
| + /* We expect only one row. We must execute another sqlite3_step()
|
| + * to complete the iteration; otherwise the table will remain locked. */
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_DONE ) return SQLITE_ROW;
|
| + if( rc==SQLITE_ROW ) return SQLITE_ERROR;
|
| + return rc;
|
| +}
|
| +
|
| +/* insert into %_segments values ([pData])
|
| +** returns assigned rowid in *piBlockid
|
| +*/
|
| +static int block_insert(fulltext_vtab *v, const char *pData, int nData,
|
| + sqlite_int64 *piBlockid){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_ROW ) return SQLITE_ERROR;
|
| + if( rc!=SQLITE_DONE ) return rc;
|
| +
|
| + *piBlockid = sqlite3_last_insert_rowid(v->db);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* delete from %_segments
|
| +** where rowid between [iStartBlockid] and [iEndBlockid]
|
| +**
|
| +** Deletes the range of blocks, inclusive, used to delete the blocks
|
| +** which form a segment.
|
| +*/
|
| +static int block_delete(fulltext_vtab *v,
|
| + sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 1, iStartBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 2, iEndBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return sql_single_step(s);
|
| +}
|
| +
|
| +/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
|
| +** at iLevel. Returns SQLITE_DONE if there are no segments at
|
| +** iLevel. Otherwise returns an error.
|
| +*/
|
| +static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int(s, 1, iLevel);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + /* Should always get at least one row due to how max() works. */
|
| + if( rc==SQLITE_DONE ) return SQLITE_DONE;
|
| + if( rc!=SQLITE_ROW ) return rc;
|
| +
|
| + /* NULL means that there were no inputs to max(). */
|
| + if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_ROW ) return SQLITE_ERROR;
|
| + return rc;
|
| + }
|
| +
|
| + *pidx = sqlite3_column_int(s, 0);
|
| +
|
| + /* We expect only one row. We must execute another sqlite3_step()
|
| + * to complete the iteration; otherwise the table will remain locked. */
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_ROW ) return SQLITE_ERROR;
|
| + if( rc!=SQLITE_DONE ) return rc;
|
| + return SQLITE_ROW;
|
| +}
|
| +
|
| +/* insert into %_segdir values (
|
| +** [iLevel], [idx],
|
| +** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
|
| +** [pRootData]
|
| +** )
|
| +*/
|
| +static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
|
| + sqlite_int64 iStartBlockid,
|
| + sqlite_int64 iLeavesEndBlockid,
|
| + sqlite_int64 iEndBlockid,
|
| + const char *pRootData, int nRootData){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int(s, 1, iLevel);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int(s, 2, idx);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 3, iStartBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 5, iEndBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return sql_single_step(s);
|
| +}
|
| +
|
| +/* Queries %_segdir for the block span of the segments in level
|
| +** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
|
| +** SQLITE_ROW if there are blocks, else an error.
|
| +*/
|
| +static int segdir_span(fulltext_vtab *v, int iLevel,
|
| + sqlite_int64 *piStartBlockid,
|
| + sqlite_int64 *piEndBlockid){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int(s, 1, iLevel);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
|
| + if( rc!=SQLITE_ROW ) return rc;
|
| +
|
| + /* This happens if all segments at this level are entirely inline. */
|
| + if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
|
| + /* We expect only one row. We must execute another sqlite3_step()
|
| + * to complete the iteration; otherwise the table will remain locked. */
|
| + int rc2 = sqlite3_step(s);
|
| + if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
|
| + return rc2;
|
| + }
|
| +
|
| + *piStartBlockid = sqlite3_column_int64(s, 0);
|
| + *piEndBlockid = sqlite3_column_int64(s, 1);
|
| +
|
| + /* We expect only one row. We must execute another sqlite3_step()
|
| + * to complete the iteration; otherwise the table will remain locked. */
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_ROW ) return SQLITE_ERROR;
|
| + if( rc!=SQLITE_DONE ) return rc;
|
| + return SQLITE_ROW;
|
| +}
|
| +
|
| +/* Delete the segment blocks and segment directory records for all
|
| +** segments at iLevel.
|
| +*/
|
| +static int segdir_delete(fulltext_vtab *v, int iLevel){
|
| + sqlite3_stmt *s;
|
| + sqlite_int64 iStartBlockid, iEndBlockid;
|
| + int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
|
| + if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
|
| +
|
| + if( rc==SQLITE_ROW ){
|
| + rc = block_delete(v, iStartBlockid, iEndBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| +
|
| + /* Delete the segment directory itself. */
|
| + rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 1, iLevel);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return sql_single_step(s);
|
| +}
|
| +
|
| +/* Delete entire fts index, SQLITE_OK on success, relevant error on
|
| +** failure.
|
| +*/
|
| +static int segdir_delete_all(fulltext_vtab *v){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sql_single_step(s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return sql_single_step(s);
|
| +}
|
| +
|
| +/* Returns SQLITE_OK with *pnSegments set to the number of entries in
|
| +** %_segdir and *piMaxLevel set to the highest level which has a
|
| +** segment. Otherwise returns the SQLite error which caused failure.
|
| +*/
|
| +static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + /* TODO(shess): This case should not be possible? Should stronger
|
| + ** measures be taken if it happens?
|
| + */
|
| + if( rc==SQLITE_DONE ){
|
| + *pnSegments = 0;
|
| + *piMaxLevel = 0;
|
| + return SQLITE_OK;
|
| + }
|
| + if( rc!=SQLITE_ROW ) return rc;
|
| +
|
| + *pnSegments = sqlite3_column_int(s, 0);
|
| + *piMaxLevel = sqlite3_column_int(s, 1);
|
| +
|
| + /* We expect only one row. We must execute another sqlite3_step()
|
| + * to complete the iteration; otherwise the table will remain locked. */
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_DONE ) return SQLITE_OK;
|
| + if( rc==SQLITE_ROW ) return SQLITE_ERROR;
|
| + return rc;
|
| +}
|
| +
|
| +/* TODO(shess) clearPendingTerms() is far down the file because
|
| +** writeZeroSegment() is far down the file because LeafWriter is far
|
| +** down the file. Consider refactoring the code to move the non-vtab
|
| +** code above the vtab code so that we don't need this forward
|
| +** reference.
|
| +*/
|
| +static int clearPendingTerms(fulltext_vtab *v);
|
| +
|
| +/*
|
| +** Free the memory used to contain a fulltext_vtab structure.
|
| +*/
|
| +static void fulltext_vtab_destroy(fulltext_vtab *v){
|
| + int iStmt, i;
|
| +
|
| + TRACE(("FTS2 Destroy %p\n", v));
|
| + for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
|
| + if( v->pFulltextStatements[iStmt]!=NULL ){
|
| + sqlite3_finalize(v->pFulltextStatements[iStmt]);
|
| + v->pFulltextStatements[iStmt] = NULL;
|
| + }
|
| + }
|
| +
|
| + for( i=0; i<MERGE_COUNT; i++ ){
|
| + if( v->pLeafSelectStmts[i]!=NULL ){
|
| + sqlite3_finalize(v->pLeafSelectStmts[i]);
|
| + v->pLeafSelectStmts[i] = NULL;
|
| + }
|
| + }
|
| +
|
| + if( v->pTokenizer!=NULL ){
|
| + v->pTokenizer->pModule->xDestroy(v->pTokenizer);
|
| + v->pTokenizer = NULL;
|
| + }
|
| +
|
| + clearPendingTerms(v);
|
| +
|
| + sqlite3_free(v->azColumn);
|
| + for(i = 0; i < v->nColumn; ++i) {
|
| + sqlite3_free(v->azContentColumn[i]);
|
| + }
|
| + sqlite3_free(v->azContentColumn);
|
| + sqlite3_free(v);
|
| +}
|
| +
|
| +/*
|
| +** Token types for parsing the arguments to xConnect or xCreate.
|
| +*/
|
| +#define TOKEN_EOF 0 /* End of file */
|
| +#define TOKEN_SPACE 1 /* Any kind of whitespace */
|
| +#define TOKEN_ID 2 /* An identifier */
|
| +#define TOKEN_STRING 3 /* A string literal */
|
| +#define TOKEN_PUNCT 4 /* A single punctuation character */
|
| +
|
| +/*
|
| +** If X is a character that can be used in an identifier then
|
| +** IdChar(X) will be true. Otherwise it is false.
|
| +**
|
| +** For ASCII, any character with the high-order bit set is
|
| +** allowed in an identifier. For 7-bit characters,
|
| +** sqlite3IsIdChar[X] must be 1.
|
| +**
|
| +** Ticket #1066. the SQL standard does not allow '$' in the
|
| +** middle of identfiers. But many SQL implementations do.
|
| +** SQLite will allow '$' in identifiers for compatibility.
|
| +** But the feature is undocumented.
|
| +*/
|
| +static const char isIdChar[] = {
|
| +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
|
| + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
|
| + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
|
| + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
|
| +};
|
| +#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isIdChar[c-0x20]))
|
| +
|
| +
|
| +/*
|
| +** Return the length of the token that begins at z[0].
|
| +** Store the token type in *tokenType before returning.
|
| +*/
|
| +static int getToken(const char *z, int *tokenType){
|
| + int i, c;
|
| + switch( *z ){
|
| + case 0: {
|
| + *tokenType = TOKEN_EOF;
|
| + return 0;
|
| + }
|
| + case ' ': case '\t': case '\n': case '\f': case '\r': {
|
| + for(i=1; safe_isspace(z[i]); i++){}
|
| + *tokenType = TOKEN_SPACE;
|
| + return i;
|
| + }
|
| + case '`':
|
| + case '\'':
|
| + case '"': {
|
| + int delim = z[0];
|
| + for(i=1; (c=z[i])!=0; i++){
|
| + if( c==delim ){
|
| + if( z[i+1]==delim ){
|
| + i++;
|
| + }else{
|
| + break;
|
| + }
|
| + }
|
| + }
|
| + *tokenType = TOKEN_STRING;
|
| + return i + (c!=0);
|
| + }
|
| + case '[': {
|
| + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
|
| + *tokenType = TOKEN_ID;
|
| + return i;
|
| + }
|
| + default: {
|
| + if( !IdChar(*z) ){
|
| + break;
|
| + }
|
| + for(i=1; IdChar(z[i]); i++){}
|
| + *tokenType = TOKEN_ID;
|
| + return i;
|
| + }
|
| + }
|
| + *tokenType = TOKEN_PUNCT;
|
| + return 1;
|
| +}
|
| +
|
| +/*
|
| +** A token extracted from a string is an instance of the following
|
| +** structure.
|
| +*/
|
| +typedef struct Token {
|
| + const char *z; /* Pointer to token text. Not '\000' terminated */
|
| + short int n; /* Length of the token text in bytes. */
|
| +} Token;
|
| +
|
| +/*
|
| +** Given a input string (which is really one of the argv[] parameters
|
| +** passed into xConnect or xCreate) split the string up into tokens.
|
| +** Return an array of pointers to '\000' terminated strings, one string
|
| +** for each non-whitespace token.
|
| +**
|
| +** The returned array is terminated by a single NULL pointer.
|
| +**
|
| +** Space to hold the returned array is obtained from a single
|
| +** malloc and should be freed by passing the return value to free().
|
| +** The individual strings within the token list are all a part of
|
| +** the single memory allocation and will all be freed at once.
|
| +*/
|
| +static char **tokenizeString(const char *z, int *pnToken){
|
| + int nToken = 0;
|
| + Token *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) );
|
| + int n = 1;
|
| + int e, i;
|
| + int totalSize = 0;
|
| + char **azToken;
|
| + char *zCopy;
|
| + while( n>0 ){
|
| + n = getToken(z, &e);
|
| + if( e!=TOKEN_SPACE ){
|
| + aToken[nToken].z = z;
|
| + aToken[nToken].n = n;
|
| + nToken++;
|
| + totalSize += n+1;
|
| + }
|
| + z += n;
|
| + }
|
| + azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize );
|
| + zCopy = (char*)&azToken[nToken];
|
| + nToken--;
|
| + for(i=0; i<nToken; i++){
|
| + azToken[i] = zCopy;
|
| + n = aToken[i].n;
|
| + memcpy(zCopy, aToken[i].z, n);
|
| + zCopy[n] = 0;
|
| + zCopy += n+1;
|
| + }
|
| + azToken[nToken] = 0;
|
| + sqlite3_free(aToken);
|
| + *pnToken = nToken;
|
| + return azToken;
|
| +}
|
| +
|
| +/*
|
| +** Convert an SQL-style quoted string into a normal string by removing
|
| +** the quote characters. The conversion is done in-place. If the
|
| +** input does not begin with a quote character, then this routine
|
| +** is a no-op.
|
| +**
|
| +** Examples:
|
| +**
|
| +** "abc" becomes abc
|
| +** 'xyz' becomes xyz
|
| +** [pqr] becomes pqr
|
| +** `mno` becomes mno
|
| +*/
|
| +static void dequoteString(char *z){
|
| + int quote;
|
| + int i, j;
|
| + if( z==0 ) return;
|
| + quote = z[0];
|
| + switch( quote ){
|
| + case '\'': break;
|
| + case '"': break;
|
| + case '`': break; /* For MySQL compatibility */
|
| + case '[': quote = ']'; break; /* For MS SqlServer compatibility */
|
| + default: return;
|
| + }
|
| + for(i=1, j=0; z[i]; i++){
|
| + if( z[i]==quote ){
|
| + if( z[i+1]==quote ){
|
| + z[j++] = quote;
|
| + i++;
|
| + }else{
|
| + z[j++] = 0;
|
| + break;
|
| + }
|
| + }else{
|
| + z[j++] = z[i];
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** The input azIn is a NULL-terminated list of tokens. Remove the first
|
| +** token and all punctuation tokens. Remove the quotes from
|
| +** around string literal tokens.
|
| +**
|
| +** Example:
|
| +**
|
| +** input: tokenize chinese ( 'simplifed' , 'mixed' )
|
| +** output: chinese simplifed mixed
|
| +**
|
| +** Another example:
|
| +**
|
| +** input: delimiters ( '[' , ']' , '...' )
|
| +** output: [ ] ...
|
| +*/
|
| +static void tokenListToIdList(char **azIn){
|
| + int i, j;
|
| + if( azIn ){
|
| + for(i=0, j=-1; azIn[i]; i++){
|
| + if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
|
| + dequoteString(azIn[i]);
|
| + if( j>=0 ){
|
| + azIn[j] = azIn[i];
|
| + }
|
| + j++;
|
| + }
|
| + }
|
| + azIn[j] = 0;
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Find the first alphanumeric token in the string zIn. Null-terminate
|
| +** this token. Remove any quotation marks. And return a pointer to
|
| +** the result.
|
| +*/
|
| +static char *firstToken(char *zIn, char **pzTail){
|
| + int n, ttype;
|
| + while(1){
|
| + n = getToken(zIn, &ttype);
|
| + if( ttype==TOKEN_SPACE ){
|
| + zIn += n;
|
| + }else if( ttype==TOKEN_EOF ){
|
| + *pzTail = zIn;
|
| + return 0;
|
| + }else{
|
| + zIn[n] = 0;
|
| + *pzTail = &zIn[1];
|
| + dequoteString(zIn);
|
| + return zIn;
|
| + }
|
| + }
|
| + /*NOTREACHED*/
|
| +}
|
| +
|
| +/* Return true if...
|
| +**
|
| +** * s begins with the string t, ignoring case
|
| +** * s is longer than t
|
| +** * The first character of s beyond t is not a alphanumeric
|
| +**
|
| +** Ignore leading space in *s.
|
| +**
|
| +** To put it another way, return true if the first token of
|
| +** s[] is t[].
|
| +*/
|
| +static int startsWith(const char *s, const char *t){
|
| + while( safe_isspace(*s) ){ s++; }
|
| + while( *t ){
|
| + if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
|
| + }
|
| + return *s!='_' && !safe_isalnum(*s);
|
| +}
|
| +
|
| +/*
|
| +** An instance of this structure defines the "spec" of a
|
| +** full text index. This structure is populated by parseSpec
|
| +** and use by fulltextConnect and fulltextCreate.
|
| +*/
|
| +typedef struct TableSpec {
|
| + const char *zDb; /* Logical database name */
|
| + const char *zName; /* Name of the full-text index */
|
| + int nColumn; /* Number of columns to be indexed */
|
| + char **azColumn; /* Original names of columns to be indexed */
|
| + char **azContentColumn; /* Column names for %_content */
|
| + char **azTokenizer; /* Name of tokenizer and its arguments */
|
| +} TableSpec;
|
| +
|
| +/*
|
| +** Reclaim all of the memory used by a TableSpec
|
| +*/
|
| +static void clearTableSpec(TableSpec *p) {
|
| + sqlite3_free(p->azColumn);
|
| + sqlite3_free(p->azContentColumn);
|
| + sqlite3_free(p->azTokenizer);
|
| +}
|
| +
|
| +/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
|
| + *
|
| + * CREATE VIRTUAL TABLE email
|
| + * USING fts2(subject, body, tokenize mytokenizer(myarg))
|
| + *
|
| + * We return parsed information in a TableSpec structure.
|
| + *
|
| + */
|
| +static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv,
|
| + char**pzErr){
|
| + int i, n;
|
| + char *z, *zDummy;
|
| + char **azArg;
|
| + const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */
|
| +
|
| + assert( argc>=3 );
|
| + /* Current interface:
|
| + ** argv[0] - module name
|
| + ** argv[1] - database name
|
| + ** argv[2] - table name
|
| + ** argv[3..] - columns, optionally followed by tokenizer specification
|
| + ** and snippet delimiters specification.
|
| + */
|
| +
|
| + /* Make a copy of the complete argv[][] array in a single allocation.
|
| + ** The argv[][] array is read-only and transient. We can write to the
|
| + ** copy in order to modify things and the copy is persistent.
|
| + */
|
| + CLEAR(pSpec);
|
| + for(i=n=0; i<argc; i++){
|
| + n += strlen(argv[i]) + 1;
|
| + }
|
| + azArg = sqlite3_malloc( sizeof(char*)*argc + n );
|
| + if( azArg==0 ){
|
| + return SQLITE_NOMEM;
|
| + }
|
| + z = (char*)&azArg[argc];
|
| + for(i=0; i<argc; i++){
|
| + azArg[i] = z;
|
| + strcpy(z, argv[i]);
|
| + z += strlen(z)+1;
|
| + }
|
| +
|
| + /* Identify the column names and the tokenizer and delimiter arguments
|
| + ** in the argv[][] array.
|
| + */
|
| + pSpec->zDb = azArg[1];
|
| + pSpec->zName = azArg[2];
|
| + pSpec->nColumn = 0;
|
| + pSpec->azColumn = azArg;
|
| + zTokenizer = "tokenize simple";
|
| + for(i=3; i<argc; ++i){
|
| + if( startsWith(azArg[i],"tokenize") ){
|
| + zTokenizer = azArg[i];
|
| + }else{
|
| + z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
|
| + pSpec->nColumn++;
|
| + }
|
| + }
|
| + if( pSpec->nColumn==0 ){
|
| + azArg[0] = "content";
|
| + pSpec->nColumn = 1;
|
| + }
|
| +
|
| + /*
|
| + ** Construct the list of content column names.
|
| + **
|
| + ** Each content column name will be of the form cNNAAAA
|
| + ** where NN is the column number and AAAA is the sanitized
|
| + ** column name. "sanitized" means that special characters are
|
| + ** converted to "_". The cNN prefix guarantees that all column
|
| + ** names are unique.
|
| + **
|
| + ** The AAAA suffix is not strictly necessary. It is included
|
| + ** for the convenience of people who might examine the generated
|
| + ** %_content table and wonder what the columns are used for.
|
| + */
|
| + pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
|
| + if( pSpec->azContentColumn==0 ){
|
| + clearTableSpec(pSpec);
|
| + return SQLITE_NOMEM;
|
| + }
|
| + for(i=0; i<pSpec->nColumn; i++){
|
| + char *p;
|
| + pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
|
| + for (p = pSpec->azContentColumn[i]; *p ; ++p) {
|
| + if( !safe_isalnum(*p) ) *p = '_';
|
| + }
|
| + }
|
| +
|
| + /*
|
| + ** Parse the tokenizer specification string.
|
| + */
|
| + pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
|
| + tokenListToIdList(pSpec->azTokenizer);
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Generate a CREATE TABLE statement that describes the schema of
|
| +** the virtual table. Return a pointer to this schema string.
|
| +**
|
| +** Space is obtained from sqlite3_mprintf() and should be freed
|
| +** using sqlite3_free().
|
| +*/
|
| +static char *fulltextSchema(
|
| + int nColumn, /* Number of columns */
|
| + const char *const* azColumn, /* List of columns */
|
| + const char *zTableName /* Name of the table */
|
| +){
|
| + int i;
|
| + char *zSchema, *zNext;
|
| + const char *zSep = "(";
|
| + zSchema = sqlite3_mprintf("CREATE TABLE x");
|
| + for(i=0; i<nColumn; i++){
|
| + zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
|
| + sqlite3_free(zSchema);
|
| + zSchema = zNext;
|
| + zSep = ",";
|
| + }
|
| + zNext = sqlite3_mprintf("%s,%Q)", zSchema, zTableName);
|
| + sqlite3_free(zSchema);
|
| + return zNext;
|
| +}
|
| +
|
| +/*
|
| +** Build a new sqlite3_vtab structure that will describe the
|
| +** fulltext index defined by spec.
|
| +*/
|
| +static int constructVtab(
|
| + sqlite3 *db, /* The SQLite database connection */
|
| + fts2Hash *pHash, /* Hash table containing tokenizers */
|
| + TableSpec *spec, /* Parsed spec information from parseSpec() */
|
| + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
|
| + char **pzErr /* Write any error message here */
|
| +){
|
| + int rc;
|
| + int n;
|
| + fulltext_vtab *v = 0;
|
| + const sqlite3_tokenizer_module *m = NULL;
|
| + char *schema;
|
| +
|
| + char const *zTok; /* Name of tokenizer to use for this fts table */
|
| + int nTok; /* Length of zTok, including nul terminator */
|
| +
|
| + v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
|
| + if( v==0 ) return SQLITE_NOMEM;
|
| + CLEAR(v);
|
| + /* sqlite will initialize v->base */
|
| + v->db = db;
|
| + v->zDb = spec->zDb; /* Freed when azColumn is freed */
|
| + v->zName = spec->zName; /* Freed when azColumn is freed */
|
| + v->nColumn = spec->nColumn;
|
| + v->azContentColumn = spec->azContentColumn;
|
| + spec->azContentColumn = 0;
|
| + v->azColumn = spec->azColumn;
|
| + spec->azColumn = 0;
|
| +
|
| + if( spec->azTokenizer==0 ){
|
| + return SQLITE_NOMEM;
|
| + }
|
| +
|
| + zTok = spec->azTokenizer[0];
|
| + if( !zTok ){
|
| + zTok = "simple";
|
| + }
|
| + nTok = strlen(zTok)+1;
|
| +
|
| + m = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zTok, nTok);
|
| + if( !m ){
|
| + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
|
| + rc = SQLITE_ERROR;
|
| + goto err;
|
| + }
|
| +
|
| + for(n=0; spec->azTokenizer[n]; n++){}
|
| + if( n ){
|
| + rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1],
|
| + &v->pTokenizer);
|
| + }else{
|
| + rc = m->xCreate(0, 0, &v->pTokenizer);
|
| + }
|
| + if( rc!=SQLITE_OK ) goto err;
|
| + v->pTokenizer->pModule = m;
|
| +
|
| + /* TODO: verify the existence of backing tables foo_content, foo_term */
|
| +
|
| + schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn,
|
| + spec->zName);
|
| + rc = sqlite3_declare_vtab(db, schema);
|
| + sqlite3_free(schema);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| +
|
| + memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
|
| +
|
| + /* Indicate that the buffer is not live. */
|
| + v->nPendingData = -1;
|
| +
|
| + *ppVTab = &v->base;
|
| + TRACE(("FTS2 Connect %p\n", v));
|
| +
|
| + return rc;
|
| +
|
| +err:
|
| + fulltext_vtab_destroy(v);
|
| + return rc;
|
| +}
|
| +
|
| +static int fulltextConnect(
|
| + sqlite3 *db,
|
| + void *pAux,
|
| + int argc, const char *const*argv,
|
| + sqlite3_vtab **ppVTab,
|
| + char **pzErr
|
| +){
|
| + TableSpec spec;
|
| + int rc = parseSpec(&spec, argc, argv, pzErr);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr);
|
| + clearTableSpec(&spec);
|
| + return rc;
|
| +}
|
| +
|
| +/* The %_content table holds the text of each document, with
|
| +** the rowid used as the docid.
|
| +*/
|
| +/* TODO(shess) This comment needs elaboration to match the updated
|
| +** code. Work it into the top-of-file comment at that time.
|
| +*/
|
| +static int fulltextCreate(sqlite3 *db, void *pAux,
|
| + int argc, const char * const *argv,
|
| + sqlite3_vtab **ppVTab, char **pzErr){
|
| + int rc;
|
| + TableSpec spec;
|
| + StringBuffer schema;
|
| + TRACE(("FTS2 Create\n"));
|
| +
|
| + rc = parseSpec(&spec, argc, argv, pzErr);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + initStringBuffer(&schema);
|
| + append(&schema, "CREATE TABLE %_content(");
|
| + appendList(&schema, spec.nColumn, spec.azContentColumn);
|
| + append(&schema, ")");
|
| + rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
|
| + stringBufferDestroy(&schema);
|
| + if( rc!=SQLITE_OK ) goto out;
|
| +
|
| + rc = sql_exec(db, spec.zDb, spec.zName,
|
| + "create table %_segments(block blob);");
|
| + if( rc!=SQLITE_OK ) goto out;
|
| +
|
| + rc = sql_exec(db, spec.zDb, spec.zName,
|
| + "create table %_segdir("
|
| + " level integer,"
|
| + " idx integer,"
|
| + " start_block integer,"
|
| + " leaves_end_block integer,"
|
| + " end_block integer,"
|
| + " root blob,"
|
| + " primary key(level, idx)"
|
| + ");");
|
| + if( rc!=SQLITE_OK ) goto out;
|
| +
|
| + rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr);
|
| +
|
| +out:
|
| + clearTableSpec(&spec);
|
| + return rc;
|
| +}
|
| +
|
| +/* Decide how to handle an SQL query. */
|
| +static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
|
| + int i;
|
| + TRACE(("FTS2 BestIndex\n"));
|
| +
|
| + for(i=0; i<pInfo->nConstraint; ++i){
|
| + const struct sqlite3_index_constraint *pConstraint;
|
| + pConstraint = &pInfo->aConstraint[i];
|
| + if( pConstraint->usable ) {
|
| + if( pConstraint->iColumn==-1 &&
|
| + pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
|
| + pInfo->idxNum = QUERY_ROWID; /* lookup by rowid */
|
| + TRACE(("FTS2 QUERY_ROWID\n"));
|
| + } else if( pConstraint->iColumn>=0 &&
|
| + pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
|
| + /* full-text search */
|
| + pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
|
| + TRACE(("FTS2 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
|
| + } else continue;
|
| +
|
| + pInfo->aConstraintUsage[i].argvIndex = 1;
|
| + pInfo->aConstraintUsage[i].omit = 1;
|
| +
|
| + /* An arbitrary value for now.
|
| + * TODO: Perhaps rowid matches should be considered cheaper than
|
| + * full-text searches. */
|
| + pInfo->estimatedCost = 1.0;
|
| +
|
| + return SQLITE_OK;
|
| + }
|
| + }
|
| + pInfo->idxNum = QUERY_GENERIC;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int fulltextDisconnect(sqlite3_vtab *pVTab){
|
| + TRACE(("FTS2 Disconnect %p\n", pVTab));
|
| + fulltext_vtab_destroy((fulltext_vtab *)pVTab);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int fulltextDestroy(sqlite3_vtab *pVTab){
|
| + fulltext_vtab *v = (fulltext_vtab *)pVTab;
|
| + int rc;
|
| +
|
| + TRACE(("FTS2 Destroy %p\n", pVTab));
|
| + rc = sql_exec(v->db, v->zDb, v->zName,
|
| + "drop table if exists %_content;"
|
| + "drop table if exists %_segments;"
|
| + "drop table if exists %_segdir;"
|
| + );
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + fulltext_vtab_destroy((fulltext_vtab *)pVTab);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
|
| + fulltext_cursor *c;
|
| +
|
| + c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
|
| + if( c ){
|
| + memset(c, 0, sizeof(fulltext_cursor));
|
| + /* sqlite will initialize c->base */
|
| + *ppCursor = &c->base;
|
| + TRACE(("FTS2 Open %p: %p\n", pVTab, c));
|
| + return SQLITE_OK;
|
| + }else{
|
| + return SQLITE_NOMEM;
|
| + }
|
| +}
|
| +
|
| +
|
| +/* Free all of the dynamically allocated memory held by *q
|
| +*/
|
| +static void queryClear(Query *q){
|
| + int i;
|
| + for(i = 0; i < q->nTerms; ++i){
|
| + sqlite3_free(q->pTerms[i].pTerm);
|
| + }
|
| + sqlite3_free(q->pTerms);
|
| + CLEAR(q);
|
| +}
|
| +
|
| +/* Free all of the dynamically allocated memory held by the
|
| +** Snippet
|
| +*/
|
| +static void snippetClear(Snippet *p){
|
| + sqlite3_free(p->aMatch);
|
| + sqlite3_free(p->zOffset);
|
| + sqlite3_free(p->zSnippet);
|
| + CLEAR(p);
|
| +}
|
| +/*
|
| +** Append a single entry to the p->aMatch[] log.
|
| +*/
|
| +static void snippetAppendMatch(
|
| + Snippet *p, /* Append the entry to this snippet */
|
| + int iCol, int iTerm, /* The column and query term */
|
| + int iStart, int nByte /* Offset and size of the match */
|
| +){
|
| + int i;
|
| + struct snippetMatch *pMatch;
|
| + if( p->nMatch+1>=p->nAlloc ){
|
| + p->nAlloc = p->nAlloc*2 + 10;
|
| + p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
|
| + if( p->aMatch==0 ){
|
| + p->nMatch = 0;
|
| + p->nAlloc = 0;
|
| + return;
|
| + }
|
| + }
|
| + i = p->nMatch++;
|
| + pMatch = &p->aMatch[i];
|
| + pMatch->iCol = iCol;
|
| + pMatch->iTerm = iTerm;
|
| + pMatch->iStart = iStart;
|
| + pMatch->nByte = nByte;
|
| +}
|
| +
|
| +/*
|
| +** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
|
| +*/
|
| +#define FTS2_ROTOR_SZ (32)
|
| +#define FTS2_ROTOR_MASK (FTS2_ROTOR_SZ-1)
|
| +
|
| +/*
|
| +** Add entries to pSnippet->aMatch[] for every match that occurs against
|
| +** document zDoc[0..nDoc-1] which is stored in column iColumn.
|
| +*/
|
| +static void snippetOffsetsOfColumn(
|
| + Query *pQuery,
|
| + Snippet *pSnippet,
|
| + int iColumn,
|
| + const char *zDoc,
|
| + int nDoc
|
| +){
|
| + const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */
|
| + sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */
|
| + sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */
|
| + fulltext_vtab *pVtab; /* The full text index */
|
| + int nColumn; /* Number of columns in the index */
|
| + const QueryTerm *aTerm; /* Query string terms */
|
| + int nTerm; /* Number of query string terms */
|
| + int i, j; /* Loop counters */
|
| + int rc; /* Return code */
|
| + unsigned int match, prevMatch; /* Phrase search bitmasks */
|
| + const char *zToken; /* Next token from the tokenizer */
|
| + int nToken; /* Size of zToken */
|
| + int iBegin, iEnd, iPos; /* Offsets of beginning and end */
|
| +
|
| + /* The following variables keep a circular buffer of the last
|
| + ** few tokens */
|
| + unsigned int iRotor = 0; /* Index of current token */
|
| + int iRotorBegin[FTS2_ROTOR_SZ]; /* Beginning offset of token */
|
| + int iRotorLen[FTS2_ROTOR_SZ]; /* Length of token */
|
| +
|
| + pVtab = pQuery->pFts;
|
| + nColumn = pVtab->nColumn;
|
| + pTokenizer = pVtab->pTokenizer;
|
| + pTModule = pTokenizer->pModule;
|
| + rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
|
| + if( rc ) return;
|
| + pTCursor->pTokenizer = pTokenizer;
|
| + aTerm = pQuery->pTerms;
|
| + nTerm = pQuery->nTerms;
|
| + if( nTerm>=FTS2_ROTOR_SZ ){
|
| + nTerm = FTS2_ROTOR_SZ - 1;
|
| + }
|
| + prevMatch = 0;
|
| + while(1){
|
| + rc = pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
|
| + if( rc ) break;
|
| + iRotorBegin[iRotor&FTS2_ROTOR_MASK] = iBegin;
|
| + iRotorLen[iRotor&FTS2_ROTOR_MASK] = iEnd-iBegin;
|
| + match = 0;
|
| + for(i=0; i<nTerm; i++){
|
| + int iCol;
|
| + iCol = aTerm[i].iColumn;
|
| + if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
|
| + if( aTerm[i].nTerm>nToken ) continue;
|
| + if( !aTerm[i].isPrefix && aTerm[i].nTerm<nToken ) continue;
|
| + assert( aTerm[i].nTerm<=nToken );
|
| + if( memcmp(aTerm[i].pTerm, zToken, aTerm[i].nTerm) ) continue;
|
| + if( aTerm[i].iPhrase>1 && (prevMatch & (1<<i))==0 ) continue;
|
| + match |= 1<<i;
|
| + if( i==nTerm-1 || aTerm[i+1].iPhrase==1 ){
|
| + for(j=aTerm[i].iPhrase-1; j>=0; j--){
|
| + int k = (iRotor-j) & FTS2_ROTOR_MASK;
|
| + snippetAppendMatch(pSnippet, iColumn, i-j,
|
| + iRotorBegin[k], iRotorLen[k]);
|
| + }
|
| + }
|
| + }
|
| + prevMatch = match<<1;
|
| + iRotor++;
|
| + }
|
| + pTModule->xClose(pTCursor);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Compute all offsets for the current row of the query.
|
| +** If the offsets have already been computed, this routine is a no-op.
|
| +*/
|
| +static void snippetAllOffsets(fulltext_cursor *p){
|
| + int nColumn;
|
| + int iColumn, i;
|
| + int iFirst, iLast;
|
| + fulltext_vtab *pFts;
|
| +
|
| + if( p->snippet.nMatch ) return;
|
| + if( p->q.nTerms==0 ) return;
|
| + pFts = p->q.pFts;
|
| + nColumn = pFts->nColumn;
|
| + iColumn = (p->iCursorType - QUERY_FULLTEXT);
|
| + if( iColumn<0 || iColumn>=nColumn ){
|
| + iFirst = 0;
|
| + iLast = nColumn-1;
|
| + }else{
|
| + iFirst = iColumn;
|
| + iLast = iColumn;
|
| + }
|
| + for(i=iFirst; i<=iLast; i++){
|
| + const char *zDoc;
|
| + int nDoc;
|
| + zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
|
| + nDoc = sqlite3_column_bytes(p->pStmt, i+1);
|
| + snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Convert the information in the aMatch[] array of the snippet
|
| +** into the string zOffset[0..nOffset-1].
|
| +*/
|
| +static void snippetOffsetText(Snippet *p){
|
| + int i;
|
| + int cnt = 0;
|
| + StringBuffer sb;
|
| + char zBuf[200];
|
| + if( p->zOffset ) return;
|
| + initStringBuffer(&sb);
|
| + for(i=0; i<p->nMatch; i++){
|
| + struct snippetMatch *pMatch = &p->aMatch[i];
|
| + zBuf[0] = ' ';
|
| + sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
|
| + pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
|
| + append(&sb, zBuf);
|
| + cnt++;
|
| + }
|
| + p->zOffset = stringBufferData(&sb);
|
| + p->nOffset = stringBufferLength(&sb);
|
| +}
|
| +
|
| +/*
|
| +** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
|
| +** of matching words some of which might be in zDoc. zDoc is column
|
| +** number iCol.
|
| +**
|
| +** iBreak is suggested spot in zDoc where we could begin or end an
|
| +** excerpt. Return a value similar to iBreak but possibly adjusted
|
| +** to be a little left or right so that the break point is better.
|
| +*/
|
| +static int wordBoundary(
|
| + int iBreak, /* The suggested break point */
|
| + const char *zDoc, /* Document text */
|
| + int nDoc, /* Number of bytes in zDoc[] */
|
| + struct snippetMatch *aMatch, /* Matching words */
|
| + int nMatch, /* Number of entries in aMatch[] */
|
| + int iCol /* The column number for zDoc[] */
|
| +){
|
| + int i;
|
| + if( iBreak<=10 ){
|
| + return 0;
|
| + }
|
| + if( iBreak>=nDoc-10 ){
|
| + return nDoc;
|
| + }
|
| + for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
|
| + while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
|
| + if( i<nMatch ){
|
| + if( aMatch[i].iStart<iBreak+10 ){
|
| + return aMatch[i].iStart;
|
| + }
|
| + if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
|
| + return aMatch[i-1].iStart;
|
| + }
|
| + }
|
| + for(i=1; i<=10; i++){
|
| + if( safe_isspace(zDoc[iBreak-i]) ){
|
| + return iBreak - i + 1;
|
| + }
|
| + if( safe_isspace(zDoc[iBreak+i]) ){
|
| + return iBreak + i + 1;
|
| + }
|
| + }
|
| + return iBreak;
|
| +}
|
| +
|
| +
|
| +
|
| +/*
|
| +** Allowed values for Snippet.aMatch[].snStatus
|
| +*/
|
| +#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
|
| +#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
|
| +
|
| +/*
|
| +** Generate the text of a snippet.
|
| +*/
|
| +static void snippetText(
|
| + fulltext_cursor *pCursor, /* The cursor we need the snippet for */
|
| + const char *zStartMark, /* Markup to appear before each match */
|
| + const char *zEndMark, /* Markup to appear after each match */
|
| + const char *zEllipsis /* Ellipsis mark */
|
| +){
|
| + int i, j;
|
| + struct snippetMatch *aMatch;
|
| + int nMatch;
|
| + int nDesired;
|
| + StringBuffer sb;
|
| + int tailCol;
|
| + int tailOffset;
|
| + int iCol;
|
| + int nDoc;
|
| + const char *zDoc;
|
| + int iStart, iEnd;
|
| + int tailEllipsis = 0;
|
| + int iMatch;
|
| +
|
| +
|
| + sqlite3_free(pCursor->snippet.zSnippet);
|
| + pCursor->snippet.zSnippet = 0;
|
| + aMatch = pCursor->snippet.aMatch;
|
| + nMatch = pCursor->snippet.nMatch;
|
| + initStringBuffer(&sb);
|
| +
|
| + for(i=0; i<nMatch; i++){
|
| + aMatch[i].snStatus = SNIPPET_IGNORE;
|
| + }
|
| + nDesired = 0;
|
| + for(i=0; i<pCursor->q.nTerms; i++){
|
| + for(j=0; j<nMatch; j++){
|
| + if( aMatch[j].iTerm==i ){
|
| + aMatch[j].snStatus = SNIPPET_DESIRED;
|
| + nDesired++;
|
| + break;
|
| + }
|
| + }
|
| + }
|
| +
|
| + iMatch = 0;
|
| + tailCol = -1;
|
| + tailOffset = 0;
|
| + for(i=0; i<nMatch && nDesired>0; i++){
|
| + if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
|
| + nDesired--;
|
| + iCol = aMatch[i].iCol;
|
| + zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
|
| + nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
|
| + iStart = aMatch[i].iStart - 40;
|
| + iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
|
| + if( iStart<=10 ){
|
| + iStart = 0;
|
| + }
|
| + if( iCol==tailCol && iStart<=tailOffset+20 ){
|
| + iStart = tailOffset;
|
| + }
|
| + if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
|
| + trimWhiteSpace(&sb);
|
| + appendWhiteSpace(&sb);
|
| + append(&sb, zEllipsis);
|
| + appendWhiteSpace(&sb);
|
| + }
|
| + iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
|
| + iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
|
| + if( iEnd>=nDoc-10 ){
|
| + iEnd = nDoc;
|
| + tailEllipsis = 0;
|
| + }else{
|
| + tailEllipsis = 1;
|
| + }
|
| + while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
|
| + while( iStart<iEnd ){
|
| + while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
|
| + && aMatch[iMatch].iCol<=iCol ){
|
| + iMatch++;
|
| + }
|
| + if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
|
| + && aMatch[iMatch].iCol==iCol ){
|
| + nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
|
| + iStart = aMatch[iMatch].iStart;
|
| + append(&sb, zStartMark);
|
| + nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
|
| + append(&sb, zEndMark);
|
| + iStart += aMatch[iMatch].nByte;
|
| + for(j=iMatch+1; j<nMatch; j++){
|
| + if( aMatch[j].iTerm==aMatch[iMatch].iTerm
|
| + && aMatch[j].snStatus==SNIPPET_DESIRED ){
|
| + nDesired--;
|
| + aMatch[j].snStatus = SNIPPET_IGNORE;
|
| + }
|
| + }
|
| + }else{
|
| + nappend(&sb, &zDoc[iStart], iEnd - iStart);
|
| + iStart = iEnd;
|
| + }
|
| + }
|
| + tailCol = iCol;
|
| + tailOffset = iEnd;
|
| + }
|
| + trimWhiteSpace(&sb);
|
| + if( tailEllipsis ){
|
| + appendWhiteSpace(&sb);
|
| + append(&sb, zEllipsis);
|
| + }
|
| + pCursor->snippet.zSnippet = stringBufferData(&sb);
|
| + pCursor->snippet.nSnippet = stringBufferLength(&sb);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Close the cursor. For additional information see the documentation
|
| +** on the xClose method of the virtual table interface.
|
| +*/
|
| +static int fulltextClose(sqlite3_vtab_cursor *pCursor){
|
| + fulltext_cursor *c = (fulltext_cursor *) pCursor;
|
| + TRACE(("FTS2 Close %p\n", c));
|
| + sqlite3_finalize(c->pStmt);
|
| + queryClear(&c->q);
|
| + snippetClear(&c->snippet);
|
| + if( c->result.nData!=0 ) dlrDestroy(&c->reader);
|
| + dataBufferDestroy(&c->result);
|
| + sqlite3_free(c);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int fulltextNext(sqlite3_vtab_cursor *pCursor){
|
| + fulltext_cursor *c = (fulltext_cursor *) pCursor;
|
| + int rc;
|
| +
|
| + TRACE(("FTS2 Next %p\n", pCursor));
|
| + snippetClear(&c->snippet);
|
| + if( c->iCursorType < QUERY_FULLTEXT ){
|
| + /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
|
| + rc = sqlite3_step(c->pStmt);
|
| + switch( rc ){
|
| + case SQLITE_ROW:
|
| + c->eof = 0;
|
| + return SQLITE_OK;
|
| + case SQLITE_DONE:
|
| + c->eof = 1;
|
| + return SQLITE_OK;
|
| + default:
|
| + c->eof = 1;
|
| + return rc;
|
| + }
|
| + } else { /* full-text query */
|
| + rc = sqlite3_reset(c->pStmt);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + if( c->result.nData==0 || dlrAtEnd(&c->reader) ){
|
| + c->eof = 1;
|
| + return SQLITE_OK;
|
| + }
|
| + rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
|
| + dlrStep(&c->reader);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
|
| + rc = sqlite3_step(c->pStmt);
|
| + if( rc==SQLITE_ROW ){ /* the case we expect */
|
| + c->eof = 0;
|
| + return SQLITE_OK;
|
| + }
|
| + /* an error occurred; abort */
|
| + return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
|
| + }
|
| +}
|
| +
|
| +
|
| +/* TODO(shess) If we pushed LeafReader to the top of the file, or to
|
| +** another file, term_select() could be pushed above
|
| +** docListOfTerm().
|
| +*/
|
| +static int termSelect(fulltext_vtab *v, int iColumn,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + DocListType iType, DataBuffer *out);
|
| +
|
| +/* Return a DocList corresponding to the query term *pTerm. If *pTerm
|
| +** is the first term of a phrase query, go ahead and evaluate the phrase
|
| +** query and return the doclist for the entire phrase query.
|
| +**
|
| +** The resulting DL_DOCIDS doclist is stored in pResult, which is
|
| +** overwritten.
|
| +*/
|
| +static int docListOfTerm(
|
| + fulltext_vtab *v, /* The full text index */
|
| + int iColumn, /* column to restrict to. No restriction if >=nColumn */
|
| + QueryTerm *pQTerm, /* Term we are looking for, or 1st term of a phrase */
|
| + DataBuffer *pResult /* Write the result here */
|
| +){
|
| + DataBuffer left, right, new;
|
| + int i, rc;
|
| +
|
| + /* No phrase search if no position info. */
|
| + assert( pQTerm->nPhrase==0 || DL_DEFAULT!=DL_DOCIDS );
|
| +
|
| + /* This code should never be called with buffered updates. */
|
| + assert( v->nPendingData<0 );
|
| +
|
| + dataBufferInit(&left, 0);
|
| + rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pQTerm->isPrefix,
|
| + 0<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &left);
|
| + if( rc ) return rc;
|
| + for(i=1; i<=pQTerm->nPhrase && left.nData>0; i++){
|
| + dataBufferInit(&right, 0);
|
| + rc = termSelect(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm,
|
| + pQTerm[i].isPrefix, DL_POSITIONS, &right);
|
| + if( rc ){
|
| + dataBufferDestroy(&left);
|
| + return rc;
|
| + }
|
| + dataBufferInit(&new, 0);
|
| + docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
|
| + i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new);
|
| + dataBufferDestroy(&left);
|
| + dataBufferDestroy(&right);
|
| + left = new;
|
| + }
|
| + *pResult = left;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Add a new term pTerm[0..nTerm-1] to the query *q.
|
| +*/
|
| +static void queryAdd(Query *q, const char *pTerm, int nTerm){
|
| + QueryTerm *t;
|
| + ++q->nTerms;
|
| + q->pTerms = sqlite3_realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0]));
|
| + if( q->pTerms==0 ){
|
| + q->nTerms = 0;
|
| + return;
|
| + }
|
| + t = &q->pTerms[q->nTerms - 1];
|
| + CLEAR(t);
|
| + t->pTerm = sqlite3_malloc(nTerm+1);
|
| + memcpy(t->pTerm, pTerm, nTerm);
|
| + t->pTerm[nTerm] = 0;
|
| + t->nTerm = nTerm;
|
| + t->isOr = q->nextIsOr;
|
| + t->isPrefix = 0;
|
| + q->nextIsOr = 0;
|
| + t->iColumn = q->nextColumn;
|
| + q->nextColumn = q->dfltColumn;
|
| +}
|
| +
|
| +/*
|
| +** Check to see if the string zToken[0...nToken-1] matches any
|
| +** column name in the virtual table. If it does,
|
| +** return the zero-indexed column number. If not, return -1.
|
| +*/
|
| +static int checkColumnSpecifier(
|
| + fulltext_vtab *pVtab, /* The virtual table */
|
| + const char *zToken, /* Text of the token */
|
| + int nToken /* Number of characters in the token */
|
| +){
|
| + int i;
|
| + for(i=0; i<pVtab->nColumn; i++){
|
| + if( memcmp(pVtab->azColumn[i], zToken, nToken)==0
|
| + && pVtab->azColumn[i][nToken]==0 ){
|
| + return i;
|
| + }
|
| + }
|
| + return -1;
|
| +}
|
| +
|
| +/*
|
| +** Parse the text at pSegment[0..nSegment-1]. Add additional terms
|
| +** to the query being assemblied in pQuery.
|
| +**
|
| +** inPhrase is true if pSegment[0..nSegement-1] is contained within
|
| +** double-quotes. If inPhrase is true, then the first term
|
| +** is marked with the number of terms in the phrase less one and
|
| +** OR and "-" syntax is ignored. If inPhrase is false, then every
|
| +** term found is marked with nPhrase=0 and OR and "-" syntax is significant.
|
| +*/
|
| +static int tokenizeSegment(
|
| + sqlite3_tokenizer *pTokenizer, /* The tokenizer to use */
|
| + const char *pSegment, int nSegment, /* Query expression being parsed */
|
| + int inPhrase, /* True if within "..." */
|
| + Query *pQuery /* Append results here */
|
| +){
|
| + const sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
|
| + sqlite3_tokenizer_cursor *pCursor;
|
| + int firstIndex = pQuery->nTerms;
|
| + int iCol;
|
| + int nTerm = 1;
|
| +
|
| + int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + pCursor->pTokenizer = pTokenizer;
|
| +
|
| + while( 1 ){
|
| + const char *pToken;
|
| + int nToken, iBegin, iEnd, iPos;
|
| +
|
| + rc = pModule->xNext(pCursor,
|
| + &pToken, &nToken,
|
| + &iBegin, &iEnd, &iPos);
|
| + if( rc!=SQLITE_OK ) break;
|
| + if( !inPhrase &&
|
| + pSegment[iEnd]==':' &&
|
| + (iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){
|
| + pQuery->nextColumn = iCol;
|
| + continue;
|
| + }
|
| + if( !inPhrase && pQuery->nTerms>0 && nToken==2
|
| + && pSegment[iBegin]=='O' && pSegment[iBegin+1]=='R' ){
|
| + pQuery->nextIsOr = 1;
|
| + continue;
|
| + }
|
| + queryAdd(pQuery, pToken, nToken);
|
| + if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
|
| + pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
|
| + }
|
| + if( iEnd<nSegment && pSegment[iEnd]=='*' ){
|
| + pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
|
| + }
|
| + pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm;
|
| + if( inPhrase ){
|
| + nTerm++;
|
| + }
|
| + }
|
| +
|
| + if( inPhrase && pQuery->nTerms>firstIndex ){
|
| + pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1;
|
| + }
|
| +
|
| + return pModule->xClose(pCursor);
|
| +}
|
| +
|
| +/* Parse a query string, yielding a Query object pQuery.
|
| +**
|
| +** The calling function will need to queryClear() to clean up
|
| +** the dynamically allocated memory held by pQuery.
|
| +*/
|
| +static int parseQuery(
|
| + fulltext_vtab *v, /* The fulltext index */
|
| + const char *zInput, /* Input text of the query string */
|
| + int nInput, /* Size of the input text */
|
| + int dfltColumn, /* Default column of the index to match against */
|
| + Query *pQuery /* Write the parse results here. */
|
| +){
|
| + int iInput, inPhrase = 0;
|
| +
|
| + if( zInput==0 ) nInput = 0;
|
| + if( nInput<0 ) nInput = strlen(zInput);
|
| + pQuery->nTerms = 0;
|
| + pQuery->pTerms = NULL;
|
| + pQuery->nextIsOr = 0;
|
| + pQuery->nextColumn = dfltColumn;
|
| + pQuery->dfltColumn = dfltColumn;
|
| + pQuery->pFts = v;
|
| +
|
| + for(iInput=0; iInput<nInput; ++iInput){
|
| + int i;
|
| + for(i=iInput; i<nInput && zInput[i]!='"'; ++i){}
|
| + if( i>iInput ){
|
| + tokenizeSegment(v->pTokenizer, zInput+iInput, i-iInput, inPhrase,
|
| + pQuery);
|
| + }
|
| + iInput = i;
|
| + if( i<nInput ){
|
| + assert( zInput[i]=='"' );
|
| + inPhrase = !inPhrase;
|
| + }
|
| + }
|
| +
|
| + if( inPhrase ){
|
| + /* unmatched quote */
|
| + queryClear(pQuery);
|
| + return SQLITE_ERROR;
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* TODO(shess) Refactor the code to remove this forward decl. */
|
| +static int flushPendingTerms(fulltext_vtab *v);
|
| +
|
| +/* Perform a full-text query using the search expression in
|
| +** zInput[0..nInput-1]. Return a list of matching documents
|
| +** in pResult.
|
| +**
|
| +** Queries must match column iColumn. Or if iColumn>=nColumn
|
| +** they are allowed to match against any column.
|
| +*/
|
| +static int fulltextQuery(
|
| + fulltext_vtab *v, /* The full text index */
|
| + int iColumn, /* Match against this column by default */
|
| + const char *zInput, /* The query string */
|
| + int nInput, /* Number of bytes in zInput[] */
|
| + DataBuffer *pResult, /* Write the result doclist here */
|
| + Query *pQuery /* Put parsed query string here */
|
| +){
|
| + int i, iNext, rc;
|
| + DataBuffer left, right, or, new;
|
| + int nNot = 0;
|
| + QueryTerm *aTerm;
|
| +
|
| + /* TODO(shess) Instead of flushing pendingTerms, we could query for
|
| + ** the relevant term and merge the doclist into what we receive from
|
| + ** the database. Wait and see if this is a common issue, first.
|
| + **
|
| + ** A good reason not to flush is to not generate update-related
|
| + ** error codes from here.
|
| + */
|
| +
|
| + /* Flush any buffered updates before executing the query. */
|
| + rc = flushPendingTerms(v);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* TODO(shess) I think that the queryClear() calls below are not
|
| + ** necessary, because fulltextClose() already clears the query.
|
| + */
|
| + rc = parseQuery(v, zInput, nInput, iColumn, pQuery);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* Empty or NULL queries return no results. */
|
| + if( pQuery->nTerms==0 ){
|
| + dataBufferInit(pResult, 0);
|
| + return SQLITE_OK;
|
| + }
|
| +
|
| + /* Merge AND terms. */
|
| + /* TODO(shess) I think we can early-exit if( i>nNot && left.nData==0 ). */
|
| + aTerm = pQuery->pTerms;
|
| + for(i = 0; i<pQuery->nTerms; i=iNext){
|
| + if( aTerm[i].isNot ){
|
| + /* Handle all NOT terms in a separate pass */
|
| + nNot++;
|
| + iNext = i + aTerm[i].nPhrase+1;
|
| + continue;
|
| + }
|
| + iNext = i + aTerm[i].nPhrase + 1;
|
| + rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
|
| + if( rc ){
|
| + if( i!=nNot ) dataBufferDestroy(&left);
|
| + queryClear(pQuery);
|
| + return rc;
|
| + }
|
| + while( iNext<pQuery->nTerms && aTerm[iNext].isOr ){
|
| + rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &or);
|
| + iNext += aTerm[iNext].nPhrase + 1;
|
| + if( rc ){
|
| + if( i!=nNot ) dataBufferDestroy(&left);
|
| + dataBufferDestroy(&right);
|
| + queryClear(pQuery);
|
| + return rc;
|
| + }
|
| + dataBufferInit(&new, 0);
|
| + docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
|
| + dataBufferDestroy(&right);
|
| + dataBufferDestroy(&or);
|
| + right = new;
|
| + }
|
| + if( i==nNot ){ /* first term processed. */
|
| + left = right;
|
| + }else{
|
| + dataBufferInit(&new, 0);
|
| + docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new);
|
| + dataBufferDestroy(&right);
|
| + dataBufferDestroy(&left);
|
| + left = new;
|
| + }
|
| + }
|
| +
|
| + if( nNot==pQuery->nTerms ){
|
| + /* We do not yet know how to handle a query of only NOT terms */
|
| + return SQLITE_ERROR;
|
| + }
|
| +
|
| + /* Do the EXCEPT terms */
|
| + for(i=0; i<pQuery->nTerms; i += aTerm[i].nPhrase + 1){
|
| + if( !aTerm[i].isNot ) continue;
|
| + rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
|
| + if( rc ){
|
| + queryClear(pQuery);
|
| + dataBufferDestroy(&left);
|
| + return rc;
|
| + }
|
| + dataBufferInit(&new, 0);
|
| + docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new);
|
| + dataBufferDestroy(&right);
|
| + dataBufferDestroy(&left);
|
| + left = new;
|
| + }
|
| +
|
| + *pResult = left;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** This is the xFilter interface for the virtual table. See
|
| +** the virtual table xFilter method documentation for additional
|
| +** information.
|
| +**
|
| +** If idxNum==QUERY_GENERIC then do a full table scan against
|
| +** the %_content table.
|
| +**
|
| +** If idxNum==QUERY_ROWID then do a rowid lookup for a single entry
|
| +** in the %_content table.
|
| +**
|
| +** If idxNum>=QUERY_FULLTEXT then use the full text index. The
|
| +** column on the left-hand side of the MATCH operator is column
|
| +** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
|
| +** side of the MATCH operator.
|
| +*/
|
| +/* TODO(shess) Upgrade the cursor initialization and destruction to
|
| +** account for fulltextFilter() being called multiple times on the
|
| +** same cursor. The current solution is very fragile. Apply fix to
|
| +** fts2 as appropriate.
|
| +*/
|
| +static int fulltextFilter(
|
| + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
|
| + int idxNum, const char *idxStr, /* Which indexing scheme to use */
|
| + int argc, sqlite3_value **argv /* Arguments for the indexing scheme */
|
| +){
|
| + fulltext_cursor *c = (fulltext_cursor *) pCursor;
|
| + fulltext_vtab *v = cursor_vtab(c);
|
| + int rc;
|
| +
|
| + TRACE(("FTS2 Filter %p\n",pCursor));
|
| +
|
| + /* If the cursor has a statement that was not prepared according to
|
| + ** idxNum, clear it. I believe all calls to fulltextFilter with a
|
| + ** given cursor will have the same idxNum , but in this case it's
|
| + ** easy to be safe.
|
| + */
|
| + if( c->pStmt && c->iCursorType!=idxNum ){
|
| + sqlite3_finalize(c->pStmt);
|
| + c->pStmt = NULL;
|
| + }
|
| +
|
| + /* Get a fresh statement appropriate to idxNum. */
|
| + /* TODO(shess): Add a prepared-statement cache in the vt structure.
|
| + ** The cache must handle multiple open cursors. Easier to cache the
|
| + ** statement variants at the vt to reduce malloc/realloc/free here.
|
| + ** Or we could have a StringBuffer variant which allowed stack
|
| + ** construction for small values.
|
| + */
|
| + if( !c->pStmt ){
|
| + char *zSql = sqlite3_mprintf("select rowid, * from %%_content %s",
|
| + idxNum==QUERY_GENERIC ? "" : "where rowid=?");
|
| + rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql);
|
| + sqlite3_free(zSql);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + c->iCursorType = idxNum;
|
| + }else{
|
| + sqlite3_reset(c->pStmt);
|
| + assert( c->iCursorType==idxNum );
|
| + }
|
| +
|
| + switch( idxNum ){
|
| + case QUERY_GENERIC:
|
| + break;
|
| +
|
| + case QUERY_ROWID:
|
| + rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + break;
|
| +
|
| + default: /* full-text search */
|
| + {
|
| + const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
|
| + assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
|
| + assert( argc==1 );
|
| + queryClear(&c->q);
|
| + if( c->result.nData!=0 ){
|
| + /* This case happens if the same cursor is used repeatedly. */
|
| + dlrDestroy(&c->reader);
|
| + dataBufferReset(&c->result);
|
| + }else{
|
| + dataBufferInit(&c->result, 0);
|
| + }
|
| + rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + if( c->result.nData!=0 ){
|
| + dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
|
| + }
|
| + break;
|
| + }
|
| + }
|
| +
|
| + return fulltextNext(pCursor);
|
| +}
|
| +
|
| +/* This is the xEof method of the virtual table. The SQLite core
|
| +** calls this routine to find out if it has reached the end of
|
| +** a query's results set.
|
| +*/
|
| +static int fulltextEof(sqlite3_vtab_cursor *pCursor){
|
| + fulltext_cursor *c = (fulltext_cursor *) pCursor;
|
| + return c->eof;
|
| +}
|
| +
|
| +/* This is the xColumn method of the virtual table. The SQLite
|
| +** core calls this method during a query when it needs the value
|
| +** of a column from the virtual table. This method needs to use
|
| +** one of the sqlite3_result_*() routines to store the requested
|
| +** value back in the pContext.
|
| +*/
|
| +static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
|
| + sqlite3_context *pContext, int idxCol){
|
| + fulltext_cursor *c = (fulltext_cursor *) pCursor;
|
| + fulltext_vtab *v = cursor_vtab(c);
|
| +
|
| + if( idxCol<v->nColumn ){
|
| + sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
|
| + sqlite3_result_value(pContext, pVal);
|
| + }else if( idxCol==v->nColumn ){
|
| + /* The extra column whose name is the same as the table.
|
| + ** Return a blob which is a pointer to the cursor
|
| + */
|
| + sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* This is the xRowid method. The SQLite core calls this routine to
|
| +** retrive the rowid for the current row of the result set. The
|
| +** rowid should be written to *pRowid.
|
| +*/
|
| +static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
|
| + fulltext_cursor *c = (fulltext_cursor *) pCursor;
|
| +
|
| + *pRowid = sqlite3_column_int64(c->pStmt, 0);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
|
| +** we also store positions and offsets in the hash table using that
|
| +** column number.
|
| +*/
|
| +static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
|
| + const char *zText, int iColumn){
|
| + sqlite3_tokenizer *pTokenizer = v->pTokenizer;
|
| + sqlite3_tokenizer_cursor *pCursor;
|
| + const char *pToken;
|
| + int nTokenBytes;
|
| + int iStartOffset, iEndOffset, iPosition;
|
| + int rc;
|
| +
|
| + rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + pCursor->pTokenizer = pTokenizer;
|
| + while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
|
| + &pToken, &nTokenBytes,
|
| + &iStartOffset, &iEndOffset,
|
| + &iPosition)) ){
|
| + DLCollector *p;
|
| + int nData; /* Size of doclist before our update. */
|
| +
|
| + /* Positions can't be negative; we use -1 as a terminator
|
| + * internally. Token can't be NULL or empty. */
|
| + if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
|
| + rc = SQLITE_ERROR;
|
| + break;
|
| + }
|
| +
|
| + p = fts2HashFind(&v->pendingTerms, pToken, nTokenBytes);
|
| + if( p==NULL ){
|
| + nData = 0;
|
| + p = dlcNew(iDocid, DL_DEFAULT);
|
| + fts2HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
|
| +
|
| + /* Overhead for our hash table entry, the key, and the value. */
|
| + v->nPendingData += sizeof(struct fts2HashElem)+sizeof(*p)+nTokenBytes;
|
| + }else{
|
| + nData = p->b.nData;
|
| + if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
|
| + }
|
| + if( iColumn>=0 ){
|
| + dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
|
| + }
|
| +
|
| + /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
|
| + v->nPendingData += p->b.nData-nData;
|
| + }
|
| +
|
| + /* TODO(shess) Check return? Should this be able to cause errors at
|
| + ** this point? Actually, same question about sqlite3_finalize(),
|
| + ** though one could argue that failure there means that the data is
|
| + ** not durable. *ponder*
|
| + */
|
| + pTokenizer->pModule->xClose(pCursor);
|
| + if( SQLITE_DONE == rc ) return SQLITE_OK;
|
| + return rc;
|
| +}
|
| +
|
| +/* Add doclists for all terms in [pValues] to pendingTerms table. */
|
| +static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid,
|
| + sqlite3_value **pValues){
|
| + int i;
|
| + for(i = 0; i < v->nColumn ; ++i){
|
| + char *zText = (char*)sqlite3_value_text(pValues[i]);
|
| + int rc = buildTerms(v, iRowid, zText, i);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Add empty doclists for all terms in the given row's content to
|
| +** pendingTerms.
|
| +*/
|
| +static int deleteTerms(fulltext_vtab *v, sqlite_int64 iRowid){
|
| + const char **pValues;
|
| + int i, rc;
|
| +
|
| + /* TODO(shess) Should we allow such tables at all? */
|
| + if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
|
| +
|
| + rc = content_select(v, iRowid, &pValues);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + for(i = 0 ; i < v->nColumn; ++i) {
|
| + rc = buildTerms(v, iRowid, pValues[i], -1);
|
| + if( rc!=SQLITE_OK ) break;
|
| + }
|
| +
|
| + freeStringArray(v->nColumn, pValues);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* TODO(shess) Refactor the code to remove this forward decl. */
|
| +static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
|
| +
|
| +/* Insert a row into the %_content table; set *piRowid to be the ID of the
|
| +** new row. Add doclists for terms to pendingTerms.
|
| +*/
|
| +static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestRowid,
|
| + sqlite3_value **pValues, sqlite_int64 *piRowid){
|
| + int rc;
|
| +
|
| + rc = content_insert(v, pRequestRowid, pValues); /* execute an SQL INSERT */
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + *piRowid = sqlite3_last_insert_rowid(v->db);
|
| + rc = initPendingTerms(v, *piRowid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return insertTerms(v, *piRowid, pValues);
|
| +}
|
| +
|
| +/* Delete a row from the %_content table; add empty doclists for terms
|
| +** to pendingTerms.
|
| +*/
|
| +static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
|
| + int rc = initPendingTerms(v, iRow);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = deleteTerms(v, iRow);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + return content_delete(v, iRow); /* execute an SQL DELETE */
|
| +}
|
| +
|
| +/* Update a row in the %_content table; add delete doclists to
|
| +** pendingTerms for old terms not in the new data, add insert doclists
|
| +** to pendingTerms for terms in the new data.
|
| +*/
|
| +static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
|
| + sqlite3_value **pValues){
|
| + int rc = initPendingTerms(v, iRow);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* Generate an empty doclist for each term that previously appeared in this
|
| + * row. */
|
| + rc = deleteTerms(v, iRow);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* Now add positions for terms which appear in the updated row. */
|
| + return insertTerms(v, iRow, pValues);
|
| +}
|
| +
|
| +/*******************************************************************/
|
| +/* InteriorWriter is used to collect terms and block references into
|
| +** interior nodes in %_segments. See commentary at top of file for
|
| +** format.
|
| +*/
|
| +
|
| +/* How large interior nodes can grow. */
|
| +#define INTERIOR_MAX 2048
|
| +
|
| +/* Minimum number of terms per interior node (except the root). This
|
| +** prevents large terms from making the tree too skinny - must be >0
|
| +** so that the tree always makes progress. Note that the min tree
|
| +** fanout will be INTERIOR_MIN_TERMS+1.
|
| +*/
|
| +#define INTERIOR_MIN_TERMS 7
|
| +#if INTERIOR_MIN_TERMS<1
|
| +# error INTERIOR_MIN_TERMS must be greater than 0.
|
| +#endif
|
| +
|
| +/* ROOT_MAX controls how much data is stored inline in the segment
|
| +** directory.
|
| +*/
|
| +/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
|
| +** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
|
| +** can both see it, but if the caller passed it in, we wouldn't even
|
| +** need a define.
|
| +*/
|
| +#define ROOT_MAX 1024
|
| +#if ROOT_MAX<VARINT_MAX*2
|
| +# error ROOT_MAX must have enough space for a header.
|
| +#endif
|
| +
|
| +/* InteriorBlock stores a linked-list of interior blocks while a lower
|
| +** layer is being constructed.
|
| +*/
|
| +typedef struct InteriorBlock {
|
| + DataBuffer term; /* Leftmost term in block's subtree. */
|
| + DataBuffer data; /* Accumulated data for the block. */
|
| + struct InteriorBlock *next;
|
| +} InteriorBlock;
|
| +
|
| +static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
|
| + const char *pTerm, int nTerm){
|
| + InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
|
| + char c[VARINT_MAX+VARINT_MAX];
|
| + int n;
|
| +
|
| + if( block ){
|
| + memset(block, 0, sizeof(*block));
|
| + dataBufferInit(&block->term, 0);
|
| + dataBufferReplace(&block->term, pTerm, nTerm);
|
| +
|
| + n = putVarint(c, iHeight);
|
| + n += putVarint(c+n, iChildBlock);
|
| + dataBufferInit(&block->data, INTERIOR_MAX);
|
| + dataBufferReplace(&block->data, c, n);
|
| + }
|
| + return block;
|
| +}
|
| +
|
| +#ifndef NDEBUG
|
| +/* Verify that the data is readable as an interior node. */
|
| +static void interiorBlockValidate(InteriorBlock *pBlock){
|
| + const char *pData = pBlock->data.pData;
|
| + int nData = pBlock->data.nData;
|
| + int n, iDummy;
|
| + sqlite_int64 iBlockid;
|
| +
|
| + assert( nData>0 );
|
| + assert( pData!=0 );
|
| + assert( pData+nData>pData );
|
| +
|
| + /* Must lead with height of node as a varint(n), n>0 */
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>0 );
|
| + assert( n<nData );
|
| + pData += n;
|
| + nData -= n;
|
| +
|
| + /* Must contain iBlockid. */
|
| + n = getVarint(pData, &iBlockid);
|
| + assert( n>0 );
|
| + assert( n<=nData );
|
| + pData += n;
|
| + nData -= n;
|
| +
|
| + /* Zero or more terms of positive length */
|
| + if( nData!=0 ){
|
| + /* First term is not delta-encoded. */
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>0 );
|
| + assert( n+iDummy>0);
|
| + assert( n+iDummy<=nData );
|
| + pData += n+iDummy;
|
| + nData -= n+iDummy;
|
| +
|
| + /* Following terms delta-encoded. */
|
| + while( nData!=0 ){
|
| + /* Length of shared prefix. */
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>=0 );
|
| + assert( n<nData );
|
| + pData += n;
|
| + nData -= n;
|
| +
|
| + /* Length and data of distinct suffix. */
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>0 );
|
| + assert( n+iDummy>0);
|
| + assert( n+iDummy<=nData );
|
| + pData += n+iDummy;
|
| + nData -= n+iDummy;
|
| + }
|
| + }
|
| +}
|
| +#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
|
| +#else
|
| +#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
|
| +#endif
|
| +
|
| +typedef struct InteriorWriter {
|
| + int iHeight; /* from 0 at leaves. */
|
| + InteriorBlock *first, *last;
|
| + struct InteriorWriter *parentWriter;
|
| +
|
| + DataBuffer term; /* Last term written to block "last". */
|
| + sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
|
| +#ifndef NDEBUG
|
| + sqlite_int64 iLastChildBlock; /* for consistency checks. */
|
| +#endif
|
| +} InteriorWriter;
|
| +
|
| +/* Initialize an interior node where pTerm[nTerm] marks the leftmost
|
| +** term in the tree. iChildBlock is the leftmost child block at the
|
| +** next level down the tree.
|
| +*/
|
| +static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
|
| + sqlite_int64 iChildBlock,
|
| + InteriorWriter *pWriter){
|
| + InteriorBlock *block;
|
| + assert( iHeight>0 );
|
| + CLEAR(pWriter);
|
| +
|
| + pWriter->iHeight = iHeight;
|
| + pWriter->iOpeningChildBlock = iChildBlock;
|
| +#ifndef NDEBUG
|
| + pWriter->iLastChildBlock = iChildBlock;
|
| +#endif
|
| + block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
|
| + pWriter->last = pWriter->first = block;
|
| + ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
|
| + dataBufferInit(&pWriter->term, 0);
|
| +}
|
| +
|
| +/* Append the child node rooted at iChildBlock to the interior node,
|
| +** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
|
| +*/
|
| +static void interiorWriterAppend(InteriorWriter *pWriter,
|
| + const char *pTerm, int nTerm,
|
| + sqlite_int64 iChildBlock){
|
| + char c[VARINT_MAX+VARINT_MAX];
|
| + int n, nPrefix = 0;
|
| +
|
| + ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
|
| +
|
| + /* The first term written into an interior node is actually
|
| + ** associated with the second child added (the first child was added
|
| + ** in interiorWriterInit, or in the if clause at the bottom of this
|
| + ** function). That term gets encoded straight up, with nPrefix left
|
| + ** at 0.
|
| + */
|
| + if( pWriter->term.nData==0 ){
|
| + n = putVarint(c, nTerm);
|
| + }else{
|
| + while( nPrefix<pWriter->term.nData &&
|
| + pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
|
| + nPrefix++;
|
| + }
|
| +
|
| + n = putVarint(c, nPrefix);
|
| + n += putVarint(c+n, nTerm-nPrefix);
|
| + }
|
| +
|
| +#ifndef NDEBUG
|
| + pWriter->iLastChildBlock++;
|
| +#endif
|
| + assert( pWriter->iLastChildBlock==iChildBlock );
|
| +
|
| + /* Overflow to a new block if the new term makes the current block
|
| + ** too big, and the current block already has enough terms.
|
| + */
|
| + if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
|
| + iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
|
| + pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
|
| + pTerm, nTerm);
|
| + pWriter->last = pWriter->last->next;
|
| + pWriter->iOpeningChildBlock = iChildBlock;
|
| + dataBufferReset(&pWriter->term);
|
| + }else{
|
| + dataBufferAppend2(&pWriter->last->data, c, n,
|
| + pTerm+nPrefix, nTerm-nPrefix);
|
| + dataBufferReplace(&pWriter->term, pTerm, nTerm);
|
| + }
|
| + ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
|
| +}
|
| +
|
| +/* Free the space used by pWriter, including the linked-list of
|
| +** InteriorBlocks, and parentWriter, if present.
|
| +*/
|
| +static int interiorWriterDestroy(InteriorWriter *pWriter){
|
| + InteriorBlock *block = pWriter->first;
|
| +
|
| + while( block!=NULL ){
|
| + InteriorBlock *b = block;
|
| + block = block->next;
|
| + dataBufferDestroy(&b->term);
|
| + dataBufferDestroy(&b->data);
|
| + sqlite3_free(b);
|
| + }
|
| + if( pWriter->parentWriter!=NULL ){
|
| + interiorWriterDestroy(pWriter->parentWriter);
|
| + sqlite3_free(pWriter->parentWriter);
|
| + }
|
| + dataBufferDestroy(&pWriter->term);
|
| + SCRAMBLE(pWriter);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
|
| +** directly, leaving *piEndBlockid unchanged. Otherwise, flush
|
| +** pWriter to %_segments, building a new layer of interior nodes, and
|
| +** recursively ask for their root into.
|
| +*/
|
| +static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter,
|
| + char **ppRootInfo, int *pnRootInfo,
|
| + sqlite_int64 *piEndBlockid){
|
| + InteriorBlock *block = pWriter->first;
|
| + sqlite_int64 iBlockid = 0;
|
| + int rc;
|
| +
|
| + /* If we can fit the segment inline */
|
| + if( block==pWriter->last && block->data.nData<ROOT_MAX ){
|
| + *ppRootInfo = block->data.pData;
|
| + *pnRootInfo = block->data.nData;
|
| + return SQLITE_OK;
|
| + }
|
| +
|
| + /* Flush the first block to %_segments, and create a new level of
|
| + ** interior node.
|
| + */
|
| + ASSERT_VALID_INTERIOR_BLOCK(block);
|
| + rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + *piEndBlockid = iBlockid;
|
| +
|
| + pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
|
| + interiorWriterInit(pWriter->iHeight+1,
|
| + block->term.pData, block->term.nData,
|
| + iBlockid, pWriter->parentWriter);
|
| +
|
| + /* Flush additional blocks and append to the higher interior
|
| + ** node.
|
| + */
|
| + for(block=block->next; block!=NULL; block=block->next){
|
| + ASSERT_VALID_INTERIOR_BLOCK(block);
|
| + rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + *piEndBlockid = iBlockid;
|
| +
|
| + interiorWriterAppend(pWriter->parentWriter,
|
| + block->term.pData, block->term.nData, iBlockid);
|
| + }
|
| +
|
| + /* Parent node gets the chance to be the root. */
|
| + return interiorWriterRootInfo(v, pWriter->parentWriter,
|
| + ppRootInfo, pnRootInfo, piEndBlockid);
|
| +}
|
| +
|
| +/****************************************************************/
|
| +/* InteriorReader is used to read off the data from an interior node
|
| +** (see comment at top of file for the format).
|
| +*/
|
| +typedef struct InteriorReader {
|
| + const char *pData;
|
| + int nData;
|
| +
|
| + DataBuffer term; /* previous term, for decoding term delta. */
|
| +
|
| + sqlite_int64 iBlockid;
|
| +} InteriorReader;
|
| +
|
| +static void interiorReaderDestroy(InteriorReader *pReader){
|
| + dataBufferDestroy(&pReader->term);
|
| + SCRAMBLE(pReader);
|
| +}
|
| +
|
| +/* TODO(shess) The assertions are great, but what if we're in NDEBUG
|
| +** and the blob is empty or otherwise contains suspect data?
|
| +*/
|
| +static void interiorReaderInit(const char *pData, int nData,
|
| + InteriorReader *pReader){
|
| + int n, nTerm;
|
| +
|
| + /* Require at least the leading flag byte */
|
| + assert( nData>0 );
|
| + assert( pData[0]!='\0' );
|
| +
|
| + CLEAR(pReader);
|
| +
|
| + /* Decode the base blockid, and set the cursor to the first term. */
|
| + n = getVarint(pData+1, &pReader->iBlockid);
|
| + assert( 1+n<=nData );
|
| + pReader->pData = pData+1+n;
|
| + pReader->nData = nData-(1+n);
|
| +
|
| + /* A single-child interior node (such as when a leaf node was too
|
| + ** large for the segment directory) won't have any terms.
|
| + ** Otherwise, decode the first term.
|
| + */
|
| + if( pReader->nData==0 ){
|
| + dataBufferInit(&pReader->term, 0);
|
| + }else{
|
| + n = getVarint32(pReader->pData, &nTerm);
|
| + dataBufferInit(&pReader->term, nTerm);
|
| + dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
|
| + assert( n+nTerm<=pReader->nData );
|
| + pReader->pData += n+nTerm;
|
| + pReader->nData -= n+nTerm;
|
| + }
|
| +}
|
| +
|
| +static int interiorReaderAtEnd(InteriorReader *pReader){
|
| + return pReader->term.nData==0;
|
| +}
|
| +
|
| +static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
|
| + return pReader->iBlockid;
|
| +}
|
| +
|
| +static int interiorReaderTermBytes(InteriorReader *pReader){
|
| + assert( !interiorReaderAtEnd(pReader) );
|
| + return pReader->term.nData;
|
| +}
|
| +static const char *interiorReaderTerm(InteriorReader *pReader){
|
| + assert( !interiorReaderAtEnd(pReader) );
|
| + return pReader->term.pData;
|
| +}
|
| +
|
| +/* Step forward to the next term in the node. */
|
| +static void interiorReaderStep(InteriorReader *pReader){
|
| + assert( !interiorReaderAtEnd(pReader) );
|
| +
|
| + /* If the last term has been read, signal eof, else construct the
|
| + ** next term.
|
| + */
|
| + if( pReader->nData==0 ){
|
| + dataBufferReset(&pReader->term);
|
| + }else{
|
| + int n, nPrefix, nSuffix;
|
| +
|
| + n = getVarint32(pReader->pData, &nPrefix);
|
| + n += getVarint32(pReader->pData+n, &nSuffix);
|
| +
|
| + /* Truncate the current term and append suffix data. */
|
| + pReader->term.nData = nPrefix;
|
| + dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
|
| +
|
| + assert( n+nSuffix<=pReader->nData );
|
| + pReader->pData += n+nSuffix;
|
| + pReader->nData -= n+nSuffix;
|
| + }
|
| + pReader->iBlockid++;
|
| +}
|
| +
|
| +/* Compare the current term to pTerm[nTerm], returning strcmp-style
|
| +** results. If isPrefix, equality means equal through nTerm bytes.
|
| +*/
|
| +static int interiorReaderTermCmp(InteriorReader *pReader,
|
| + const char *pTerm, int nTerm, int isPrefix){
|
| + const char *pReaderTerm = interiorReaderTerm(pReader);
|
| + int nReaderTerm = interiorReaderTermBytes(pReader);
|
| + int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
|
| +
|
| + if( n==0 ){
|
| + if( nReaderTerm>0 ) return -1;
|
| + if( nTerm>0 ) return 1;
|
| + return 0;
|
| + }
|
| +
|
| + c = memcmp(pReaderTerm, pTerm, n);
|
| + if( c!=0 ) return c;
|
| + if( isPrefix && n==nTerm ) return 0;
|
| + return nReaderTerm - nTerm;
|
| +}
|
| +
|
| +/****************************************************************/
|
| +/* LeafWriter is used to collect terms and associated doclist data
|
| +** into leaf blocks in %_segments (see top of file for format info).
|
| +** Expected usage is:
|
| +**
|
| +** LeafWriter writer;
|
| +** leafWriterInit(0, 0, &writer);
|
| +** while( sorted_terms_left_to_process ){
|
| +** // data is doclist data for that term.
|
| +** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
|
| +** if( rc!=SQLITE_OK ) goto err;
|
| +** }
|
| +** rc = leafWriterFinalize(v, &writer);
|
| +**err:
|
| +** leafWriterDestroy(&writer);
|
| +** return rc;
|
| +**
|
| +** leafWriterStep() may write a collected leaf out to %_segments.
|
| +** leafWriterFinalize() finishes writing any buffered data and stores
|
| +** a root node in %_segdir. leafWriterDestroy() frees all buffers and
|
| +** InteriorWriters allocated as part of writing this segment.
|
| +**
|
| +** TODO(shess) Document leafWriterStepMerge().
|
| +*/
|
| +
|
| +/* Put terms with data this big in their own block. */
|
| +#define STANDALONE_MIN 1024
|
| +
|
| +/* Keep leaf blocks below this size. */
|
| +#define LEAF_MAX 2048
|
| +
|
| +typedef struct LeafWriter {
|
| + int iLevel;
|
| + int idx;
|
| + sqlite_int64 iStartBlockid; /* needed to create the root info */
|
| + sqlite_int64 iEndBlockid; /* when we're done writing. */
|
| +
|
| + DataBuffer term; /* previous encoded term */
|
| + DataBuffer data; /* encoding buffer */
|
| +
|
| + /* bytes of first term in the current node which distinguishes that
|
| + ** term from the last term of the previous node.
|
| + */
|
| + int nTermDistinct;
|
| +
|
| + InteriorWriter parentWriter; /* if we overflow */
|
| + int has_parent;
|
| +} LeafWriter;
|
| +
|
| +static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
|
| + CLEAR(pWriter);
|
| + pWriter->iLevel = iLevel;
|
| + pWriter->idx = idx;
|
| +
|
| + dataBufferInit(&pWriter->term, 32);
|
| +
|
| + /* Start out with a reasonably sized block, though it can grow. */
|
| + dataBufferInit(&pWriter->data, LEAF_MAX);
|
| +}
|
| +
|
| +#ifndef NDEBUG
|
| +/* Verify that the data is readable as a leaf node. */
|
| +static void leafNodeValidate(const char *pData, int nData){
|
| + int n, iDummy;
|
| +
|
| + if( nData==0 ) return;
|
| + assert( nData>0 );
|
| + assert( pData!=0 );
|
| + assert( pData+nData>pData );
|
| +
|
| + /* Must lead with a varint(0) */
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( iDummy==0 );
|
| + assert( n>0 );
|
| + assert( n<nData );
|
| + pData += n;
|
| + nData -= n;
|
| +
|
| + /* Leading term length and data must fit in buffer. */
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>0 );
|
| + assert( n+iDummy>0 );
|
| + assert( n+iDummy<nData );
|
| + pData += n+iDummy;
|
| + nData -= n+iDummy;
|
| +
|
| + /* Leading term's doclist length and data must fit. */
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>0 );
|
| + assert( n+iDummy>0 );
|
| + assert( n+iDummy<=nData );
|
| + ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
|
| + pData += n+iDummy;
|
| + nData -= n+iDummy;
|
| +
|
| + /* Verify that trailing terms and doclists also are readable. */
|
| + while( nData!=0 ){
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>=0 );
|
| + assert( n<nData );
|
| + pData += n;
|
| + nData -= n;
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>0 );
|
| + assert( n+iDummy>0 );
|
| + assert( n+iDummy<nData );
|
| + pData += n+iDummy;
|
| + nData -= n+iDummy;
|
| +
|
| + n = getVarint32(pData, &iDummy);
|
| + assert( n>0 );
|
| + assert( iDummy>0 );
|
| + assert( n+iDummy>0 );
|
| + assert( n+iDummy<=nData );
|
| + ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
|
| + pData += n+iDummy;
|
| + nData -= n+iDummy;
|
| + }
|
| +}
|
| +#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
|
| +#else
|
| +#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
|
| +#endif
|
| +
|
| +/* Flush the current leaf node to %_segments, and adding the resulting
|
| +** blockid and the starting term to the interior node which will
|
| +** contain it.
|
| +*/
|
| +static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter,
|
| + int iData, int nData){
|
| + sqlite_int64 iBlockid = 0;
|
| + const char *pStartingTerm;
|
| + int nStartingTerm, rc, n;
|
| +
|
| + /* Must have the leading varint(0) flag, plus at least some
|
| + ** valid-looking data.
|
| + */
|
| + assert( nData>2 );
|
| + assert( iData>=0 );
|
| + assert( iData+nData<=pWriter->data.nData );
|
| + ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
|
| +
|
| + rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + assert( iBlockid!=0 );
|
| +
|
| + /* Reconstruct the first term in the leaf for purposes of building
|
| + ** the interior node.
|
| + */
|
| + n = getVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
|
| + pStartingTerm = pWriter->data.pData+iData+1+n;
|
| + assert( pWriter->data.nData>iData+1+n+nStartingTerm );
|
| + assert( pWriter->nTermDistinct>0 );
|
| + assert( pWriter->nTermDistinct<=nStartingTerm );
|
| + nStartingTerm = pWriter->nTermDistinct;
|
| +
|
| + if( pWriter->has_parent ){
|
| + interiorWriterAppend(&pWriter->parentWriter,
|
| + pStartingTerm, nStartingTerm, iBlockid);
|
| + }else{
|
| + interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
|
| + &pWriter->parentWriter);
|
| + pWriter->has_parent = 1;
|
| + }
|
| +
|
| + /* Track the span of this segment's leaf nodes. */
|
| + if( pWriter->iEndBlockid==0 ){
|
| + pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
|
| + }else{
|
| + pWriter->iEndBlockid++;
|
| + assert( iBlockid==pWriter->iEndBlockid );
|
| + }
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){
|
| + int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* Re-initialize the output buffer. */
|
| + dataBufferReset(&pWriter->data);
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Fetch the root info for the segment. If the entire leaf fits
|
| +** within ROOT_MAX, then it will be returned directly, otherwise it
|
| +** will be flushed and the root info will be returned from the
|
| +** interior node. *piEndBlockid is set to the blockid of the last
|
| +** interior or leaf node written to disk (0 if none are written at
|
| +** all).
|
| +*/
|
| +static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter,
|
| + char **ppRootInfo, int *pnRootInfo,
|
| + sqlite_int64 *piEndBlockid){
|
| + /* we can fit the segment entirely inline */
|
| + if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
|
| + *ppRootInfo = pWriter->data.pData;
|
| + *pnRootInfo = pWriter->data.nData;
|
| + *piEndBlockid = 0;
|
| + return SQLITE_OK;
|
| + }
|
| +
|
| + /* Flush remaining leaf data. */
|
| + if( pWriter->data.nData>0 ){
|
| + int rc = leafWriterFlush(v, pWriter);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| +
|
| + /* We must have flushed a leaf at some point. */
|
| + assert( pWriter->has_parent );
|
| +
|
| + /* Tenatively set the end leaf blockid as the end blockid. If the
|
| + ** interior node can be returned inline, this will be the final
|
| + ** blockid, otherwise it will be overwritten by
|
| + ** interiorWriterRootInfo().
|
| + */
|
| + *piEndBlockid = pWriter->iEndBlockid;
|
| +
|
| + return interiorWriterRootInfo(v, &pWriter->parentWriter,
|
| + ppRootInfo, pnRootInfo, piEndBlockid);
|
| +}
|
| +
|
| +/* Collect the rootInfo data and store it into the segment directory.
|
| +** This has the effect of flushing the segment's leaf data to
|
| +** %_segments, and also flushing any interior nodes to %_segments.
|
| +*/
|
| +static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){
|
| + sqlite_int64 iEndBlockid;
|
| + char *pRootInfo;
|
| + int rc, nRootInfo;
|
| +
|
| + rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* Don't bother storing an entirely empty segment. */
|
| + if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
|
| +
|
| + return segdir_set(v, pWriter->iLevel, pWriter->idx,
|
| + pWriter->iStartBlockid, pWriter->iEndBlockid,
|
| + iEndBlockid, pRootInfo, nRootInfo);
|
| +}
|
| +
|
| +static void leafWriterDestroy(LeafWriter *pWriter){
|
| + if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
|
| + dataBufferDestroy(&pWriter->term);
|
| + dataBufferDestroy(&pWriter->data);
|
| +}
|
| +
|
| +/* Encode a term into the leafWriter, delta-encoding as appropriate.
|
| +** Returns the length of the new term which distinguishes it from the
|
| +** previous term, which can be used to set nTermDistinct when a node
|
| +** boundary is crossed.
|
| +*/
|
| +static int leafWriterEncodeTerm(LeafWriter *pWriter,
|
| + const char *pTerm, int nTerm){
|
| + char c[VARINT_MAX+VARINT_MAX];
|
| + int n, nPrefix = 0;
|
| +
|
| + assert( nTerm>0 );
|
| + while( nPrefix<pWriter->term.nData &&
|
| + pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
|
| + nPrefix++;
|
| + /* Failing this implies that the terms weren't in order. */
|
| + assert( nPrefix<nTerm );
|
| + }
|
| +
|
| + if( pWriter->data.nData==0 ){
|
| + /* Encode the node header and leading term as:
|
| + ** varint(0)
|
| + ** varint(nTerm)
|
| + ** char pTerm[nTerm]
|
| + */
|
| + n = putVarint(c, '\0');
|
| + n += putVarint(c+n, nTerm);
|
| + dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
|
| + }else{
|
| + /* Delta-encode the term as:
|
| + ** varint(nPrefix)
|
| + ** varint(nSuffix)
|
| + ** char pTermSuffix[nSuffix]
|
| + */
|
| + n = putVarint(c, nPrefix);
|
| + n += putVarint(c+n, nTerm-nPrefix);
|
| + dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
|
| + }
|
| + dataBufferReplace(&pWriter->term, pTerm, nTerm);
|
| +
|
| + return nPrefix+1;
|
| +}
|
| +
|
| +/* Used to avoid a memmove when a large amount of doclist data is in
|
| +** the buffer. This constructs a node and term header before
|
| +** iDoclistData and flushes the resulting complete node using
|
| +** leafWriterInternalFlush().
|
| +*/
|
| +static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter,
|
| + const char *pTerm, int nTerm,
|
| + int iDoclistData){
|
| + char c[VARINT_MAX+VARINT_MAX];
|
| + int iData, n = putVarint(c, 0);
|
| + n += putVarint(c+n, nTerm);
|
| +
|
| + /* There should always be room for the header. Even if pTerm shared
|
| + ** a substantial prefix with the previous term, the entire prefix
|
| + ** could be constructed from earlier data in the doclist, so there
|
| + ** should be room.
|
| + */
|
| + assert( iDoclistData>=n+nTerm );
|
| +
|
| + iData = iDoclistData-(n+nTerm);
|
| + memcpy(pWriter->data.pData+iData, c, n);
|
| + memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
|
| +
|
| + return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
|
| +}
|
| +
|
| +/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
|
| +** %_segments.
|
| +*/
|
| +static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
|
| + const char *pTerm, int nTerm,
|
| + DLReader *pReaders, int nReaders){
|
| + char c[VARINT_MAX+VARINT_MAX];
|
| + int iTermData = pWriter->data.nData, iDoclistData;
|
| + int i, nData, n, nActualData, nActual, rc, nTermDistinct;
|
| +
|
| + ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
|
| + nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
|
| +
|
| + /* Remember nTermDistinct if opening a new node. */
|
| + if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
|
| +
|
| + iDoclistData = pWriter->data.nData;
|
| +
|
| + /* Estimate the length of the merged doclist so we can leave space
|
| + ** to encode it.
|
| + */
|
| + for(i=0, nData=0; i<nReaders; i++){
|
| + nData += dlrAllDataBytes(&pReaders[i]);
|
| + }
|
| + n = putVarint(c, nData);
|
| + dataBufferAppend(&pWriter->data, c, n);
|
| +
|
| + docListMerge(&pWriter->data, pReaders, nReaders);
|
| + ASSERT_VALID_DOCLIST(DL_DEFAULT,
|
| + pWriter->data.pData+iDoclistData+n,
|
| + pWriter->data.nData-iDoclistData-n, NULL);
|
| +
|
| + /* The actual amount of doclist data at this point could be smaller
|
| + ** than the length we encoded. Additionally, the space required to
|
| + ** encode this length could be smaller. For small doclists, this is
|
| + ** not a big deal, we can just use memmove() to adjust things.
|
| + */
|
| + nActualData = pWriter->data.nData-(iDoclistData+n);
|
| + nActual = putVarint(c, nActualData);
|
| + assert( nActualData<=nData );
|
| + assert( nActual<=n );
|
| +
|
| + /* If the new doclist is big enough for force a standalone leaf
|
| + ** node, we can immediately flush it inline without doing the
|
| + ** memmove().
|
| + */
|
| + /* TODO(shess) This test matches leafWriterStep(), which does this
|
| + ** test before it knows the cost to varint-encode the term and
|
| + ** doclist lengths. At some point, change to
|
| + ** pWriter->data.nData-iTermData>STANDALONE_MIN.
|
| + */
|
| + if( nTerm+nActualData>STANDALONE_MIN ){
|
| + /* Push leaf node from before this term. */
|
| + if( iTermData>0 ){
|
| + rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + pWriter->nTermDistinct = nTermDistinct;
|
| + }
|
| +
|
| + /* Fix the encoded doclist length. */
|
| + iDoclistData += n - nActual;
|
| + memcpy(pWriter->data.pData+iDoclistData, c, nActual);
|
| +
|
| + /* Push the standalone leaf node. */
|
| + rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* Leave the node empty. */
|
| + dataBufferReset(&pWriter->data);
|
| +
|
| + return rc;
|
| + }
|
| +
|
| + /* At this point, we know that the doclist was small, so do the
|
| + ** memmove if indicated.
|
| + */
|
| + if( nActual<n ){
|
| + memmove(pWriter->data.pData+iDoclistData+nActual,
|
| + pWriter->data.pData+iDoclistData+n,
|
| + pWriter->data.nData-(iDoclistData+n));
|
| + pWriter->data.nData -= n-nActual;
|
| + }
|
| +
|
| + /* Replace written length with actual length. */
|
| + memcpy(pWriter->data.pData+iDoclistData, c, nActual);
|
| +
|
| + /* If the node is too large, break things up. */
|
| + /* TODO(shess) This test matches leafWriterStep(), which does this
|
| + ** test before it knows the cost to varint-encode the term and
|
| + ** doclist lengths. At some point, change to
|
| + ** pWriter->data.nData>LEAF_MAX.
|
| + */
|
| + if( iTermData+nTerm+nActualData>LEAF_MAX ){
|
| + /* Flush out the leading data as a node */
|
| + rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + pWriter->nTermDistinct = nTermDistinct;
|
| +
|
| + /* Rebuild header using the current term */
|
| + n = putVarint(pWriter->data.pData, 0);
|
| + n += putVarint(pWriter->data.pData+n, nTerm);
|
| + memcpy(pWriter->data.pData+n, pTerm, nTerm);
|
| + n += nTerm;
|
| +
|
| + /* There should always be room, because the previous encoding
|
| + ** included all data necessary to construct the term.
|
| + */
|
| + assert( n<iDoclistData );
|
| + /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
|
| + ** following memcpy() is safe (as opposed to needing a memmove).
|
| + */
|
| + assert( 2*STANDALONE_MIN<=LEAF_MAX );
|
| + assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
|
| + memcpy(pWriter->data.pData+n,
|
| + pWriter->data.pData+iDoclistData,
|
| + pWriter->data.nData-iDoclistData);
|
| + pWriter->data.nData -= iDoclistData-n;
|
| + }
|
| + ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
|
| +** %_segments.
|
| +*/
|
| +/* TODO(shess) Revise writeZeroSegment() so that doclists are
|
| +** constructed directly in pWriter->data.
|
| +*/
|
| +static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
|
| + const char *pTerm, int nTerm,
|
| + const char *pData, int nData){
|
| + int rc;
|
| + DLReader reader;
|
| +
|
| + dlrInit(&reader, DL_DEFAULT, pData, nData);
|
| + rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
|
| + dlrDestroy(&reader);
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/****************************************************************/
|
| +/* LeafReader is used to iterate over an individual leaf node. */
|
| +typedef struct LeafReader {
|
| + DataBuffer term; /* copy of current term. */
|
| +
|
| + const char *pData; /* data for current term. */
|
| + int nData;
|
| +} LeafReader;
|
| +
|
| +static void leafReaderDestroy(LeafReader *pReader){
|
| + dataBufferDestroy(&pReader->term);
|
| + SCRAMBLE(pReader);
|
| +}
|
| +
|
| +static int leafReaderAtEnd(LeafReader *pReader){
|
| + return pReader->nData<=0;
|
| +}
|
| +
|
| +/* Access the current term. */
|
| +static int leafReaderTermBytes(LeafReader *pReader){
|
| + return pReader->term.nData;
|
| +}
|
| +static const char *leafReaderTerm(LeafReader *pReader){
|
| + assert( pReader->term.nData>0 );
|
| + return pReader->term.pData;
|
| +}
|
| +
|
| +/* Access the doclist data for the current term. */
|
| +static int leafReaderDataBytes(LeafReader *pReader){
|
| + int nData;
|
| + assert( pReader->term.nData>0 );
|
| + getVarint32(pReader->pData, &nData);
|
| + return nData;
|
| +}
|
| +static const char *leafReaderData(LeafReader *pReader){
|
| + int n, nData;
|
| + assert( pReader->term.nData>0 );
|
| + n = getVarint32(pReader->pData, &nData);
|
| + return pReader->pData+n;
|
| +}
|
| +
|
| +static void leafReaderInit(const char *pData, int nData,
|
| + LeafReader *pReader){
|
| + int nTerm, n;
|
| +
|
| + assert( nData>0 );
|
| + assert( pData[0]=='\0' );
|
| +
|
| + CLEAR(pReader);
|
| +
|
| + /* Read the first term, skipping the header byte. */
|
| + n = getVarint32(pData+1, &nTerm);
|
| + dataBufferInit(&pReader->term, nTerm);
|
| + dataBufferReplace(&pReader->term, pData+1+n, nTerm);
|
| +
|
| + /* Position after the first term. */
|
| + assert( 1+n+nTerm<nData );
|
| + pReader->pData = pData+1+n+nTerm;
|
| + pReader->nData = nData-1-n-nTerm;
|
| +}
|
| +
|
| +/* Step the reader forward to the next term. */
|
| +static void leafReaderStep(LeafReader *pReader){
|
| + int n, nData, nPrefix, nSuffix;
|
| + assert( !leafReaderAtEnd(pReader) );
|
| +
|
| + /* Skip previous entry's data block. */
|
| + n = getVarint32(pReader->pData, &nData);
|
| + assert( n+nData<=pReader->nData );
|
| + pReader->pData += n+nData;
|
| + pReader->nData -= n+nData;
|
| +
|
| + if( !leafReaderAtEnd(pReader) ){
|
| + /* Construct the new term using a prefix from the old term plus a
|
| + ** suffix from the leaf data.
|
| + */
|
| + n = getVarint32(pReader->pData, &nPrefix);
|
| + n += getVarint32(pReader->pData+n, &nSuffix);
|
| + assert( n+nSuffix<pReader->nData );
|
| + pReader->term.nData = nPrefix;
|
| + dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
|
| +
|
| + pReader->pData += n+nSuffix;
|
| + pReader->nData -= n+nSuffix;
|
| + }
|
| +}
|
| +
|
| +/* strcmp-style comparison of pReader's current term against pTerm.
|
| +** If isPrefix, equality means equal through nTerm bytes.
|
| +*/
|
| +static int leafReaderTermCmp(LeafReader *pReader,
|
| + const char *pTerm, int nTerm, int isPrefix){
|
| + int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
|
| + if( n==0 ){
|
| + if( pReader->term.nData>0 ) return -1;
|
| + if(nTerm>0 ) return 1;
|
| + return 0;
|
| + }
|
| +
|
| + c = memcmp(pReader->term.pData, pTerm, n);
|
| + if( c!=0 ) return c;
|
| + if( isPrefix && n==nTerm ) return 0;
|
| + return pReader->term.nData - nTerm;
|
| +}
|
| +
|
| +
|
| +/****************************************************************/
|
| +/* LeavesReader wraps LeafReader to allow iterating over the entire
|
| +** leaf layer of the tree.
|
| +*/
|
| +typedef struct LeavesReader {
|
| + int idx; /* Index within the segment. */
|
| +
|
| + sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */
|
| + int eof; /* we've seen SQLITE_DONE from pStmt. */
|
| +
|
| + LeafReader leafReader; /* reader for the current leaf. */
|
| + DataBuffer rootData; /* root data for inline. */
|
| +} LeavesReader;
|
| +
|
| +/* Access the current term. */
|
| +static int leavesReaderTermBytes(LeavesReader *pReader){
|
| + assert( !pReader->eof );
|
| + return leafReaderTermBytes(&pReader->leafReader);
|
| +}
|
| +static const char *leavesReaderTerm(LeavesReader *pReader){
|
| + assert( !pReader->eof );
|
| + return leafReaderTerm(&pReader->leafReader);
|
| +}
|
| +
|
| +/* Access the doclist data for the current term. */
|
| +static int leavesReaderDataBytes(LeavesReader *pReader){
|
| + assert( !pReader->eof );
|
| + return leafReaderDataBytes(&pReader->leafReader);
|
| +}
|
| +static const char *leavesReaderData(LeavesReader *pReader){
|
| + assert( !pReader->eof );
|
| + return leafReaderData(&pReader->leafReader);
|
| +}
|
| +
|
| +static int leavesReaderAtEnd(LeavesReader *pReader){
|
| + return pReader->eof;
|
| +}
|
| +
|
| +/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
|
| +** leaving the statement handle open, which locks the table.
|
| +*/
|
| +/* TODO(shess) This "solution" is not satisfactory. Really, there
|
| +** should be check-in function for all statement handles which
|
| +** arranges to call sqlite3_reset(). This most likely will require
|
| +** modification to control flow all over the place, though, so for now
|
| +** just punt.
|
| +**
|
| +** Note the current system assumes that segment merges will run to
|
| +** completion, which is why this particular probably hasn't arisen in
|
| +** this case. Probably a brittle assumption.
|
| +*/
|
| +static int leavesReaderReset(LeavesReader *pReader){
|
| + return sqlite3_reset(pReader->pStmt);
|
| +}
|
| +
|
| +static void leavesReaderDestroy(LeavesReader *pReader){
|
| + /* If idx is -1, that means we're using a non-cached statement
|
| + ** handle in the optimize() case, so we need to release it.
|
| + */
|
| + if( pReader->pStmt!=NULL && pReader->idx==-1 ){
|
| + sqlite3_finalize(pReader->pStmt);
|
| + }
|
| + leafReaderDestroy(&pReader->leafReader);
|
| + dataBufferDestroy(&pReader->rootData);
|
| + SCRAMBLE(pReader);
|
| +}
|
| +
|
| +/* Initialize pReader with the given root data (if iStartBlockid==0
|
| +** the leaf data was entirely contained in the root), or from the
|
| +** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
|
| +*/
|
| +static int leavesReaderInit(fulltext_vtab *v,
|
| + int idx,
|
| + sqlite_int64 iStartBlockid,
|
| + sqlite_int64 iEndBlockid,
|
| + const char *pRootData, int nRootData,
|
| + LeavesReader *pReader){
|
| + CLEAR(pReader);
|
| + pReader->idx = idx;
|
| +
|
| + dataBufferInit(&pReader->rootData, 0);
|
| + if( iStartBlockid==0 ){
|
| + /* Entire leaf level fit in root data. */
|
| + dataBufferReplace(&pReader->rootData, pRootData, nRootData);
|
| + leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
|
| + &pReader->leafReader);
|
| + }else{
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_leaf_statement(v, idx, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 1, iStartBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 2, iEndBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_DONE ){
|
| + pReader->eof = 1;
|
| + return SQLITE_OK;
|
| + }
|
| + if( rc!=SQLITE_ROW ) return rc;
|
| +
|
| + pReader->pStmt = s;
|
| + leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
|
| + sqlite3_column_bytes(pReader->pStmt, 0),
|
| + &pReader->leafReader);
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Step the current leaf forward to the next term. If we reach the
|
| +** end of the current leaf, step forward to the next leaf block.
|
| +*/
|
| +static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
|
| + assert( !leavesReaderAtEnd(pReader) );
|
| + leafReaderStep(&pReader->leafReader);
|
| +
|
| + if( leafReaderAtEnd(&pReader->leafReader) ){
|
| + int rc;
|
| + if( pReader->rootData.pData ){
|
| + pReader->eof = 1;
|
| + return SQLITE_OK;
|
| + }
|
| + rc = sqlite3_step(pReader->pStmt);
|
| + if( rc!=SQLITE_ROW ){
|
| + pReader->eof = 1;
|
| + return rc==SQLITE_DONE ? SQLITE_OK : rc;
|
| + }
|
| + leafReaderDestroy(&pReader->leafReader);
|
| + leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
|
| + sqlite3_column_bytes(pReader->pStmt, 0),
|
| + &pReader->leafReader);
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Order LeavesReaders by their term, ignoring idx. Readers at eof
|
| +** always sort to the end.
|
| +*/
|
| +static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){
|
| + if( leavesReaderAtEnd(lr1) ){
|
| + if( leavesReaderAtEnd(lr2) ) return 0;
|
| + return 1;
|
| + }
|
| + if( leavesReaderAtEnd(lr2) ) return -1;
|
| +
|
| + return leafReaderTermCmp(&lr1->leafReader,
|
| + leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
|
| + 0);
|
| +}
|
| +
|
| +/* Similar to leavesReaderTermCmp(), with additional ordering by idx
|
| +** so that older segments sort before newer segments.
|
| +*/
|
| +static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){
|
| + int c = leavesReaderTermCmp(lr1, lr2);
|
| + if( c!=0 ) return c;
|
| + return lr1->idx-lr2->idx;
|
| +}
|
| +
|
| +/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
|
| +** sorted position.
|
| +*/
|
| +static void leavesReaderReorder(LeavesReader *pLr, int nLr){
|
| + while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
|
| + LeavesReader tmp = pLr[0];
|
| + pLr[0] = pLr[1];
|
| + pLr[1] = tmp;
|
| + nLr--;
|
| + pLr++;
|
| + }
|
| +}
|
| +
|
| +/* Initializes pReaders with the segments from level iLevel, returning
|
| +** the number of segments in *piReaders. Leaves pReaders in sorted
|
| +** order.
|
| +*/
|
| +static int leavesReadersInit(fulltext_vtab *v, int iLevel,
|
| + LeavesReader *pReaders, int *piReaders){
|
| + sqlite3_stmt *s;
|
| + int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int(s, 1, iLevel);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + i = 0;
|
| + while( (rc = sqlite3_step(s))==SQLITE_ROW ){
|
| + sqlite_int64 iStart = sqlite3_column_int64(s, 0);
|
| + sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
|
| + const char *pRootData = sqlite3_column_blob(s, 2);
|
| + int nRootData = sqlite3_column_bytes(s, 2);
|
| +
|
| + assert( i<MERGE_COUNT );
|
| + rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
|
| + &pReaders[i]);
|
| + if( rc!=SQLITE_OK ) break;
|
| +
|
| + i++;
|
| + }
|
| + if( rc!=SQLITE_DONE ){
|
| + while( i-->0 ){
|
| + leavesReaderDestroy(&pReaders[i]);
|
| + }
|
| + return rc;
|
| + }
|
| +
|
| + *piReaders = i;
|
| +
|
| + /* Leave our results sorted by term, then age. */
|
| + while( i-- ){
|
| + leavesReaderReorder(pReaders+i, *piReaders-i);
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Merge doclists from pReaders[nReaders] into a single doclist, which
|
| +** is written to pWriter. Assumes pReaders is ordered oldest to
|
| +** newest.
|
| +*/
|
| +/* TODO(shess) Consider putting this inline in segmentMerge(). */
|
| +static int leavesReadersMerge(fulltext_vtab *v,
|
| + LeavesReader *pReaders, int nReaders,
|
| + LeafWriter *pWriter){
|
| + DLReader dlReaders[MERGE_COUNT];
|
| + const char *pTerm = leavesReaderTerm(pReaders);
|
| + int i, nTerm = leavesReaderTermBytes(pReaders);
|
| +
|
| + assert( nReaders<=MERGE_COUNT );
|
| +
|
| + for(i=0; i<nReaders; i++){
|
| + dlrInit(&dlReaders[i], DL_DEFAULT,
|
| + leavesReaderData(pReaders+i),
|
| + leavesReaderDataBytes(pReaders+i));
|
| + }
|
| +
|
| + return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
|
| +}
|
| +
|
| +/* Forward ref due to mutual recursion with segdirNextIndex(). */
|
| +static int segmentMerge(fulltext_vtab *v, int iLevel);
|
| +
|
| +/* Put the next available index at iLevel into *pidx. If iLevel
|
| +** already has MERGE_COUNT segments, they are merged to a higher
|
| +** level to make room.
|
| +*/
|
| +static int segdirNextIndex(fulltext_vtab *v, int iLevel, int *pidx){
|
| + int rc = segdir_max_index(v, iLevel, pidx);
|
| + if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
|
| + *pidx = 0;
|
| + }else if( rc==SQLITE_ROW ){
|
| + if( *pidx==(MERGE_COUNT-1) ){
|
| + rc = segmentMerge(v, iLevel);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + *pidx = 0;
|
| + }else{
|
| + (*pidx)++;
|
| + }
|
| + }else{
|
| + return rc;
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Merge MERGE_COUNT segments at iLevel into a new segment at
|
| +** iLevel+1. If iLevel+1 is already full of segments, those will be
|
| +** merged to make room.
|
| +*/
|
| +static int segmentMerge(fulltext_vtab *v, int iLevel){
|
| + LeafWriter writer;
|
| + LeavesReader lrs[MERGE_COUNT];
|
| + int i, rc, idx = 0;
|
| +
|
| + /* Determine the next available segment index at the next level,
|
| + ** merging as necessary.
|
| + */
|
| + rc = segdirNextIndex(v, iLevel+1, &idx);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* TODO(shess) This assumes that we'll always see exactly
|
| + ** MERGE_COUNT segments to merge at a given level. That will be
|
| + ** broken if we allow the developer to request preemptive or
|
| + ** deferred merging.
|
| + */
|
| + memset(&lrs, '\0', sizeof(lrs));
|
| + rc = leavesReadersInit(v, iLevel, lrs, &i);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + assert( i==MERGE_COUNT );
|
| +
|
| + leafWriterInit(iLevel+1, idx, &writer);
|
| +
|
| + /* Since leavesReaderReorder() pushes readers at eof to the end,
|
| + ** when the first reader is empty, all will be empty.
|
| + */
|
| + while( !leavesReaderAtEnd(lrs) ){
|
| + /* Figure out how many readers share their next term. */
|
| + for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
|
| + if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
|
| + }
|
| +
|
| + rc = leavesReadersMerge(v, lrs, i, &writer);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| +
|
| + /* Step forward those that were merged. */
|
| + while( i-->0 ){
|
| + rc = leavesReaderStep(v, lrs+i);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| +
|
| + /* Reorder by term, then by age. */
|
| + leavesReaderReorder(lrs+i, MERGE_COUNT-i);
|
| + }
|
| + }
|
| +
|
| + for(i=0; i<MERGE_COUNT; i++){
|
| + leavesReaderDestroy(&lrs[i]);
|
| + }
|
| +
|
| + rc = leafWriterFinalize(v, &writer);
|
| + leafWriterDestroy(&writer);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* Delete the merged segment data. */
|
| + return segdir_delete(v, iLevel);
|
| +
|
| + err:
|
| + for(i=0; i<MERGE_COUNT; i++){
|
| + leavesReaderDestroy(&lrs[i]);
|
| + }
|
| + leafWriterDestroy(&writer);
|
| + return rc;
|
| +}
|
| +
|
| +/* Accumulate the union of *acc and *pData into *acc. */
|
| +static void docListAccumulateUnion(DataBuffer *acc,
|
| + const char *pData, int nData) {
|
| + DataBuffer tmp = *acc;
|
| + dataBufferInit(acc, tmp.nData+nData);
|
| + docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
|
| + dataBufferDestroy(&tmp);
|
| +}
|
| +
|
| +/* TODO(shess) It might be interesting to explore different merge
|
| +** strategies, here. For instance, since this is a sorted merge, we
|
| +** could easily merge many doclists in parallel. With some
|
| +** comprehension of the storage format, we could merge all of the
|
| +** doclists within a leaf node directly from the leaf node's storage.
|
| +** It may be worthwhile to merge smaller doclists before larger
|
| +** doclists, since they can be traversed more quickly - but the
|
| +** results may have less overlap, making them more expensive in a
|
| +** different way.
|
| +*/
|
| +
|
| +/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
|
| +** *out (any doclists with duplicate docids overwrite those in *out).
|
| +** Internal function for loadSegmentLeaf().
|
| +*/
|
| +static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + DataBuffer *out){
|
| + /* doclist data is accumulated into pBuffers similar to how one does
|
| + ** increment in binary arithmetic. If index 0 is empty, the data is
|
| + ** stored there. If there is data there, it is merged and the
|
| + ** results carried into position 1, with further merge-and-carry
|
| + ** until an empty position is found.
|
| + */
|
| + DataBuffer *pBuffers = NULL;
|
| + int nBuffers = 0, nMaxBuffers = 0, rc;
|
| +
|
| + assert( nTerm>0 );
|
| +
|
| + for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
|
| + rc=leavesReaderStep(v, pReader)){
|
| + /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
|
| + ** already taken to compare the terms of two LeavesReaders. Think
|
| + ** on a better name. [Meanwhile, break encapsulation rather than
|
| + ** use a confusing name.]
|
| + */
|
| + int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
|
| + if( c>0 ) break; /* Past any possible matches. */
|
| + if( c==0 ){
|
| + const char *pData = leavesReaderData(pReader);
|
| + int iBuffer, nData = leavesReaderDataBytes(pReader);
|
| +
|
| + /* Find the first empty buffer. */
|
| + for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
|
| + if( 0==pBuffers[iBuffer].nData ) break;
|
| + }
|
| +
|
| + /* Out of buffers, add an empty one. */
|
| + if( iBuffer==nBuffers ){
|
| + if( nBuffers==nMaxBuffers ){
|
| + DataBuffer *p;
|
| + nMaxBuffers += 20;
|
| +
|
| + /* Manual realloc so we can handle NULL appropriately. */
|
| + p = sqlite3_malloc(nMaxBuffers*sizeof(*pBuffers));
|
| + if( p==NULL ){
|
| + rc = SQLITE_NOMEM;
|
| + break;
|
| + }
|
| +
|
| + if( nBuffers>0 ){
|
| + assert(pBuffers!=NULL);
|
| + memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers));
|
| + sqlite3_free(pBuffers);
|
| + }
|
| + pBuffers = p;
|
| + }
|
| + dataBufferInit(&(pBuffers[nBuffers]), 0);
|
| + nBuffers++;
|
| + }
|
| +
|
| + /* At this point, must have an empty at iBuffer. */
|
| + assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
|
| +
|
| + /* If empty was first buffer, no need for merge logic. */
|
| + if( iBuffer==0 ){
|
| + dataBufferReplace(&(pBuffers[0]), pData, nData);
|
| + }else{
|
| + /* pAcc is the empty buffer the merged data will end up in. */
|
| + DataBuffer *pAcc = &(pBuffers[iBuffer]);
|
| + DataBuffer *p = &(pBuffers[0]);
|
| +
|
| + /* Handle position 0 specially to avoid need to prime pAcc
|
| + ** with pData/nData.
|
| + */
|
| + dataBufferSwap(p, pAcc);
|
| + docListAccumulateUnion(pAcc, pData, nData);
|
| +
|
| + /* Accumulate remaining doclists into pAcc. */
|
| + for(++p; p<pAcc; ++p){
|
| + docListAccumulateUnion(pAcc, p->pData, p->nData);
|
| +
|
| + /* dataBufferReset() could allow a large doclist to blow up
|
| + ** our memory requirements.
|
| + */
|
| + if( p->nCapacity<1024 ){
|
| + dataBufferReset(p);
|
| + }else{
|
| + dataBufferDestroy(p);
|
| + dataBufferInit(p, 0);
|
| + }
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* Union all the doclists together into *out. */
|
| + /* TODO(shess) What if *out is big? Sigh. */
|
| + if( rc==SQLITE_OK && nBuffers>0 ){
|
| + int iBuffer;
|
| + for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
|
| + if( pBuffers[iBuffer].nData>0 ){
|
| + if( out->nData==0 ){
|
| + dataBufferSwap(out, &(pBuffers[iBuffer]));
|
| + }else{
|
| + docListAccumulateUnion(out, pBuffers[iBuffer].pData,
|
| + pBuffers[iBuffer].nData);
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + while( nBuffers-- ){
|
| + dataBufferDestroy(&(pBuffers[nBuffers]));
|
| + }
|
| + if( pBuffers!=NULL ) sqlite3_free(pBuffers);
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/* Call loadSegmentLeavesInt() with pData/nData as input. */
|
| +static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + DataBuffer *out){
|
| + LeavesReader reader;
|
| + int rc;
|
| +
|
| + assert( nData>1 );
|
| + assert( *pData=='\0' );
|
| + rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
|
| + leavesReaderReset(&reader);
|
| + leavesReaderDestroy(&reader);
|
| + return rc;
|
| +}
|
| +
|
| +/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
|
| +** iEndLeaf (inclusive) as input, and merge the resulting doclist into
|
| +** out.
|
| +*/
|
| +static int loadSegmentLeaves(fulltext_vtab *v,
|
| + sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + DataBuffer *out){
|
| + int rc;
|
| + LeavesReader reader;
|
| +
|
| + assert( iStartLeaf<=iEndLeaf );
|
| + rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
|
| + leavesReaderReset(&reader);
|
| + leavesReaderDestroy(&reader);
|
| + return rc;
|
| +}
|
| +
|
| +/* Taking pData/nData as an interior node, find the sequence of child
|
| +** nodes which could include pTerm/nTerm/isPrefix. Note that the
|
| +** interior node terms logically come between the blocks, so there is
|
| +** one more blockid than there are terms (that block contains terms >=
|
| +** the last interior-node term).
|
| +*/
|
| +/* TODO(shess) The calling code may already know that the end child is
|
| +** not worth calculating, because the end may be in a later sibling
|
| +** node. Consider whether breaking symmetry is worthwhile. I suspect
|
| +** it is not worthwhile.
|
| +*/
|
| +static void getChildrenContaining(const char *pData, int nData,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + sqlite_int64 *piStartChild,
|
| + sqlite_int64 *piEndChild){
|
| + InteriorReader reader;
|
| +
|
| + assert( nData>1 );
|
| + assert( *pData!='\0' );
|
| + interiorReaderInit(pData, nData, &reader);
|
| +
|
| + /* Scan for the first child which could contain pTerm/nTerm. */
|
| + while( !interiorReaderAtEnd(&reader) ){
|
| + if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
|
| + interiorReaderStep(&reader);
|
| + }
|
| + *piStartChild = interiorReaderCurrentBlockid(&reader);
|
| +
|
| + /* Keep scanning to find a term greater than our term, using prefix
|
| + ** comparison if indicated. If isPrefix is false, this will be the
|
| + ** same blockid as the starting block.
|
| + */
|
| + while( !interiorReaderAtEnd(&reader) ){
|
| + if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
|
| + interiorReaderStep(&reader);
|
| + }
|
| + *piEndChild = interiorReaderCurrentBlockid(&reader);
|
| +
|
| + interiorReaderDestroy(&reader);
|
| +
|
| + /* Children must ascend, and if !prefix, both must be the same. */
|
| + assert( *piEndChild>=*piStartChild );
|
| + assert( isPrefix || *piStartChild==*piEndChild );
|
| +}
|
| +
|
| +/* Read block at iBlockid and pass it with other params to
|
| +** getChildrenContaining().
|
| +*/
|
| +static int loadAndGetChildrenContaining(
|
| + fulltext_vtab *v,
|
| + sqlite_int64 iBlockid,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + sqlite_int64 *piStartChild, sqlite_int64 *piEndChild
|
| +){
|
| + sqlite3_stmt *s = NULL;
|
| + int rc;
|
| +
|
| + assert( iBlockid!=0 );
|
| + assert( pTerm!=NULL );
|
| + assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
|
| + assert( piStartChild!=NULL );
|
| + assert( piEndChild!=NULL );
|
| +
|
| + rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_bind_int64(s, 1, iBlockid);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_DONE ) return SQLITE_ERROR;
|
| + if( rc!=SQLITE_ROW ) return rc;
|
| +
|
| + getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
|
| + pTerm, nTerm, isPrefix, piStartChild, piEndChild);
|
| +
|
| + /* We expect only one row. We must execute another sqlite3_step()
|
| + * to complete the iteration; otherwise the table will remain
|
| + * locked. */
|
| + rc = sqlite3_step(s);
|
| + if( rc==SQLITE_ROW ) return SQLITE_ERROR;
|
| + if( rc!=SQLITE_DONE ) return rc;
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Traverse the tree represented by pData[nData] looking for
|
| +** pTerm[nTerm], placing its doclist into *out. This is internal to
|
| +** loadSegment() to make error-handling cleaner.
|
| +*/
|
| +static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
|
| + sqlite_int64 iLeavesEnd,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + DataBuffer *out){
|
| + /* Special case where root is a leaf. */
|
| + if( *pData=='\0' ){
|
| + return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
|
| + }else{
|
| + int rc;
|
| + sqlite_int64 iStartChild, iEndChild;
|
| +
|
| + /* Process pData as an interior node, then loop down the tree
|
| + ** until we find the set of leaf nodes to scan for the term.
|
| + */
|
| + getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
|
| + &iStartChild, &iEndChild);
|
| + while( iStartChild>iLeavesEnd ){
|
| + sqlite_int64 iNextStart, iNextEnd;
|
| + rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
|
| + &iNextStart, &iNextEnd);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* If we've branched, follow the end branch, too. */
|
| + if( iStartChild!=iEndChild ){
|
| + sqlite_int64 iDummy;
|
| + rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
|
| + &iDummy, &iNextEnd);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| +
|
| + assert( iNextStart<=iNextEnd );
|
| + iStartChild = iNextStart;
|
| + iEndChild = iNextEnd;
|
| + }
|
| + assert( iStartChild<=iLeavesEnd );
|
| + assert( iEndChild<=iLeavesEnd );
|
| +
|
| + /* Scan through the leaf segments for doclists. */
|
| + return loadSegmentLeaves(v, iStartChild, iEndChild,
|
| + pTerm, nTerm, isPrefix, out);
|
| + }
|
| +}
|
| +
|
| +/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
|
| +** merge its doclist over *out (any duplicate doclists read from the
|
| +** segment rooted at pData will overwrite those in *out).
|
| +*/
|
| +/* TODO(shess) Consider changing this to determine the depth of the
|
| +** leaves using either the first characters of interior nodes (when
|
| +** ==1, we're one level above the leaves), or the first character of
|
| +** the root (which will describe the height of the tree directly).
|
| +** Either feels somewhat tricky to me.
|
| +*/
|
| +/* TODO(shess) The current merge is likely to be slow for large
|
| +** doclists (though it should process from newest/smallest to
|
| +** oldest/largest, so it may not be that bad). It might be useful to
|
| +** modify things to allow for N-way merging. This could either be
|
| +** within a segment, with pairwise merges across segments, or across
|
| +** all segments at once.
|
| +*/
|
| +static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
|
| + sqlite_int64 iLeavesEnd,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + DataBuffer *out){
|
| + DataBuffer result;
|
| + int rc;
|
| +
|
| + assert( nData>1 );
|
| +
|
| + /* This code should never be called with buffered updates. */
|
| + assert( v->nPendingData<0 );
|
| +
|
| + dataBufferInit(&result, 0);
|
| + rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
|
| + pTerm, nTerm, isPrefix, &result);
|
| + if( rc==SQLITE_OK && result.nData>0 ){
|
| + if( out->nData==0 ){
|
| + DataBuffer tmp = *out;
|
| + *out = result;
|
| + result = tmp;
|
| + }else{
|
| + DataBuffer merged;
|
| + DLReader readers[2];
|
| +
|
| + dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
|
| + dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
|
| + dataBufferInit(&merged, out->nData+result.nData);
|
| + docListMerge(&merged, readers, 2);
|
| + dataBufferDestroy(out);
|
| + *out = merged;
|
| + dlrDestroy(&readers[0]);
|
| + dlrDestroy(&readers[1]);
|
| + }
|
| + }
|
| + dataBufferDestroy(&result);
|
| + return rc;
|
| +}
|
| +
|
| +/* Scan the database and merge together the posting lists for the term
|
| +** into *out.
|
| +*/
|
| +static int termSelect(fulltext_vtab *v, int iColumn,
|
| + const char *pTerm, int nTerm, int isPrefix,
|
| + DocListType iType, DataBuffer *out){
|
| + DataBuffer doclist;
|
| + sqlite3_stmt *s;
|
| + int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + /* This code should never be called with buffered updates. */
|
| + assert( v->nPendingData<0 );
|
| +
|
| + dataBufferInit(&doclist, 0);
|
| +
|
| + /* Traverse the segments from oldest to newest so that newer doclist
|
| + ** elements for given docids overwrite older elements.
|
| + */
|
| + while( (rc = sqlite3_step(s))==SQLITE_ROW ){
|
| + const char *pData = sqlite3_column_blob(s, 2);
|
| + const int nData = sqlite3_column_bytes(s, 2);
|
| + const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
|
| + rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
|
| + &doclist);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| + }
|
| + if( rc==SQLITE_DONE ){
|
| + if( doclist.nData!=0 ){
|
| + /* TODO(shess) The old term_select_all() code applied the column
|
| + ** restrict as we merged segments, leading to smaller buffers.
|
| + ** This is probably worthwhile to bring back, once the new storage
|
| + ** system is checked in.
|
| + */
|
| + if( iColumn==v->nColumn) iColumn = -1;
|
| + docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
|
| + iColumn, iType, out);
|
| + }
|
| + rc = SQLITE_OK;
|
| + }
|
| +
|
| + err:
|
| + dataBufferDestroy(&doclist);
|
| + return rc;
|
| +}
|
| +
|
| +/****************************************************************/
|
| +/* Used to hold hashtable data for sorting. */
|
| +typedef struct TermData {
|
| + const char *pTerm;
|
| + int nTerm;
|
| + DLCollector *pCollector;
|
| +} TermData;
|
| +
|
| +/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
|
| +** for equal, >0 for greater-than).
|
| +*/
|
| +static int termDataCmp(const void *av, const void *bv){
|
| + const TermData *a = (const TermData *)av;
|
| + const TermData *b = (const TermData *)bv;
|
| + int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
|
| + int c = memcmp(a->pTerm, b->pTerm, n);
|
| + if( c!=0 ) return c;
|
| + return a->nTerm-b->nTerm;
|
| +}
|
| +
|
| +/* Order pTerms data by term, then write a new level 0 segment using
|
| +** LeafWriter.
|
| +*/
|
| +static int writeZeroSegment(fulltext_vtab *v, fts2Hash *pTerms){
|
| + fts2HashElem *e;
|
| + int idx, rc, i, n;
|
| + TermData *pData;
|
| + LeafWriter writer;
|
| + DataBuffer dl;
|
| +
|
| + /* Determine the next index at level 0, merging as necessary. */
|
| + rc = segdirNextIndex(v, 0, &idx);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + n = fts2HashCount(pTerms);
|
| + pData = sqlite3_malloc(n*sizeof(TermData));
|
| +
|
| + for(i = 0, e = fts2HashFirst(pTerms); e; i++, e = fts2HashNext(e)){
|
| + assert( i<n );
|
| + pData[i].pTerm = fts2HashKey(e);
|
| + pData[i].nTerm = fts2HashKeysize(e);
|
| + pData[i].pCollector = fts2HashData(e);
|
| + }
|
| + assert( i==n );
|
| +
|
| + /* TODO(shess) Should we allow user-defined collation sequences,
|
| + ** here? I think we only need that once we support prefix searches.
|
| + */
|
| + if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
|
| +
|
| + /* TODO(shess) Refactor so that we can write directly to the segment
|
| + ** DataBuffer, as happens for segment merges.
|
| + */
|
| + leafWriterInit(0, idx, &writer);
|
| + dataBufferInit(&dl, 0);
|
| + for(i=0; i<n; i++){
|
| + dataBufferReset(&dl);
|
| + dlcAddDoclist(pData[i].pCollector, &dl);
|
| + rc = leafWriterStep(v, &writer,
|
| + pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| + }
|
| + rc = leafWriterFinalize(v, &writer);
|
| +
|
| + err:
|
| + dataBufferDestroy(&dl);
|
| + sqlite3_free(pData);
|
| + leafWriterDestroy(&writer);
|
| + return rc;
|
| +}
|
| +
|
| +/* If pendingTerms has data, free it. */
|
| +static int clearPendingTerms(fulltext_vtab *v){
|
| + if( v->nPendingData>=0 ){
|
| + fts2HashElem *e;
|
| + for(e=fts2HashFirst(&v->pendingTerms); e; e=fts2HashNext(e)){
|
| + dlcDelete(fts2HashData(e));
|
| + }
|
| + fts2HashClear(&v->pendingTerms);
|
| + v->nPendingData = -1;
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* If pendingTerms has data, flush it to a level-zero segment, and
|
| +** free it.
|
| +*/
|
| +static int flushPendingTerms(fulltext_vtab *v){
|
| + if( v->nPendingData>=0 ){
|
| + int rc = writeZeroSegment(v, &v->pendingTerms);
|
| + if( rc==SQLITE_OK ) clearPendingTerms(v);
|
| + return rc;
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* If pendingTerms is "too big", or docid is out of order, flush it.
|
| +** Regardless, be certain that pendingTerms is initialized for use.
|
| +*/
|
| +static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
|
| + /* TODO(shess) Explore whether partially flushing the buffer on
|
| + ** forced-flush would provide better performance. I suspect that if
|
| + ** we ordered the doclists by size and flushed the largest until the
|
| + ** buffer was half empty, that would let the less frequent terms
|
| + ** generate longer doclists.
|
| + */
|
| + if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
|
| + int rc = flushPendingTerms(v);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| + if( v->nPendingData<0 ){
|
| + fts2HashInit(&v->pendingTerms, FTS2_HASH_STRING, 1);
|
| + v->nPendingData = 0;
|
| + }
|
| + v->iPrevDocid = iDocid;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* This function implements the xUpdate callback; it is the top-level entry
|
| + * point for inserting, deleting or updating a row in a full-text table. */
|
| +static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
|
| + sqlite_int64 *pRowid){
|
| + fulltext_vtab *v = (fulltext_vtab *) pVtab;
|
| + int rc;
|
| +
|
| + TRACE(("FTS2 Update %p\n", pVtab));
|
| +
|
| + if( nArg<2 ){
|
| + rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
|
| + if( rc==SQLITE_OK ){
|
| + /* If we just deleted the last row in the table, clear out the
|
| + ** index data.
|
| + */
|
| + rc = content_exists(v);
|
| + if( rc==SQLITE_ROW ){
|
| + rc = SQLITE_OK;
|
| + }else if( rc==SQLITE_DONE ){
|
| + /* Clear the pending terms so we don't flush a useless level-0
|
| + ** segment when the transaction closes.
|
| + */
|
| + rc = clearPendingTerms(v);
|
| + if( rc==SQLITE_OK ){
|
| + rc = segdir_delete_all(v);
|
| + }
|
| + }
|
| + }
|
| + } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
|
| + /* An update:
|
| + * ppArg[0] = old rowid
|
| + * ppArg[1] = new rowid
|
| + * ppArg[2..2+v->nColumn-1] = values
|
| + * ppArg[2+v->nColumn] = value for magic column (we ignore this)
|
| + */
|
| + sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
|
| + if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
|
| + sqlite3_value_int64(ppArg[1]) != rowid ){
|
| + rc = SQLITE_ERROR; /* we don't allow changing the rowid */
|
| + } else {
|
| + assert( nArg==2+v->nColumn+1);
|
| + rc = index_update(v, rowid, &ppArg[2]);
|
| + }
|
| + } else {
|
| + /* An insert:
|
| + * ppArg[1] = requested rowid
|
| + * ppArg[2..2+v->nColumn-1] = values
|
| + * ppArg[2+v->nColumn] = value for magic column (we ignore this)
|
| + */
|
| + assert( nArg==2+v->nColumn+1);
|
| + rc = index_insert(v, ppArg[1], &ppArg[2], pRowid);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int fulltextSync(sqlite3_vtab *pVtab){
|
| + TRACE(("FTS2 xSync()\n"));
|
| + return flushPendingTerms((fulltext_vtab *)pVtab);
|
| +}
|
| +
|
| +static int fulltextBegin(sqlite3_vtab *pVtab){
|
| + fulltext_vtab *v = (fulltext_vtab *) pVtab;
|
| + TRACE(("FTS2 xBegin()\n"));
|
| +
|
| + /* Any buffered updates should have been cleared by the previous
|
| + ** transaction.
|
| + */
|
| + assert( v->nPendingData<0 );
|
| + return clearPendingTerms(v);
|
| +}
|
| +
|
| +static int fulltextCommit(sqlite3_vtab *pVtab){
|
| + fulltext_vtab *v = (fulltext_vtab *) pVtab;
|
| + TRACE(("FTS2 xCommit()\n"));
|
| +
|
| + /* Buffered updates should have been cleared by fulltextSync(). */
|
| + assert( v->nPendingData<0 );
|
| + return clearPendingTerms(v);
|
| +}
|
| +
|
| +static int fulltextRollback(sqlite3_vtab *pVtab){
|
| + TRACE(("FTS2 xRollback()\n"));
|
| + return clearPendingTerms((fulltext_vtab *)pVtab);
|
| +}
|
| +
|
| +/*
|
| +** Implementation of the snippet() function for FTS2
|
| +*/
|
| +static void snippetFunc(
|
| + sqlite3_context *pContext,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + fulltext_cursor *pCursor;
|
| + if( argc<1 ) return;
|
| + if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
|
| + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
|
| + sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
|
| + }else{
|
| + const char *zStart = "<b>";
|
| + const char *zEnd = "</b>";
|
| + const char *zEllipsis = "<b>...</b>";
|
| + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
|
| + if( argc>=2 ){
|
| + zStart = (const char*)sqlite3_value_text(argv[1]);
|
| + if( argc>=3 ){
|
| + zEnd = (const char*)sqlite3_value_text(argv[2]);
|
| + if( argc>=4 ){
|
| + zEllipsis = (const char*)sqlite3_value_text(argv[3]);
|
| + }
|
| + }
|
| + }
|
| + snippetAllOffsets(pCursor);
|
| + snippetText(pCursor, zStart, zEnd, zEllipsis);
|
| + sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
|
| + pCursor->snippet.nSnippet, SQLITE_STATIC);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Implementation of the offsets() function for FTS2
|
| +*/
|
| +static void snippetOffsetsFunc(
|
| + sqlite3_context *pContext,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + fulltext_cursor *pCursor;
|
| + if( argc<1 ) return;
|
| + if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
|
| + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
|
| + sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
|
| + }else{
|
| + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
|
| + snippetAllOffsets(pCursor);
|
| + snippetOffsetText(&pCursor->snippet);
|
| + sqlite3_result_text(pContext,
|
| + pCursor->snippet.zOffset, pCursor->snippet.nOffset,
|
| + SQLITE_STATIC);
|
| + }
|
| +}
|
| +
|
| +/* OptLeavesReader is nearly identical to LeavesReader, except that
|
| +** where LeavesReader is geared towards the merging of complete
|
| +** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
|
| +** is geared towards implementation of the optimize() function, and
|
| +** can merge all segments simultaneously. This version may be
|
| +** somewhat less efficient than LeavesReader because it merges into an
|
| +** accumulator rather than doing an N-way merge, but since segment
|
| +** size grows exponentially (so segment count logrithmically) this is
|
| +** probably not an immediate problem.
|
| +*/
|
| +/* TODO(shess): Prove that assertion, or extend the merge code to
|
| +** merge tree fashion (like the prefix-searching code does).
|
| +*/
|
| +/* TODO(shess): OptLeavesReader and LeavesReader could probably be
|
| +** merged with little or no loss of performance for LeavesReader. The
|
| +** merged code would need to handle >MERGE_COUNT segments, and would
|
| +** also need to be able to optionally optimize away deletes.
|
| +*/
|
| +typedef struct OptLeavesReader {
|
| + /* Segment number, to order readers by age. */
|
| + int segment;
|
| + LeavesReader reader;
|
| +} OptLeavesReader;
|
| +
|
| +static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
|
| + return leavesReaderAtEnd(&pReader->reader);
|
| +}
|
| +static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
|
| + return leavesReaderTermBytes(&pReader->reader);
|
| +}
|
| +static const char *optLeavesReaderData(OptLeavesReader *pReader){
|
| + return leavesReaderData(&pReader->reader);
|
| +}
|
| +static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
|
| + return leavesReaderDataBytes(&pReader->reader);
|
| +}
|
| +static const char *optLeavesReaderTerm(OptLeavesReader *pReader){
|
| + return leavesReaderTerm(&pReader->reader);
|
| +}
|
| +static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){
|
| + return leavesReaderStep(v, &pReader->reader);
|
| +}
|
| +static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
|
| + return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
|
| +}
|
| +/* Order by term ascending, segment ascending (oldest to newest), with
|
| +** exhausted readers to the end.
|
| +*/
|
| +static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
|
| + int c = optLeavesReaderTermCmp(lr1, lr2);
|
| + if( c!=0 ) return c;
|
| + return lr1->segment-lr2->segment;
|
| +}
|
| +/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
|
| +** pLr[1..nLr-1] is already sorted.
|
| +*/
|
| +static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
|
| + while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
|
| + OptLeavesReader tmp = pLr[0];
|
| + pLr[0] = pLr[1];
|
| + pLr[1] = tmp;
|
| + nLr--;
|
| + pLr++;
|
| + }
|
| +}
|
| +
|
| +/* optimize() helper function. Put the readers in order and iterate
|
| +** through them, merging doclists for matching terms into pWriter.
|
| +** Returns SQLITE_OK on success, or the SQLite error code which
|
| +** prevented success.
|
| +*/
|
| +static int optimizeInternal(fulltext_vtab *v,
|
| + OptLeavesReader *readers, int nReaders,
|
| + LeafWriter *pWriter){
|
| + int i, rc = SQLITE_OK;
|
| + DataBuffer doclist, merged, tmp;
|
| +
|
| + /* Order the readers. */
|
| + i = nReaders;
|
| + while( i-- > 0 ){
|
| + optLeavesReaderReorder(&readers[i], nReaders-i);
|
| + }
|
| +
|
| + dataBufferInit(&doclist, LEAF_MAX);
|
| + dataBufferInit(&merged, LEAF_MAX);
|
| +
|
| + /* Exhausted readers bubble to the end, so when the first reader is
|
| + ** at eof, all are at eof.
|
| + */
|
| + while( !optLeavesReaderAtEnd(&readers[0]) ){
|
| +
|
| + /* Figure out how many readers share the next term. */
|
| + for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
|
| + if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
|
| + }
|
| +
|
| + /* Special-case for no merge. */
|
| + if( i==1 ){
|
| + /* Trim deletions from the doclist. */
|
| + dataBufferReset(&merged);
|
| + docListTrim(DL_DEFAULT,
|
| + optLeavesReaderData(&readers[0]),
|
| + optLeavesReaderDataBytes(&readers[0]),
|
| + -1, DL_DEFAULT, &merged);
|
| + }else{
|
| + DLReader dlReaders[MERGE_COUNT];
|
| + int iReader, nReaders;
|
| +
|
| + /* Prime the pipeline with the first reader's doclist. After
|
| + ** one pass index 0 will reference the accumulated doclist.
|
| + */
|
| + dlrInit(&dlReaders[0], DL_DEFAULT,
|
| + optLeavesReaderData(&readers[0]),
|
| + optLeavesReaderDataBytes(&readers[0]));
|
| + iReader = 1;
|
| +
|
| + assert( iReader<i ); /* Must execute the loop at least once. */
|
| + while( iReader<i ){
|
| + /* Merge 16 inputs per pass. */
|
| + for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
|
| + iReader++, nReaders++ ){
|
| + dlrInit(&dlReaders[nReaders], DL_DEFAULT,
|
| + optLeavesReaderData(&readers[iReader]),
|
| + optLeavesReaderDataBytes(&readers[iReader]));
|
| + }
|
| +
|
| + /* Merge doclists and swap result into accumulator. */
|
| + dataBufferReset(&merged);
|
| + docListMerge(&merged, dlReaders, nReaders);
|
| + tmp = merged;
|
| + merged = doclist;
|
| + doclist = tmp;
|
| +
|
| + while( nReaders-- > 0 ){
|
| + dlrDestroy(&dlReaders[nReaders]);
|
| + }
|
| +
|
| + /* Accumulated doclist to reader 0 for next pass. */
|
| + dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
|
| + }
|
| +
|
| + /* Destroy reader that was left in the pipeline. */
|
| + dlrDestroy(&dlReaders[0]);
|
| +
|
| + /* Trim deletions from the doclist. */
|
| + dataBufferReset(&merged);
|
| + docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
|
| + -1, DL_DEFAULT, &merged);
|
| + }
|
| +
|
| + /* Only pass doclists with hits (skip if all hits deleted). */
|
| + if( merged.nData>0 ){
|
| + rc = leafWriterStep(v, pWriter,
|
| + optLeavesReaderTerm(&readers[0]),
|
| + optLeavesReaderTermBytes(&readers[0]),
|
| + merged.pData, merged.nData);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| + }
|
| +
|
| + /* Step merged readers to next term and reorder. */
|
| + while( i-- > 0 ){
|
| + rc = optLeavesReaderStep(v, &readers[i]);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| +
|
| + optLeavesReaderReorder(&readers[i], nReaders-i);
|
| + }
|
| + }
|
| +
|
| + err:
|
| + dataBufferDestroy(&doclist);
|
| + dataBufferDestroy(&merged);
|
| + return rc;
|
| +}
|
| +
|
| +/* Implement optimize() function for FTS3. optimize(t) merges all
|
| +** segments in the fts index into a single segment. 't' is the magic
|
| +** table-named column.
|
| +*/
|
| +static void optimizeFunc(sqlite3_context *pContext,
|
| + int argc, sqlite3_value **argv){
|
| + fulltext_cursor *pCursor;
|
| + if( argc>1 ){
|
| + sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
|
| + }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
|
| + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
|
| + sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
|
| + }else{
|
| + fulltext_vtab *v;
|
| + int i, rc, iMaxLevel;
|
| + OptLeavesReader *readers;
|
| + int nReaders;
|
| + LeafWriter writer;
|
| + sqlite3_stmt *s;
|
| +
|
| + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
|
| + v = cursor_vtab(pCursor);
|
| +
|
| + /* Flush any buffered updates before optimizing. */
|
| + rc = flushPendingTerms(v);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| +
|
| + rc = segdir_count(v, &nReaders, &iMaxLevel);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| + if( nReaders==0 || nReaders==1 ){
|
| + sqlite3_result_text(pContext, "Index already optimal", -1,
|
| + SQLITE_STATIC);
|
| + return;
|
| + }
|
| +
|
| + rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
|
| + if( rc!=SQLITE_OK ) goto err;
|
| +
|
| + readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
|
| + if( readers==NULL ) goto err;
|
| +
|
| + /* Note that there will already be a segment at this position
|
| + ** until we call segdir_delete() on iMaxLevel.
|
| + */
|
| + leafWriterInit(iMaxLevel, 0, &writer);
|
| +
|
| + i = 0;
|
| + while( (rc = sqlite3_step(s))==SQLITE_ROW ){
|
| + sqlite_int64 iStart = sqlite3_column_int64(s, 0);
|
| + sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
|
| + const char *pRootData = sqlite3_column_blob(s, 2);
|
| + int nRootData = sqlite3_column_bytes(s, 2);
|
| +
|
| + assert( i<nReaders );
|
| + rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
|
| + &readers[i].reader);
|
| + if( rc!=SQLITE_OK ) break;
|
| +
|
| + readers[i].segment = i;
|
| + i++;
|
| + }
|
| +
|
| + /* If we managed to successfully read them all, optimize them. */
|
| + if( rc==SQLITE_DONE ){
|
| + assert( i==nReaders );
|
| + rc = optimizeInternal(v, readers, nReaders, &writer);
|
| + }
|
| +
|
| + while( i-- > 0 ){
|
| + leavesReaderDestroy(&readers[i].reader);
|
| + }
|
| + sqlite3_free(readers);
|
| +
|
| + /* If we've successfully gotten to here, delete the old segments
|
| + ** and flush the interior structure of the new segment.
|
| + */
|
| + if( rc==SQLITE_OK ){
|
| + for( i=0; i<=iMaxLevel; i++ ){
|
| + rc = segdir_delete(v, i);
|
| + if( rc!=SQLITE_OK ) break;
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
|
| + }
|
| +
|
| + leafWriterDestroy(&writer);
|
| +
|
| + if( rc!=SQLITE_OK ) goto err;
|
| +
|
| + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
|
| + return;
|
| +
|
| + /* TODO(shess): Error-handling needs to be improved along the
|
| + ** lines of the dump_ functions.
|
| + */
|
| + err:
|
| + {
|
| + char buf[512];
|
| + sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
|
| + sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
|
| + sqlite3_result_error(pContext, buf, -1);
|
| + }
|
| + }
|
| +}
|
| +
|
| +#ifdef SQLITE_TEST
|
| +/* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
|
| +** pull the error from the context's db handle.
|
| +*/
|
| +static void generateError(sqlite3_context *pContext,
|
| + const char *prefix, const char *msg){
|
| + char buf[512];
|
| + if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
|
| + sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
|
| + sqlite3_result_error(pContext, buf, -1);
|
| +}
|
| +
|
| +/* Helper function to collect the set of terms in the segment into
|
| +** pTerms. The segment is defined by the leaf nodes between
|
| +** iStartBlockid and iEndBlockid, inclusive, or by the contents of
|
| +** pRootData if iStartBlockid is 0 (in which case the entire segment
|
| +** fit in a leaf).
|
| +*/
|
| +static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s,
|
| + fts2Hash *pTerms){
|
| + const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
|
| + const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
|
| + const char *pRootData = sqlite3_column_blob(s, 2);
|
| + const int nRootData = sqlite3_column_bytes(s, 2);
|
| + LeavesReader reader;
|
| + int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
|
| + pRootData, nRootData, &reader);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| +
|
| + while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
|
| + const char *pTerm = leavesReaderTerm(&reader);
|
| + const int nTerm = leavesReaderTermBytes(&reader);
|
| + void *oldValue = sqlite3Fts2HashFind(pTerms, pTerm, nTerm);
|
| + void *newValue = (void *)((char *)oldValue+1);
|
| +
|
| + /* From the comment before sqlite3Fts2HashInsert in fts2_hash.c,
|
| + ** the data value passed is returned in case of malloc failure.
|
| + */
|
| + if( newValue==sqlite3Fts2HashInsert(pTerms, pTerm, nTerm, newValue) ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + rc = leavesReaderStep(v, &reader);
|
| + }
|
| + }
|
| +
|
| + leavesReaderDestroy(&reader);
|
| + return rc;
|
| +}
|
| +
|
| +/* Helper function to build the result string for dump_terms(). */
|
| +static int generateTermsResult(sqlite3_context *pContext, fts2Hash *pTerms){
|
| + int iTerm, nTerms, nResultBytes, iByte;
|
| + char *result;
|
| + TermData *pData;
|
| + fts2HashElem *e;
|
| +
|
| + /* Iterate pTerms to generate an array of terms in pData for
|
| + ** sorting.
|
| + */
|
| + nTerms = fts2HashCount(pTerms);
|
| + assert( nTerms>0 );
|
| + pData = sqlite3_malloc(nTerms*sizeof(TermData));
|
| + if( pData==NULL ) return SQLITE_NOMEM;
|
| +
|
| + nResultBytes = 0;
|
| + for(iTerm = 0, e = fts2HashFirst(pTerms); e; iTerm++, e = fts2HashNext(e)){
|
| + nResultBytes += fts2HashKeysize(e)+1; /* Term plus trailing space */
|
| + assert( iTerm<nTerms );
|
| + pData[iTerm].pTerm = fts2HashKey(e);
|
| + pData[iTerm].nTerm = fts2HashKeysize(e);
|
| + pData[iTerm].pCollector = fts2HashData(e); /* unused */
|
| + }
|
| + assert( iTerm==nTerms );
|
| +
|
| + assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
|
| + result = sqlite3_malloc(nResultBytes);
|
| + if( result==NULL ){
|
| + sqlite3_free(pData);
|
| + return SQLITE_NOMEM;
|
| + }
|
| +
|
| + if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
|
| +
|
| + /* Read the terms in order to build the result. */
|
| + iByte = 0;
|
| + for(iTerm=0; iTerm<nTerms; ++iTerm){
|
| + memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
|
| + iByte += pData[iTerm].nTerm;
|
| + result[iByte++] = ' ';
|
| + }
|
| + assert( iByte==nResultBytes );
|
| + assert( result[nResultBytes-1]==' ' );
|
| + result[nResultBytes-1] = '\0';
|
| +
|
| + /* Passes away ownership of result. */
|
| + sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
|
| + sqlite3_free(pData);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Implements dump_terms() for use in inspecting the fts2 index from
|
| +** tests. TEXT result containing the ordered list of terms joined by
|
| +** spaces. dump_terms(t, level, idx) dumps the terms for the segment
|
| +** specified by level, idx (in %_segdir), while dump_terms(t) dumps
|
| +** all terms in the index. In both cases t is the fts table's magic
|
| +** table-named column.
|
| +*/
|
| +static void dumpTermsFunc(
|
| + sqlite3_context *pContext,
|
| + int argc, sqlite3_value **argv
|
| +){
|
| + fulltext_cursor *pCursor;
|
| + if( argc!=3 && argc!=1 ){
|
| + generateError(pContext, "dump_terms", "incorrect arguments");
|
| + }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
|
| + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
|
| + generateError(pContext, "dump_terms", "illegal first argument");
|
| + }else{
|
| + fulltext_vtab *v;
|
| + fts2Hash terms;
|
| + sqlite3_stmt *s = NULL;
|
| + int rc;
|
| +
|
| + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
|
| + v = cursor_vtab(pCursor);
|
| +
|
| + /* If passed only the cursor column, get all segments. Otherwise
|
| + ** get the segment described by the following two arguments.
|
| + */
|
| + if( argc==1 ){
|
| + rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
|
| + }else{
|
| + rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + generateError(pContext, "dump_terms", NULL);
|
| + return;
|
| + }
|
| +
|
| + /* Collect the terms for each segment. */
|
| + sqlite3Fts2HashInit(&terms, FTS2_HASH_STRING, 1);
|
| + while( (rc = sqlite3_step(s))==SQLITE_ROW ){
|
| + rc = collectSegmentTerms(v, s, &terms);
|
| + if( rc!=SQLITE_OK ) break;
|
| + }
|
| +
|
| + if( rc!=SQLITE_DONE ){
|
| + sqlite3_reset(s);
|
| + generateError(pContext, "dump_terms", NULL);
|
| + }else{
|
| + const int nTerms = fts2HashCount(&terms);
|
| + if( nTerms>0 ){
|
| + rc = generateTermsResult(pContext, &terms);
|
| + if( rc==SQLITE_NOMEM ){
|
| + generateError(pContext, "dump_terms", "out of memory");
|
| + }else{
|
| + assert( rc==SQLITE_OK );
|
| + }
|
| + }else if( argc==3 ){
|
| + /* The specific segment asked for could not be found. */
|
| + generateError(pContext, "dump_terms", "segment not found");
|
| + }else{
|
| + /* No segments found. */
|
| + /* TODO(shess): It should be impossible to reach this. This
|
| + ** case can only happen for an empty table, in which case
|
| + ** SQLite has no rows to call this function on.
|
| + */
|
| + sqlite3_result_null(pContext);
|
| + }
|
| + }
|
| + sqlite3Fts2HashClear(&terms);
|
| + }
|
| +}
|
| +
|
| +/* Expand the DL_DEFAULT doclist in pData into a text result in
|
| +** pContext.
|
| +*/
|
| +static void createDoclistResult(sqlite3_context *pContext,
|
| + const char *pData, int nData){
|
| + DataBuffer dump;
|
| + DLReader dlReader;
|
| +
|
| + assert( pData!=NULL && nData>0 );
|
| +
|
| + dataBufferInit(&dump, 0);
|
| + dlrInit(&dlReader, DL_DEFAULT, pData, nData);
|
| + for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
|
| + char buf[256];
|
| + PLReader plReader;
|
| +
|
| + plrInit(&plReader, &dlReader);
|
| + if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
|
| + sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
|
| + dataBufferAppend(&dump, buf, strlen(buf));
|
| + }else{
|
| + int iColumn = plrColumn(&plReader);
|
| +
|
| + sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
|
| + dlrDocid(&dlReader), iColumn);
|
| + dataBufferAppend(&dump, buf, strlen(buf));
|
| +
|
| + for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
|
| + if( plrColumn(&plReader)!=iColumn ){
|
| + iColumn = plrColumn(&plReader);
|
| + sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
|
| + assert( dump.nData>0 );
|
| + dump.nData--; /* Overwrite trailing space. */
|
| + assert( dump.pData[dump.nData]==' ');
|
| + dataBufferAppend(&dump, buf, strlen(buf));
|
| + }
|
| + if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
|
| + sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
|
| + plrPosition(&plReader),
|
| + plrStartOffset(&plReader), plrEndOffset(&plReader));
|
| + }else if( DL_DEFAULT==DL_POSITIONS ){
|
| + sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
|
| + }else{
|
| + assert( NULL=="Unhandled DL_DEFAULT value");
|
| + }
|
| + dataBufferAppend(&dump, buf, strlen(buf));
|
| + }
|
| + plrDestroy(&plReader);
|
| +
|
| + assert( dump.nData>0 );
|
| + dump.nData--; /* Overwrite trailing space. */
|
| + assert( dump.pData[dump.nData]==' ');
|
| + dataBufferAppend(&dump, "]] ", 3);
|
| + }
|
| + }
|
| + dlrDestroy(&dlReader);
|
| +
|
| + assert( dump.nData>0 );
|
| + dump.nData--; /* Overwrite trailing space. */
|
| + assert( dump.pData[dump.nData]==' ');
|
| + dump.pData[dump.nData] = '\0';
|
| + assert( dump.nData>0 );
|
| +
|
| + /* Passes ownership of dump's buffer to pContext. */
|
| + sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
|
| + dump.pData = NULL;
|
| + dump.nData = dump.nCapacity = 0;
|
| +}
|
| +
|
| +/* Implements dump_doclist() for use in inspecting the fts2 index from
|
| +** tests. TEXT result containing a string representation of the
|
| +** doclist for the indicated term. dump_doclist(t, term, level, idx)
|
| +** dumps the doclist for term from the segment specified by level, idx
|
| +** (in %_segdir), while dump_doclist(t, term) dumps the logical
|
| +** doclist for the term across all segments. The per-segment doclist
|
| +** can contain deletions, while the full-index doclist will not
|
| +** (deletions are omitted).
|
| +**
|
| +** Result formats differ with the setting of DL_DEFAULTS. Examples:
|
| +**
|
| +** DL_DOCIDS: [1] [3] [7]
|
| +** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
|
| +** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
|
| +**
|
| +** In each case the number after the outer '[' is the docid. In the
|
| +** latter two cases, the number before the inner '[' is the column
|
| +** associated with the values within. For DL_POSITIONS the numbers
|
| +** within are the positions, for DL_POSITIONS_OFFSETS they are the
|
| +** position, the start offset, and the end offset.
|
| +*/
|
| +static void dumpDoclistFunc(
|
| + sqlite3_context *pContext,
|
| + int argc, sqlite3_value **argv
|
| +){
|
| + fulltext_cursor *pCursor;
|
| + if( argc!=2 && argc!=4 ){
|
| + generateError(pContext, "dump_doclist", "incorrect arguments");
|
| + }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
|
| + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
|
| + generateError(pContext, "dump_doclist", "illegal first argument");
|
| + }else if( sqlite3_value_text(argv[1])==NULL ||
|
| + sqlite3_value_text(argv[1])[0]=='\0' ){
|
| + generateError(pContext, "dump_doclist", "empty second argument");
|
| + }else{
|
| + const char *pTerm = (const char *)sqlite3_value_text(argv[1]);
|
| + const int nTerm = strlen(pTerm);
|
| + fulltext_vtab *v;
|
| + int rc;
|
| + DataBuffer doclist;
|
| +
|
| + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
|
| + v = cursor_vtab(pCursor);
|
| +
|
| + dataBufferInit(&doclist, 0);
|
| +
|
| + /* termSelect() yields the same logical doclist that queries are
|
| + ** run against.
|
| + */
|
| + if( argc==2 ){
|
| + rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
|
| + }else{
|
| + sqlite3_stmt *s = NULL;
|
| +
|
| + /* Get our specific segment's information. */
|
| + rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_step(s);
|
| +
|
| + if( rc==SQLITE_DONE ){
|
| + dataBufferDestroy(&doclist);
|
| + generateError(pContext, "dump_doclist", "segment not found");
|
| + return;
|
| + }
|
| +
|
| + /* Found a segment, load it into doclist. */
|
| + if( rc==SQLITE_ROW ){
|
| + const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
|
| + const char *pData = sqlite3_column_blob(s, 2);
|
| + const int nData = sqlite3_column_bytes(s, 2);
|
| +
|
| + /* loadSegment() is used by termSelect() to load each
|
| + ** segment's data.
|
| + */
|
| + rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
|
| + &doclist);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_step(s);
|
| +
|
| + /* Should not have more than one matching segment. */
|
| + if( rc!=SQLITE_DONE ){
|
| + sqlite3_reset(s);
|
| + dataBufferDestroy(&doclist);
|
| + generateError(pContext, "dump_doclist", "invalid segdir");
|
| + return;
|
| + }
|
| + rc = SQLITE_OK;
|
| + }
|
| + }
|
| + }
|
| +
|
| + sqlite3_reset(s);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + if( doclist.nData>0 ){
|
| + createDoclistResult(pContext, doclist.pData, doclist.nData);
|
| + }else{
|
| + /* TODO(shess): This can happen if the term is not present, or
|
| + ** if all instances of the term have been deleted and this is
|
| + ** an all-index dump. It may be interesting to distinguish
|
| + ** these cases.
|
| + */
|
| + sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
|
| + }
|
| + }else if( rc==SQLITE_NOMEM ){
|
| + /* Handle out-of-memory cases specially because if they are
|
| + ** generated in fts2 code they may not be reflected in the db
|
| + ** handle.
|
| + */
|
| + /* TODO(shess): Handle this more comprehensively.
|
| + ** sqlite3ErrStr() has what I need, but is internal.
|
| + */
|
| + generateError(pContext, "dump_doclist", "out of memory");
|
| + }else{
|
| + generateError(pContext, "dump_doclist", NULL);
|
| + }
|
| +
|
| + dataBufferDestroy(&doclist);
|
| + }
|
| +}
|
| +#endif
|
| +
|
| +/*
|
| +** This routine implements the xFindFunction method for the FTS2
|
| +** virtual table.
|
| +*/
|
| +static int fulltextFindFunction(
|
| + sqlite3_vtab *pVtab,
|
| + int nArg,
|
| + const char *zName,
|
| + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
|
| + void **ppArg
|
| +){
|
| + if( strcmp(zName,"snippet")==0 ){
|
| + *pxFunc = snippetFunc;
|
| + return 1;
|
| + }else if( strcmp(zName,"offsets")==0 ){
|
| + *pxFunc = snippetOffsetsFunc;
|
| + return 1;
|
| + }else if( strcmp(zName,"optimize")==0 ){
|
| + *pxFunc = optimizeFunc;
|
| + return 1;
|
| +#ifdef SQLITE_TEST
|
| + /* NOTE(shess): These functions are present only for testing
|
| + ** purposes. No particular effort is made to optimize their
|
| + ** execution or how they build their results.
|
| + */
|
| + }else if( strcmp(zName,"dump_terms")==0 ){
|
| + /* fprintf(stderr, "Found dump_terms\n"); */
|
| + *pxFunc = dumpTermsFunc;
|
| + return 1;
|
| + }else if( strcmp(zName,"dump_doclist")==0 ){
|
| + /* fprintf(stderr, "Found dump_doclist\n"); */
|
| + *pxFunc = dumpDoclistFunc;
|
| + return 1;
|
| +#endif
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +/*
|
| +** Rename an fts2 table.
|
| +*/
|
| +static int fulltextRename(
|
| + sqlite3_vtab *pVtab,
|
| + const char *zName
|
| +){
|
| + fulltext_vtab *p = (fulltext_vtab *)pVtab;
|
| + int rc = SQLITE_NOMEM;
|
| + char *zSql = sqlite3_mprintf(
|
| + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
|
| + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
|
| + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
|
| + , p->zDb, p->zName, zName
|
| + , p->zDb, p->zName, zName
|
| + , p->zDb, p->zName, zName
|
| + );
|
| + if( zSql ){
|
| + rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
|
| + sqlite3_free(zSql);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static const sqlite3_module fts2Module = {
|
| + /* iVersion */ 0,
|
| + /* xCreate */ fulltextCreate,
|
| + /* xConnect */ fulltextConnect,
|
| + /* xBestIndex */ fulltextBestIndex,
|
| + /* xDisconnect */ fulltextDisconnect,
|
| + /* xDestroy */ fulltextDestroy,
|
| + /* xOpen */ fulltextOpen,
|
| + /* xClose */ fulltextClose,
|
| + /* xFilter */ fulltextFilter,
|
| + /* xNext */ fulltextNext,
|
| + /* xEof */ fulltextEof,
|
| + /* xColumn */ fulltextColumn,
|
| + /* xRowid */ fulltextRowid,
|
| + /* xUpdate */ fulltextUpdate,
|
| + /* xBegin */ fulltextBegin,
|
| + /* xSync */ fulltextSync,
|
| + /* xCommit */ fulltextCommit,
|
| + /* xRollback */ fulltextRollback,
|
| + /* xFindFunction */ fulltextFindFunction,
|
| + /* xRename */ fulltextRename,
|
| +};
|
| +
|
| +static void hashDestroy(void *p){
|
| + fts2Hash *pHash = (fts2Hash *)p;
|
| + sqlite3Fts2HashClear(pHash);
|
| + sqlite3_free(pHash);
|
| +}
|
| +
|
| +/*
|
| +** The fts2 built-in tokenizers - "simple" and "porter" - are implemented
|
| +** in files fts2_tokenizer1.c and fts2_porter.c respectively. The following
|
| +** two forward declarations are for functions declared in these files
|
| +** used to retrieve the respective implementations.
|
| +**
|
| +** Calling sqlite3Fts2SimpleTokenizerModule() sets the value pointed
|
| +** to by the argument to point a the "simple" tokenizer implementation.
|
| +** Function ...PorterTokenizerModule() sets *pModule to point to the
|
| +** porter tokenizer/stemmer implementation.
|
| +*/
|
| +void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
| +void sqlite3Fts2PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
| +void sqlite3Fts2IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
| +
|
| +int sqlite3Fts2InitHashTable(sqlite3 *, fts2Hash *, const char *);
|
| +
|
| +/*
|
| +** Initialize the fts2 extension. If this extension is built as part
|
| +** of the sqlite library, then this function is called directly by
|
| +** SQLite. If fts2 is built as a dynamically loadable extension, this
|
| +** function is called by the sqlite3_extension_init() entry point.
|
| +*/
|
| +int sqlite3Fts2Init(sqlite3 *db){
|
| + int rc = SQLITE_OK;
|
| + fts2Hash *pHash = 0;
|
| + const sqlite3_tokenizer_module *pSimple = 0;
|
| + const sqlite3_tokenizer_module *pPorter = 0;
|
| + const sqlite3_tokenizer_module *pIcu = 0;
|
| +
|
| + sqlite3Fts2SimpleTokenizerModule(&pSimple);
|
| + sqlite3Fts2PorterTokenizerModule(&pPorter);
|
| +#ifdef SQLITE_ENABLE_ICU
|
| + sqlite3Fts2IcuTokenizerModule(&pIcu);
|
| +#endif
|
| +
|
| + /* Allocate and initialize the hash-table used to store tokenizers. */
|
| + pHash = sqlite3_malloc(sizeof(fts2Hash));
|
| + if( !pHash ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
|
| + }
|
| +
|
| + /* Load the built-in tokenizers into the hash table */
|
| + if( rc==SQLITE_OK ){
|
| + if( sqlite3Fts2HashInsert(pHash, "simple", 7, (void *)pSimple)
|
| + || sqlite3Fts2HashInsert(pHash, "porter", 7, (void *)pPorter)
|
| + || (pIcu && sqlite3Fts2HashInsert(pHash, "icu", 4, (void *)pIcu))
|
| + ){
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| +
|
| + /* Create the virtual table wrapper around the hash-table and overload
|
| + ** the two scalar functions. If this is successful, register the
|
| + ** module with sqlite.
|
| + */
|
| + if( SQLITE_OK==rc
|
| + && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer"))
|
| + && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
|
| + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
|
| + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
|
| +#ifdef SQLITE_TEST
|
| + && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
|
| + && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
|
| +#endif
|
| + ){
|
| + return sqlite3_create_module_v2(
|
| + db, "fts2", &fts2Module, (void *)pHash, hashDestroy
|
| + );
|
| + }
|
| +
|
| + /* An error has occurred. Delete the hash table and return the error code. */
|
| + assert( rc!=SQLITE_OK );
|
| + if( pHash ){
|
| + sqlite3Fts2HashClear(pHash);
|
| + sqlite3_free(pHash);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +#if !SQLITE_CORE
|
| +#ifdef _WIN32
|
| +__declspec(dllexport)
|
| +#endif
|
| +int sqlite3_fts2_init(
|
| + sqlite3 *db,
|
| + char **pzErrMsg,
|
| + const sqlite3_api_routines *pApi
|
| +){
|
| + SQLITE_EXTENSION_INIT2(pApi)
|
| + return sqlite3Fts2Init(db);
|
| +}
|
| +#endif
|
| +
|
| +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
|
|
|
Chromium Code Reviews
Issue 949043002:
Add //third_party/sqlite to dirs_to_snapshot, remove net_sql.patch (Closed)
Base URL: git@github.com:domokit/mojo.git@master