Index: third_party/sqlite/amalgamation/sqlite3.07.c |
diff --git a/third_party/sqlite/amalgamation/sqlite3.07.c b/third_party/sqlite/amalgamation/sqlite3.07.c |
new file mode 100644 |
index 0000000000000000000000000000000000000000..142642a5fbe2b77b749c005276999d2ccfe01843 |
--- /dev/null |
+++ b/third_party/sqlite/amalgamation/sqlite3.07.c |
@@ -0,0 +1,23748 @@ |
+/************** Begin file fts3.c ********************************************/ |
+/* |
+** 2006 Oct 10 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This is an SQLite module implementing full-text search. |
+*/ |
+ |
+/* |
+** The code in this file is only compiled if: |
+** |
+** * The FTS3 module is being built as an extension |
+** (in which case SQLITE_CORE is not defined), or |
+** |
+** * The FTS3 module is being built into the core of |
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). |
+*/ |
+ |
+/* The full-text index is stored in a series of b+tree (-like) |
+** structures called segments which map terms to doclists. The |
+** structures are like b+trees in layout, but are constructed from the |
+** bottom up in optimal fashion and are not updatable. Since trees |
+** are built from the bottom up, things will be described from the |
+** bottom up. |
+** |
+** |
+**** Varints **** |
+** The basic unit of encoding is a variable-length integer called a |
+** varint. We encode variable-length integers in little-endian order |
+** using seven bits * per byte as follows: |
+** |
+** KEY: |
+** A = 0xxxxxxx 7 bits of data and one flag bit |
+** B = 1xxxxxxx 7 bits of data and one flag bit |
+** |
+** 7 bits - A |
+** 14 bits - BA |
+** 21 bits - BBA |
+** and so on. |
+** |
+** This is similar in concept to how sqlite encodes "varints" but |
+** the encoding is not the same. SQLite varints are big-endian |
+** are are limited to 9 bytes in length whereas FTS3 varints are |
+** little-endian and can be up to 10 bytes in length (in theory). |
+** |
+** Example encodings: |
+** |
+** 1: 0x01 |
+** 127: 0x7f |
+** 128: 0x81 0x00 |
+** |
+** |
+**** Document lists **** |
+** A doclist (document list) holds a docid-sorted list of hits for a |
+** given term. Doclists hold docids and associated token positions. |
+** A docid is the unique integer identifier for a single document. |
+** A position is the index of a word within the document. The first |
+** word of the document has a position of 0. |
+** |
+** FTS3 used to optionally store character offsets using a compile-time |
+** option. But that functionality is no longer supported. |
+** |
+** A doclist is stored like this: |
+** |
+** array { |
+** varint docid; (delta from previous doclist) |
+** array { (position list for column 0) |
+** varint position; (2 more than the delta from previous position) |
+** } |
+** array { |
+** varint POS_COLUMN; (marks start of position list for new column) |
+** varint column; (index of new column) |
+** array { |
+** varint position; (2 more than the delta from previous position) |
+** } |
+** } |
+** varint POS_END; (marks end of positions for this document. |
+** } |
+** |
+** Here, array { X } means zero or more occurrences of X, adjacent in |
+** memory. A "position" is an index of a token in the token stream |
+** generated by the tokenizer. Note that POS_END and POS_COLUMN occur |
+** in the same logical place as the position element, and act as sentinals |
+** ending a position list array. POS_END is 0. POS_COLUMN is 1. |
+** The positions numbers are not stored literally but rather as two more |
+** than the difference from the prior position, or the just the position plus |
+** 2 for the first position. Example: |
+** |
+** label: A B C D E F G H I J K |
+** value: 123 5 9 1 1 14 35 0 234 72 0 |
+** |
+** The 123 value is the first docid. For column zero in this document |
+** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 |
+** at D signals the start of a new column; the 1 at E indicates that the |
+** new column is column number 1. There are two positions at 12 and 45 |
+** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The |
+** 234 at I is the delta to next docid (357). It has one position 70 |
+** (72-2) and then terminates with the 0 at K. |
+** |
+** A "position-list" is the list of positions for multiple columns for |
+** a single docid. A "column-list" is the set of positions for a single |
+** column. Hence, a position-list consists of one or more column-lists, |
+** a document record consists of a docid followed by a position-list and |
+** a doclist consists of one or more document records. |
+** |
+** A bare doclist omits the position information, becoming an |
+** array of varint-encoded docids. |
+** |
+**** Segment leaf nodes **** |
+** Segment leaf nodes store terms and doclists, ordered by term. Leaf |
+** nodes are written using LeafWriter, and read using LeafReader (to |
+** iterate through a single leaf node's data) and LeavesReader (to |
+** iterate through a segment's entire leaf layer). Leaf nodes have |
+** the format: |
+** |
+** varint iHeight; (height from leaf level, always 0) |
+** varint nTerm; (length of first term) |
+** char pTerm[nTerm]; (content of first term) |
+** varint nDoclist; (length of term's associated doclist) |
+** char pDoclist[nDoclist]; (content of doclist) |
+** array { |
+** (further terms are delta-encoded) |
+** varint nPrefix; (length of prefix shared with previous term) |
+** varint nSuffix; (length of unshared suffix) |
+** char pTermSuffix[nSuffix];(unshared suffix of next term) |
+** varint nDoclist; (length of term's associated doclist) |
+** char pDoclist[nDoclist]; (content of doclist) |
+** } |
+** |
+** Here, array { X } means zero or more occurrences of X, adjacent in |
+** memory. |
+** |
+** Leaf nodes are broken into blocks which are stored contiguously in |
+** the %_segments table in sorted order. This means that when the end |
+** of a node is reached, the next term is in the node with the next |
+** greater node id. |
+** |
+** New data is spilled to a new leaf node when the current node |
+** exceeds LEAF_MAX bytes (default 2048). New data which itself is |
+** larger than STANDALONE_MIN (default 1024) is placed in a standalone |
+** node (a leaf node with a single term and doclist). The goal of |
+** these settings is to pack together groups of small doclists while |
+** making it efficient to directly access large doclists. The |
+** assumption is that large doclists represent terms which are more |
+** likely to be query targets. |
+** |
+** TODO(shess) It may be useful for blocking decisions to be more |
+** dynamic. For instance, it may make more sense to have a 2.5k leaf |
+** node rather than splitting into 2k and .5k nodes. My intuition is |
+** that this might extend through 2x or 4x the pagesize. |
+** |
+** |
+**** Segment interior nodes **** |
+** Segment interior nodes store blockids for subtree nodes and terms |
+** to describe what data is stored by the each subtree. Interior |
+** nodes are written using InteriorWriter, and read using |
+** InteriorReader. InteriorWriters are created as needed when |
+** SegmentWriter creates new leaf nodes, or when an interior node |
+** itself grows too big and must be split. The format of interior |
+** nodes: |
+** |
+** varint iHeight; (height from leaf level, always >0) |
+** varint iBlockid; (block id of node's leftmost subtree) |
+** optional { |
+** varint nTerm; (length of first term) |
+** char pTerm[nTerm]; (content of first term) |
+** array { |
+** (further terms are delta-encoded) |
+** varint nPrefix; (length of shared prefix with previous term) |
+** varint nSuffix; (length of unshared suffix) |
+** char pTermSuffix[nSuffix]; (unshared suffix of next term) |
+** } |
+** } |
+** |
+** Here, optional { X } means an optional element, while array { X } |
+** means zero or more occurrences of X, adjacent in memory. |
+** |
+** An interior node encodes n terms separating n+1 subtrees. The |
+** subtree blocks are contiguous, so only the first subtree's blockid |
+** is encoded. The subtree at iBlockid will contain all terms less |
+** than the first term encoded (or all terms if no term is encoded). |
+** Otherwise, for terms greater than or equal to pTerm[i] but less |
+** than pTerm[i+1], the subtree for that term will be rooted at |
+** iBlockid+i. Interior nodes only store enough term data to |
+** distinguish adjacent children (if the rightmost term of the left |
+** child is "something", and the leftmost term of the right child is |
+** "wicked", only "w" is stored). |
+** |
+** New data is spilled to a new interior node at the same height when |
+** the current node exceeds INTERIOR_MAX bytes (default 2048). |
+** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing |
+** interior nodes and making the tree too skinny. The interior nodes |
+** at a given height are naturally tracked by interior nodes at |
+** height+1, and so on. |
+** |
+** |
+**** Segment directory **** |
+** The segment directory in table %_segdir stores meta-information for |
+** merging and deleting segments, and also the root node of the |
+** segment's tree. |
+** |
+** The root node is the top node of the segment's tree after encoding |
+** the entire segment, restricted to ROOT_MAX bytes (default 1024). |
+** This could be either a leaf node or an interior node. If the top |
+** node requires more than ROOT_MAX bytes, it is flushed to %_segments |
+** and a new root interior node is generated (which should always fit |
+** within ROOT_MAX because it only needs space for 2 varints, the |
+** height and the blockid of the previous root). |
+** |
+** The meta-information in the segment directory is: |
+** level - segment level (see below) |
+** idx - index within level |
+** - (level,idx uniquely identify a segment) |
+** start_block - first leaf node |
+** leaves_end_block - last leaf node |
+** end_block - last block (including interior nodes) |
+** root - contents of root node |
+** |
+** If the root node is a leaf node, then start_block, |
+** leaves_end_block, and end_block are all 0. |
+** |
+** |
+**** Segment merging **** |
+** To amortize update costs, segments are grouped into levels and |
+** merged in batches. Each increase in level represents exponentially |
+** more documents. |
+** |
+** New documents (actually, document updates) are tokenized and |
+** written individually (using LeafWriter) to a level 0 segment, with |
+** incrementing idx. When idx reaches MERGE_COUNT (default 16), all |
+** level 0 segments are merged into a single level 1 segment. Level 1 |
+** is populated like level 0, and eventually MERGE_COUNT level 1 |
+** segments are merged to a single level 2 segment (representing |
+** MERGE_COUNT^2 updates), and so on. |
+** |
+** A segment merge traverses all segments at a given level in |
+** parallel, performing a straightforward sorted merge. Since segment |
+** leaf nodes are written in to the %_segments table in order, this |
+** merge traverses the underlying sqlite disk structures efficiently. |
+** After the merge, all segment blocks from the merged level are |
+** deleted. |
+** |
+** MERGE_COUNT controls how often we merge segments. 16 seems to be |
+** somewhat of a sweet spot for insertion performance. 32 and 64 show |
+** very similar performance numbers to 16 on insertion, though they're |
+** a tiny bit slower (perhaps due to more overhead in merge-time |
+** sorting). 8 is about 20% slower than 16, 4 about 50% slower than |
+** 16, 2 about 66% slower than 16. |
+** |
+** At query time, high MERGE_COUNT increases the number of segments |
+** which need to be scanned and merged. For instance, with 100k docs |
+** inserted: |
+** |
+** MERGE_COUNT segments |
+** 16 25 |
+** 8 12 |
+** 4 10 |
+** 2 6 |
+** |
+** This appears to have only a moderate impact on queries for very |
+** frequent terms (which are somewhat dominated by segment merge |
+** costs), and infrequent and non-existent terms still seem to be fast |
+** even with many segments. |
+** |
+** TODO(shess) That said, it would be nice to have a better query-side |
+** argument for MERGE_COUNT of 16. Also, it is possible/likely that |
+** optimizations to things like doclist merging will swing the sweet |
+** spot around. |
+** |
+** |
+** |
+**** Handling of deletions and updates **** |
+** Since we're using a segmented structure, with no docid-oriented |
+** index into the term index, we clearly cannot simply update the term |
+** index when a document is deleted or updated. For deletions, we |
+** write an empty doclist (varint(docid) varint(POS_END)), for updates |
+** we simply write the new doclist. Segment merges overwrite older |
+** data for a particular docid with newer data, so deletes or updates |
+** will eventually overtake the earlier data and knock it out. The |
+** query logic likewise merges doclists so that newer data knocks out |
+** older data. |
+*/ |
+#define CHROMIUM_FTS3_CHANGES 1 |
+ |
+/************** Include fts3Int.h in the middle of fts3.c ********************/ |
+/************** Begin file fts3Int.h *****************************************/ |
+/* |
+** 2009 Nov 12 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+*/ |
+#ifndef _FTSINT_H |
+#define _FTSINT_H |
+ |
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) |
+# define NDEBUG 1 |
+#endif |
+ |
+/* FTS3/FTS4 require virtual tables */ |
+#ifdef SQLITE_OMIT_VIRTUALTABLE |
+# undef SQLITE_ENABLE_FTS3 |
+# undef SQLITE_ENABLE_FTS4 |
+#endif |
+ |
+/* |
+** FTS4 is really an extension for FTS3. It is enabled using the |
+** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all |
+** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. |
+*/ |
+#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) |
+# define SQLITE_ENABLE_FTS3 |
+#endif |
+ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* If not building as part of the core, include sqlite3ext.h. */ |
+#ifndef SQLITE_CORE |
+/* # include "sqlite3ext.h" */ |
+SQLITE_EXTENSION_INIT3 |
+#endif |
+ |
+/* #include "sqlite3.h" */ |
+/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ |
+/************** Begin file fts3_tokenizer.h **********************************/ |
+/* |
+** 2006 July 10 |
+** |
+** The author disclaims copyright to this source code. |
+** |
+************************************************************************* |
+** Defines the interface to tokenizers used by fulltext-search. There |
+** are three basic components: |
+** |
+** sqlite3_tokenizer_module is a singleton defining the tokenizer |
+** interface functions. This is essentially the class structure for |
+** tokenizers. |
+** |
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps |
+** including customization information defined at creation time. |
+** |
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate |
+** tokens from a particular input. |
+*/ |
+#ifndef _FTS3_TOKENIZER_H_ |
+#define _FTS3_TOKENIZER_H_ |
+ |
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. |
+** If tokenizers are to be allowed to call sqlite3_*() functions, then |
+** we will need a way to register the API consistently. |
+*/ |
+/* #include "sqlite3.h" */ |
+ |
+/* |
+** Structures used by the tokenizer interface. When a new tokenizer |
+** implementation is registered, the caller provides a pointer to |
+** an sqlite3_tokenizer_module containing pointers to the callback |
+** functions that make up an implementation. |
+** |
+** When an fts3 table is created, it passes any arguments passed to |
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the |
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer |
+** implementation. The xCreate() function in turn returns an |
+** sqlite3_tokenizer structure representing the specific tokenizer to |
+** be used for the fts3 table (customized by the tokenizer clause arguments). |
+** |
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() |
+** method is called. It returns an sqlite3_tokenizer_cursor object |
+** that may be used to tokenize a specific input buffer based on |
+** the tokenization rules supplied by a specific sqlite3_tokenizer |
+** object. |
+*/ |
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; |
+typedef struct sqlite3_tokenizer sqlite3_tokenizer; |
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; |
+ |
+struct sqlite3_tokenizer_module { |
+ |
+ /* |
+ ** Structure version. Should always be set to 0 or 1. |
+ */ |
+ int iVersion; |
+ |
+ /* |
+ ** Create a new tokenizer. The values in the argv[] array are the |
+ ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL |
+ ** TABLE statement that created the fts3 table. For example, if |
+ ** the following SQL is executed: |
+ ** |
+ ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2) |
+ ** |
+ ** then argc is set to 2, and the argv[] array contains pointers |
+ ** to the strings "arg1" and "arg2". |
+ ** |
+ ** This method should return either SQLITE_OK (0), or an SQLite error |
+ ** code. If SQLITE_OK is returned, then *ppTokenizer should be set |
+ ** to point at the newly created tokenizer structure. The generic |
+ ** sqlite3_tokenizer.pModule variable should not be initialized by |
+ ** this callback. The caller will do so. |
+ */ |
+ int (*xCreate)( |
+ int argc, /* Size of argv array */ |
+ const char *const*argv, /* Tokenizer argument strings */ |
+ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ |
+ ); |
+ |
+ /* |
+ ** Destroy an existing tokenizer. The fts3 module calls this method |
+ ** exactly once for each successful call to xCreate(). |
+ */ |
+ int (*xDestroy)(sqlite3_tokenizer *pTokenizer); |
+ |
+ /* |
+ ** Create a tokenizer cursor to tokenize an input buffer. The caller |
+ ** is responsible for ensuring that the input buffer remains valid |
+ ** until the cursor is closed (using the xClose() method). |
+ */ |
+ int (*xOpen)( |
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ |
+ const char *pInput, int nBytes, /* Input buffer */ |
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ |
+ ); |
+ |
+ /* |
+ ** Destroy an existing tokenizer cursor. The fts3 module calls this |
+ ** method exactly once for each successful call to xOpen(). |
+ */ |
+ int (*xClose)(sqlite3_tokenizer_cursor *pCursor); |
+ |
+ /* |
+ ** Retrieve the next token from the tokenizer cursor pCursor. This |
+ ** method should either return SQLITE_OK and set the values of the |
+ ** "OUT" variables identified below, or SQLITE_DONE to indicate that |
+ ** the end of the buffer has been reached, or an SQLite error code. |
+ ** |
+ ** *ppToken should be set to point at a buffer containing the |
+ ** normalized version of the token (i.e. after any case-folding and/or |
+ ** stemming has been performed). *pnBytes should be set to the length |
+ ** of this buffer in bytes. The input text that generated the token is |
+ ** identified by the byte offsets returned in *piStartOffset and |
+ ** *piEndOffset. *piStartOffset should be set to the index of the first |
+ ** byte of the token in the input buffer. *piEndOffset should be set |
+ ** to the index of the first byte just past the end of the token in |
+ ** the input buffer. |
+ ** |
+ ** The buffer *ppToken is set to point at is managed by the tokenizer |
+ ** implementation. It is only required to be valid until the next call |
+ ** to xNext() or xClose(). |
+ */ |
+ /* TODO(shess) current implementation requires pInput to be |
+ ** nul-terminated. This should either be fixed, or pInput/nBytes |
+ ** should be converted to zInput. |
+ */ |
+ int (*xNext)( |
+ sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ |
+ const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ |
+ int *piStartOffset, /* OUT: Byte offset of token in input buffer */ |
+ int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ |
+ int *piPosition /* OUT: Number of tokens returned before this one */ |
+ ); |
+ |
+ /*********************************************************************** |
+ ** Methods below this point are only available if iVersion>=1. |
+ */ |
+ |
+ /* |
+ ** Configure the language id of a tokenizer cursor. |
+ */ |
+ int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); |
+}; |
+ |
+struct sqlite3_tokenizer { |
+ const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ |
+ /* Tokenizer implementations will typically add additional fields */ |
+}; |
+ |
+struct sqlite3_tokenizer_cursor { |
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ |
+ /* Tokenizer implementations will typically add additional fields */ |
+}; |
+ |
+int fts3_global_term_cnt(int iTerm, int iCol); |
+int fts3_term_cnt(int iTerm, int iCol); |
+ |
+ |
+#endif /* _FTS3_TOKENIZER_H_ */ |
+ |
+/************** End of fts3_tokenizer.h **************************************/ |
+/************** Continuing where we left off in fts3Int.h ********************/ |
+/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ |
+/************** Begin file fts3_hash.h ***************************************/ |
+/* |
+** 2001 September 22 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+************************************************************************* |
+** This is the header file for the generic hash-table implementation |
+** used in SQLite. We've modified it slightly to serve as a standalone |
+** hash table implementation for the full-text indexing module. |
+** |
+*/ |
+#ifndef _FTS3_HASH_H_ |
+#define _FTS3_HASH_H_ |
+ |
+/* Forward declarations of structures. */ |
+typedef struct Fts3Hash Fts3Hash; |
+typedef struct Fts3HashElem Fts3HashElem; |
+ |
+/* A complete hash table is an instance of the following structure. |
+** The internals of this structure are intended to be opaque -- client |
+** code should not attempt to access or modify the fields of this structure |
+** directly. Change this structure only by using the routines below. |
+** However, many of the "procedures" and "functions" for modifying and |
+** accessing this structure are really macros, so we can't really make |
+** this structure opaque. |
+*/ |
+struct Fts3Hash { |
+ char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ |
+ char copyKey; /* True if copy of key made on insert */ |
+ int count; /* Number of entries in this table */ |
+ Fts3HashElem *first; /* The first element of the array */ |
+ int htsize; /* Number of buckets in the hash table */ |
+ struct _fts3ht { /* the hash table */ |
+ int count; /* Number of entries with this hash */ |
+ Fts3HashElem *chain; /* Pointer to first entry with this hash */ |
+ } *ht; |
+}; |
+ |
+/* Each element in the hash table is an instance of the following |
+** structure. All elements are stored on a single doubly-linked list. |
+** |
+** Again, this structure is intended to be opaque, but it can't really |
+** be opaque because it is used by macros. |
+*/ |
+struct Fts3HashElem { |
+ Fts3HashElem *next, *prev; /* Next and previous elements in the table */ |
+ void *data; /* Data associated with this element */ |
+ void *pKey; int nKey; /* Key associated with this element */ |
+}; |
+ |
+/* |
+** There are 2 different modes of operation for a hash table: |
+** |
+** FTS3_HASH_STRING pKey points to a string that is nKey bytes long |
+** (including the null-terminator, if any). Case |
+** is respected in comparisons. |
+** |
+** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. |
+** memcmp() is used to compare keys. |
+** |
+** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. |
+*/ |
+#define FTS3_HASH_STRING 1 |
+#define FTS3_HASH_BINARY 2 |
+ |
+/* |
+** Access routines. To delete, insert a NULL pointer. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); |
+SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); |
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); |
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); |
+SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); |
+ |
+/* |
+** Shorthand for the functions above |
+*/ |
+#define fts3HashInit sqlite3Fts3HashInit |
+#define fts3HashInsert sqlite3Fts3HashInsert |
+#define fts3HashFind sqlite3Fts3HashFind |
+#define fts3HashClear sqlite3Fts3HashClear |
+#define fts3HashFindElem sqlite3Fts3HashFindElem |
+ |
+/* |
+** Macros for looping over all elements of a hash table. The idiom is |
+** like this: |
+** |
+** Fts3Hash h; |
+** Fts3HashElem *p; |
+** ... |
+** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ |
+** SomeStructure *pData = fts3HashData(p); |
+** // do something with pData |
+** } |
+*/ |
+#define fts3HashFirst(H) ((H)->first) |
+#define fts3HashNext(E) ((E)->next) |
+#define fts3HashData(E) ((E)->data) |
+#define fts3HashKey(E) ((E)->pKey) |
+#define fts3HashKeysize(E) ((E)->nKey) |
+ |
+/* |
+** Number of entries in a hash table |
+*/ |
+#define fts3HashCount(H) ((H)->count) |
+ |
+#endif /* _FTS3_HASH_H_ */ |
+ |
+/************** End of fts3_hash.h *******************************************/ |
+/************** Continuing where we left off in fts3Int.h ********************/ |
+ |
+/* |
+** This constant determines the maximum depth of an FTS expression tree |
+** that the library will create and use. FTS uses recursion to perform |
+** various operations on the query tree, so the disadvantage of a large |
+** limit is that it may allow very large queries to use large amounts |
+** of stack space (perhaps causing a stack overflow). |
+*/ |
+#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH |
+# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 |
+#endif |
+ |
+ |
+/* |
+** This constant controls how often segments are merged. Once there are |
+** FTS3_MERGE_COUNT segments of level N, they are merged into a single |
+** segment of level N+1. |
+*/ |
+#define FTS3_MERGE_COUNT 16 |
+ |
+/* |
+** This is the maximum amount of data (in bytes) to store in the |
+** Fts3Table.pendingTerms hash table. Normally, the hash table is |
+** populated as documents are inserted/updated/deleted in a transaction |
+** and used to create a new segment when the transaction is committed. |
+** However if this limit is reached midway through a transaction, a new |
+** segment is created and the hash table cleared immediately. |
+*/ |
+#define FTS3_MAX_PENDING_DATA (1*1024*1024) |
+ |
+/* |
+** Macro to return the number of elements in an array. SQLite has a |
+** similar macro called ArraySize(). Use a different name to avoid |
+** a collision when building an amalgamation with built-in FTS3. |
+*/ |
+#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) |
+ |
+ |
+#ifndef MIN |
+# define MIN(x,y) ((x)<(y)?(x):(y)) |
+#endif |
+#ifndef MAX |
+# define MAX(x,y) ((x)>(y)?(x):(y)) |
+#endif |
+ |
+/* |
+** Maximum length of a varint encoded integer. The varint format is different |
+** from that used by SQLite, so the maximum length is 10, not 9. |
+*/ |
+#define FTS3_VARINT_MAX 10 |
+ |
+/* |
+** FTS4 virtual tables may maintain multiple indexes - one index of all terms |
+** in the document set and zero or more prefix indexes. All indexes are stored |
+** as one or more b+-trees in the %_segments and %_segdir tables. |
+** |
+** It is possible to determine which index a b+-tree belongs to based on the |
+** value stored in the "%_segdir.level" column. Given this value L, the index |
+** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with |
+** level values between 0 and 1023 (inclusive) belong to index 0, all levels |
+** between 1024 and 2047 to index 1, and so on. |
+** |
+** It is considered impossible for an index to use more than 1024 levels. In |
+** theory though this may happen, but only after at least |
+** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. |
+*/ |
+#define FTS3_SEGDIR_MAXLEVEL 1024 |
+#define FTS3_SEGDIR_MAXLEVEL_STR "1024" |
+ |
+/* |
+** The testcase() macro is only used by the amalgamation. If undefined, |
+** make it a no-op. |
+*/ |
+#ifndef testcase |
+# define testcase(X) |
+#endif |
+ |
+/* |
+** Terminator values for position-lists and column-lists. |
+*/ |
+#define POS_COLUMN (1) /* Column-list terminator */ |
+#define POS_END (0) /* Position-list terminator */ |
+ |
+/* |
+** This section provides definitions to allow the |
+** FTS3 extension to be compiled outside of the |
+** amalgamation. |
+*/ |
+#ifndef SQLITE_AMALGAMATION |
+/* |
+** Macros indicating that conditional expressions are always true or |
+** false. |
+*/ |
+#ifdef SQLITE_COVERAGE_TEST |
+# define ALWAYS(x) (1) |
+# define NEVER(X) (0) |
+#elif defined(SQLITE_DEBUG) |
+# define ALWAYS(x) sqlite3Fts3Always((x)!=0) |
+# define NEVER(x) sqlite3Fts3Never((x)!=0) |
+SQLITE_PRIVATE int sqlite3Fts3Always(int b); |
+SQLITE_PRIVATE int sqlite3Fts3Never(int b); |
+#else |
+# define ALWAYS(x) (x) |
+# define NEVER(x) (x) |
+#endif |
+ |
+/* |
+** Internal types used by SQLite. |
+*/ |
+typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ |
+typedef short int i16; /* 2-byte (or larger) signed integer */ |
+typedef unsigned int u32; /* 4-byte unsigned integer */ |
+typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ |
+typedef sqlite3_int64 i64; /* 8-byte signed integer */ |
+ |
+/* |
+** Macro used to suppress compiler warnings for unused parameters. |
+*/ |
+#define UNUSED_PARAMETER(x) (void)(x) |
+ |
+/* |
+** Activate assert() only if SQLITE_TEST is enabled. |
+*/ |
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) |
+# define NDEBUG 1 |
+#endif |
+ |
+/* |
+** The TESTONLY macro is used to enclose variable declarations or |
+** other bits of code that are needed to support the arguments |
+** within testcase() and assert() macros. |
+*/ |
+#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) |
+# define TESTONLY(X) X |
+#else |
+# define TESTONLY(X) |
+#endif |
+ |
+#endif /* SQLITE_AMALGAMATION */ |
+ |
+#ifdef SQLITE_DEBUG |
+SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); |
+# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() |
+#else |
+# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB |
+#endif |
+ |
+typedef struct Fts3Table Fts3Table; |
+typedef struct Fts3Cursor Fts3Cursor; |
+typedef struct Fts3Expr Fts3Expr; |
+typedef struct Fts3Phrase Fts3Phrase; |
+typedef struct Fts3PhraseToken Fts3PhraseToken; |
+ |
+typedef struct Fts3Doclist Fts3Doclist; |
+typedef struct Fts3SegFilter Fts3SegFilter; |
+typedef struct Fts3DeferredToken Fts3DeferredToken; |
+typedef struct Fts3SegReader Fts3SegReader; |
+typedef struct Fts3MultiSegReader Fts3MultiSegReader; |
+ |
+typedef struct MatchinfoBuffer MatchinfoBuffer; |
+ |
+/* |
+** A connection to a fulltext index is an instance of the following |
+** structure. The xCreate and xConnect methods create an instance |
+** of this structure and xDestroy and xDisconnect free that instance. |
+** All other methods receive a pointer to the structure as one of their |
+** arguments. |
+*/ |
+struct Fts3Table { |
+ sqlite3_vtab base; /* Base class used by SQLite core */ |
+ sqlite3 *db; /* The database connection */ |
+ const char *zDb; /* logical database name */ |
+ const char *zName; /* virtual table name */ |
+ int nColumn; /* number of named columns in virtual table */ |
+ char **azColumn; /* column names. malloced */ |
+ u8 *abNotindexed; /* True for 'notindexed' columns */ |
+ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ |
+ char *zContentTbl; /* content=xxx option, or NULL */ |
+ char *zLanguageid; /* languageid=xxx option, or NULL */ |
+ int nAutoincrmerge; /* Value configured by 'automerge' */ |
+ u32 nLeafAdd; /* Number of leaf blocks added this trans */ |
+ |
+ /* Precompiled statements used by the implementation. Each of these |
+ ** statements is run and reset within a single virtual table API call. |
+ */ |
+ sqlite3_stmt *aStmt[40]; |
+ sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ |
+ |
+ char *zReadExprlist; |
+ char *zWriteExprlist; |
+ |
+ int nNodeSize; /* Soft limit for node size */ |
+ u8 bFts4; /* True for FTS4, false for FTS3 */ |
+ u8 bHasStat; /* True if %_stat table exists (2==unknown) */ |
+ u8 bHasDocsize; /* True if %_docsize table exists */ |
+ u8 bDescIdx; /* True if doclists are in reverse order */ |
+ u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ |
+ int nPgsz; /* Page size for host database */ |
+ char *zSegmentsTbl; /* Name of %_segments table */ |
+ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ |
+ |
+ /* |
+ ** The following array of hash tables is used to buffer pending index |
+ ** updates during transactions. All pending updates buffered at any one |
+ ** time must share a common language-id (see the FTS4 langid= feature). |
+ ** The current language id is stored in variable iPrevLangid. |
+ ** |
+ ** A single FTS4 table may have multiple full-text indexes. For each index |
+ ** there is an entry in the aIndex[] array. Index 0 is an index of all the |
+ ** terms that appear in the document set. Each subsequent index in aIndex[] |
+ ** is an index of prefixes of a specific length. |
+ ** |
+ ** Variable nPendingData contains an estimate the memory consumed by the |
+ ** pending data structures, including hash table overhead, but not including |
+ ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash |
+ ** tables are flushed to disk. Variable iPrevDocid is the docid of the most |
+ ** recently inserted record. |
+ */ |
+ int nIndex; /* Size of aIndex[] */ |
+ struct Fts3Index { |
+ int nPrefix; /* Prefix length (0 for main terms index) */ |
+ Fts3Hash hPending; /* Pending terms table for this index */ |
+ } *aIndex; |
+ int nMaxPendingData; /* Max pending data before flush to disk */ |
+ int nPendingData; /* Current bytes of pending data */ |
+ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ |
+ int iPrevLangid; /* Langid of recently inserted document */ |
+ int bPrevDelete; /* True if last operation was a delete */ |
+ |
+#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) |
+ /* State variables used for validating that the transaction control |
+ ** methods of the virtual table are called at appropriate times. These |
+ ** values do not contribute to FTS functionality; they are used for |
+ ** verifying the operation of the SQLite core. |
+ */ |
+ int inTransaction; /* True after xBegin but before xCommit/xRollback */ |
+ int mxSavepoint; /* Largest valid xSavepoint integer */ |
+#endif |
+ |
+#ifdef SQLITE_TEST |
+ /* True to disable the incremental doclist optimization. This is controled |
+ ** by special insert command 'test-no-incr-doclist'. */ |
+ int bNoIncrDoclist; |
+#endif |
+}; |
+ |
+/* |
+** When the core wants to read from the virtual table, it creates a |
+** virtual table cursor (an instance of the following structure) using |
+** the xOpen method. Cursors are destroyed using the xClose method. |
+*/ |
+struct Fts3Cursor { |
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */ |
+ i16 eSearch; /* Search strategy (see below) */ |
+ u8 isEof; /* True if at End Of Results */ |
+ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ |
+ u8 bSeekStmt; /* True if pStmt is a seek */ |
+ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ |
+ Fts3Expr *pExpr; /* Parsed MATCH query string */ |
+ int iLangid; /* Language being queried for */ |
+ int nPhrase; /* Number of matchable phrases in query */ |
+ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ |
+ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ |
+ char *pNextId; /* Pointer into the body of aDoclist */ |
+ char *aDoclist; /* List of docids for full-text queries */ |
+ int nDoclist; /* Size of buffer at aDoclist */ |
+ u8 bDesc; /* True to sort in descending order */ |
+ int eEvalmode; /* An FTS3_EVAL_XX constant */ |
+ int nRowAvg; /* Average size of database rows, in pages */ |
+ sqlite3_int64 nDoc; /* Documents in table */ |
+ i64 iMinDocid; /* Minimum docid to return */ |
+ i64 iMaxDocid; /* Maximum docid to return */ |
+ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ |
+ MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ |
+}; |
+ |
+#define FTS3_EVAL_FILTER 0 |
+#define FTS3_EVAL_NEXT 1 |
+#define FTS3_EVAL_MATCHINFO 2 |
+ |
+/* |
+** The Fts3Cursor.eSearch member is always set to one of the following. |
+** Actualy, Fts3Cursor.eSearch can be greater than or equal to |
+** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index |
+** of the column to be searched. For example, in |
+** |
+** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); |
+** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; |
+** |
+** Because the LHS of the MATCH operator is 2nd column "b", |
+** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, |
+** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" |
+** indicating that all columns should be searched, |
+** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. |
+*/ |
+#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ |
+#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ |
+#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ |
+ |
+/* |
+** The lower 16-bits of the sqlite3_index_info.idxNum value set by |
+** the xBestIndex() method contains the Fts3Cursor.eSearch value described |
+** above. The upper 16-bits contain a combination of the following |
+** bits, used to describe extra constraints on full-text searches. |
+*/ |
+#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ |
+#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ |
+#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ |
+ |
+struct Fts3Doclist { |
+ char *aAll; /* Array containing doclist (or NULL) */ |
+ int nAll; /* Size of a[] in bytes */ |
+ char *pNextDocid; /* Pointer to next docid */ |
+ |
+ sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ |
+ int bFreeList; /* True if pList should be sqlite3_free()d */ |
+ char *pList; /* Pointer to position list following iDocid */ |
+ int nList; /* Length of position list */ |
+}; |
+ |
+/* |
+** A "phrase" is a sequence of one or more tokens that must match in |
+** sequence. A single token is the base case and the most common case. |
+** For a sequence of tokens contained in double-quotes (i.e. "one two three") |
+** nToken will be the number of tokens in the string. |
+*/ |
+struct Fts3PhraseToken { |
+ char *z; /* Text of the token */ |
+ int n; /* Number of bytes in buffer z */ |
+ int isPrefix; /* True if token ends with a "*" character */ |
+ int bFirst; /* True if token must appear at position 0 */ |
+ |
+ /* Variables above this point are populated when the expression is |
+ ** parsed (by code in fts3_expr.c). Below this point the variables are |
+ ** used when evaluating the expression. */ |
+ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ |
+ Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ |
+}; |
+ |
+struct Fts3Phrase { |
+ /* Cache of doclist for this phrase. */ |
+ Fts3Doclist doclist; |
+ int bIncr; /* True if doclist is loaded incrementally */ |
+ int iDoclistToken; |
+ |
+ /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an |
+ ** OR condition. */ |
+ char *pOrPoslist; |
+ i64 iOrDocid; |
+ |
+ /* Variables below this point are populated by fts3_expr.c when parsing |
+ ** a MATCH expression. Everything above is part of the evaluation phase. |
+ */ |
+ int nToken; /* Number of tokens in the phrase */ |
+ int iColumn; /* Index of column this phrase must match */ |
+ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ |
+}; |
+ |
+/* |
+** A tree of these objects forms the RHS of a MATCH operator. |
+** |
+** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist |
+** points to a malloced buffer, size nDoclist bytes, containing the results |
+** of this phrase query in FTS3 doclist format. As usual, the initial |
+** "Length" field found in doclists stored on disk is omitted from this |
+** buffer. |
+** |
+** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global |
+** matchinfo data. If it is not NULL, it points to an array of size nCol*3, |
+** where nCol is the number of columns in the queried FTS table. The array |
+** is populated as follows: |
+** |
+** aMI[iCol*3 + 0] = Undefined |
+** aMI[iCol*3 + 1] = Number of occurrences |
+** aMI[iCol*3 + 2] = Number of rows containing at least one instance |
+** |
+** The aMI array is allocated using sqlite3_malloc(). It should be freed |
+** when the expression node is. |
+*/ |
+struct Fts3Expr { |
+ int eType; /* One of the FTSQUERY_XXX values defined below */ |
+ int nNear; /* Valid if eType==FTSQUERY_NEAR */ |
+ Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ |
+ Fts3Expr *pLeft; /* Left operand */ |
+ Fts3Expr *pRight; /* Right operand */ |
+ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ |
+ |
+ /* The following are used by the fts3_eval.c module. */ |
+ sqlite3_int64 iDocid; /* Current docid */ |
+ u8 bEof; /* True this expression is at EOF already */ |
+ u8 bStart; /* True if iDocid is valid */ |
+ u8 bDeferred; /* True if this expression is entirely deferred */ |
+ |
+ /* The following are used by the fts3_snippet.c module. */ |
+ int iPhrase; /* Index of this phrase in matchinfo() results */ |
+ u32 *aMI; /* See above */ |
+}; |
+ |
+/* |
+** Candidate values for Fts3Query.eType. Note that the order of the first |
+** four values is in order of precedence when parsing expressions. For |
+** example, the following: |
+** |
+** "a OR b AND c NOT d NEAR e" |
+** |
+** is equivalent to: |
+** |
+** "a OR (b AND (c NOT (d NEAR e)))" |
+*/ |
+#define FTSQUERY_NEAR 1 |
+#define FTSQUERY_NOT 2 |
+#define FTSQUERY_AND 3 |
+#define FTSQUERY_OR 4 |
+#define FTSQUERY_PHRASE 5 |
+ |
+ |
+/* fts3_write.c */ |
+SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); |
+SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); |
+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); |
+SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, |
+ sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( |
+ Fts3Table*,int,const char*,int,int,Fts3SegReader**); |
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); |
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); |
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); |
+ |
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); |
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); |
+ |
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED |
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); |
+SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); |
+SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); |
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); |
+SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); |
+#else |
+# define sqlite3Fts3FreeDeferredTokens(x) |
+# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK |
+# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK |
+# define sqlite3Fts3FreeDeferredDoclists(x) |
+# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK |
+#endif |
+ |
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); |
+SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); |
+ |
+/* Special values interpreted by sqlite3SegReaderCursor() */ |
+#define FTS3_SEGCURSOR_PENDING -1 |
+#define FTS3_SEGCURSOR_ALL -2 |
+ |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); |
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); |
+ |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, |
+ int, int, int, const char *, int, int, int, Fts3MultiSegReader *); |
+ |
+/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ |
+#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 |
+#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 |
+#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 |
+#define FTS3_SEGMENT_PREFIX 0x00000008 |
+#define FTS3_SEGMENT_SCAN 0x00000010 |
+#define FTS3_SEGMENT_FIRST 0x00000020 |
+ |
+/* Type passed as 4th argument to SegmentReaderIterate() */ |
+struct Fts3SegFilter { |
+ const char *zTerm; |
+ int nTerm; |
+ int iCol; |
+ int flags; |
+}; |
+ |
+struct Fts3MultiSegReader { |
+ /* Used internally by sqlite3Fts3SegReaderXXX() calls */ |
+ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ |
+ int nSegment; /* Size of apSegment array */ |
+ int nAdvance; /* How many seg-readers to advance */ |
+ Fts3SegFilter *pFilter; /* Pointer to filter object */ |
+ char *aBuffer; /* Buffer to merge doclists in */ |
+ int nBuffer; /* Allocated size of aBuffer[] in bytes */ |
+ |
+ int iColFilter; /* If >=0, filter for this column */ |
+ int bRestart; |
+ |
+ /* Used by fts3.c only. */ |
+ int nCost; /* Cost of running iterator */ |
+ int bLookup; /* True if a lookup of a single entry. */ |
+ |
+ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ |
+ char *zTerm; /* Pointer to term buffer */ |
+ int nTerm; /* Size of zTerm in bytes */ |
+ char *aDoclist; /* Pointer to doclist buffer */ |
+ int nDoclist; /* Size of aDoclist[] in bytes */ |
+}; |
+ |
+SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); |
+ |
+#define fts3GetVarint32(p, piVal) ( \ |
+ (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ |
+) |
+ |
+/* fts3.c */ |
+SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); |
+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); |
+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); |
+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); |
+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); |
+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); |
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); |
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); |
+SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); |
+SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); |
+SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); |
+ |
+/* fts3_tokenizer.c */ |
+SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); |
+SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); |
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, |
+ sqlite3_tokenizer **, char ** |
+); |
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); |
+ |
+/* fts3_snippet.c */ |
+SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); |
+SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, |
+ const char *, const char *, int, int |
+); |
+SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); |
+SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); |
+ |
+/* fts3_expr.c */ |
+SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, |
+ char **, int, int, int, const char *, int, Fts3Expr **, char ** |
+); |
+SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); |
+#ifdef SQLITE_TEST |
+SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); |
+SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); |
+#endif |
+ |
+SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, |
+ sqlite3_tokenizer_cursor ** |
+); |
+ |
+/* fts3_aux.c */ |
+SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); |
+ |
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); |
+ |
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( |
+ Fts3Table*, Fts3MultiSegReader*, int, const char*, int); |
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( |
+ Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); |
+SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); |
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); |
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); |
+ |
+/* fts3_tokenize_vtab.c */ |
+SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); |
+ |
+/* fts3_unicode2.c (functions generated by parsing unicode text files) */ |
+#ifndef SQLITE_DISABLE_FTS3_UNICODE |
+SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); |
+SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); |
+SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); |
+#endif |
+ |
+#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ |
+#endif /* _FTSINT_H */ |
+ |
+/************** End of fts3Int.h *********************************************/ |
+/************** Continuing where we left off in fts3.c ***********************/ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) |
+# define SQLITE_CORE 1 |
+#endif |
+ |
+/* #include <assert.h> */ |
+/* #include <stdlib.h> */ |
+/* #include <stddef.h> */ |
+/* #include <stdio.h> */ |
+/* #include <string.h> */ |
+/* #include <stdarg.h> */ |
+ |
+/* #include "fts3.h" */ |
+#ifndef SQLITE_CORE |
+/* # include "sqlite3ext.h" */ |
+ SQLITE_EXTENSION_INIT1 |
+#endif |
+ |
+static int fts3EvalNext(Fts3Cursor *pCsr); |
+static int fts3EvalStart(Fts3Cursor *pCsr); |
+static int fts3TermSegReaderCursor( |
+ Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); |
+ |
+#ifndef SQLITE_AMALGAMATION |
+# if defined(SQLITE_DEBUG) |
+SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } |
+SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } |
+# endif |
+#endif |
+ |
+/* |
+** Write a 64-bit variable-length integer to memory starting at p[0]. |
+** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. |
+** The number of bytes written is returned. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ |
+ unsigned char *q = (unsigned char *) p; |
+ sqlite_uint64 vu = v; |
+ do{ |
+ *q++ = (unsigned char) ((vu & 0x7f) | 0x80); |
+ vu >>= 7; |
+ }while( vu!=0 ); |
+ q[-1] &= 0x7f; /* turn off high bit in final byte */ |
+ assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); |
+ return (int) (q - (unsigned char *)p); |
+} |
+ |
+#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ |
+ v = (v & mask1) | ( (*ptr++) << shift ); \ |
+ if( (v & mask2)==0 ){ var = v; return ret; } |
+#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ |
+ v = (*ptr++); \ |
+ if( (v & mask2)==0 ){ var = v; return ret; } |
+ |
+/* |
+** Read a 64-bit variable-length integer from memory starting at p[0]. |
+** Return the number of bytes read, or 0 on error. |
+** The value is stored in *v. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ |
+ const char *pStart = p; |
+ u32 a; |
+ u64 b; |
+ int shift; |
+ |
+ GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); |
+ GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); |
+ GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); |
+ GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); |
+ b = (a & 0x0FFFFFFF ); |
+ |
+ for(shift=28; shift<=63; shift+=7){ |
+ u64 c = *p++; |
+ b += (c&0x7F) << shift; |
+ if( (c & 0x80)==0 ) break; |
+ } |
+ *v = b; |
+ return (int)(p - pStart); |
+} |
+ |
+/* |
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a |
+** 32-bit integer before it is returned. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ |
+ u32 a; |
+ |
+#ifndef fts3GetVarint32 |
+ GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); |
+#else |
+ a = (*p++); |
+ assert( a & 0x80 ); |
+#endif |
+ |
+ GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); |
+ GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); |
+ GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); |
+ a = (a & 0x0FFFFFFF ); |
+ *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); |
+ return 5; |
+} |
+ |
+/* |
+** Return the number of bytes required to encode v as a varint |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ |
+ int i = 0; |
+ do{ |
+ i++; |
+ v >>= 7; |
+ }while( v!=0 ); |
+ return i; |
+} |
+ |
+/* |
+** Convert an SQL-style quoted string into a normal string by removing |
+** the quote characters. The conversion is done in-place. If the |
+** input does not begin with a quote character, then this routine |
+** is a no-op. |
+** |
+** Examples: |
+** |
+** "abc" becomes abc |
+** 'xyz' becomes xyz |
+** [pqr] becomes pqr |
+** `mno` becomes mno |
+** |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ |
+ char quote; /* Quote character (if any ) */ |
+ |
+ quote = z[0]; |
+ if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ |
+ int iIn = 1; /* Index of next byte to read from input */ |
+ int iOut = 0; /* Index of next byte to write to output */ |
+ |
+ /* If the first byte was a '[', then the close-quote character is a ']' */ |
+ if( quote=='[' ) quote = ']'; |
+ |
+ while( z[iIn] ){ |
+ if( z[iIn]==quote ){ |
+ if( z[iIn+1]!=quote ) break; |
+ z[iOut++] = quote; |
+ iIn += 2; |
+ }else{ |
+ z[iOut++] = z[iIn++]; |
+ } |
+ } |
+ z[iOut] = '\0'; |
+ } |
+} |
+ |
+/* |
+** Read a single varint from the doclist at *pp and advance *pp to point |
+** to the first byte past the end of the varint. Add the value of the varint |
+** to *pVal. |
+*/ |
+static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ |
+ sqlite3_int64 iVal; |
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal); |
+ *pVal += iVal; |
+} |
+ |
+/* |
+** When this function is called, *pp points to the first byte following a |
+** varint that is part of a doclist (or position-list, or any other list |
+** of varints). This function moves *pp to point to the start of that varint, |
+** and sets *pVal by the varint value. |
+** |
+** Argument pStart points to the first byte of the doclist that the |
+** varint is part of. |
+*/ |
+static void fts3GetReverseVarint( |
+ char **pp, |
+ char *pStart, |
+ sqlite3_int64 *pVal |
+){ |
+ sqlite3_int64 iVal; |
+ char *p; |
+ |
+ /* Pointer p now points at the first byte past the varint we are |
+ ** interested in. So, unless the doclist is corrupt, the 0x80 bit is |
+ ** clear on character p[-1]. */ |
+ for(p = (*pp)-2; p>=pStart && *p&0x80; p--); |
+ p++; |
+ *pp = p; |
+ |
+ sqlite3Fts3GetVarint(p, &iVal); |
+ *pVal = iVal; |
+} |
+ |
+/* |
+** The xDisconnect() virtual table method. |
+*/ |
+static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ |
+ Fts3Table *p = (Fts3Table *)pVtab; |
+ int i; |
+ |
+ assert( p->nPendingData==0 ); |
+ assert( p->pSegments==0 ); |
+ |
+ /* Free any prepared statements held */ |
+ sqlite3_finalize(p->pSeekStmt); |
+ for(i=0; i<SizeofArray(p->aStmt); i++){ |
+ sqlite3_finalize(p->aStmt[i]); |
+ } |
+ sqlite3_free(p->zSegmentsTbl); |
+ sqlite3_free(p->zReadExprlist); |
+ sqlite3_free(p->zWriteExprlist); |
+ sqlite3_free(p->zContentTbl); |
+ sqlite3_free(p->zLanguageid); |
+ |
+ /* Invoke the tokenizer destructor to free the tokenizer. */ |
+ p->pTokenizer->pModule->xDestroy(p->pTokenizer); |
+ |
+ sqlite3_free(p); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Write an error message into *pzErr |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ |
+ va_list ap; |
+ sqlite3_free(*pzErr); |
+ va_start(ap, zFormat); |
+ *pzErr = sqlite3_vmprintf(zFormat, ap); |
+ va_end(ap); |
+} |
+ |
+/* |
+** Construct one or more SQL statements from the format string given |
+** and then evaluate those statements. The success code is written |
+** into *pRc. |
+** |
+** If *pRc is initially non-zero then this routine is a no-op. |
+*/ |
+static void fts3DbExec( |
+ int *pRc, /* Success code */ |
+ sqlite3 *db, /* Database in which to run SQL */ |
+ const char *zFormat, /* Format string for SQL */ |
+ ... /* Arguments to the format string */ |
+){ |
+ va_list ap; |
+ char *zSql; |
+ if( *pRc ) return; |
+ va_start(ap, zFormat); |
+ zSql = sqlite3_vmprintf(zFormat, ap); |
+ va_end(ap); |
+ if( zSql==0 ){ |
+ *pRc = SQLITE_NOMEM; |
+ }else{ |
+ *pRc = sqlite3_exec(db, zSql, 0, 0, 0); |
+ sqlite3_free(zSql); |
+ } |
+} |
+ |
+/* |
+** The xDestroy() virtual table method. |
+*/ |
+static int fts3DestroyMethod(sqlite3_vtab *pVtab){ |
+ Fts3Table *p = (Fts3Table *)pVtab; |
+ int rc = SQLITE_OK; /* Return code */ |
+ const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ |
+ sqlite3 *db = p->db; /* Database handle */ |
+ |
+ /* Drop the shadow tables */ |
+ if( p->zContentTbl==0 ){ |
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); |
+ } |
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); |
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); |
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); |
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); |
+ |
+ /* If everything has worked, invoke fts3DisconnectMethod() to free the |
+ ** memory associated with the Fts3Table structure and return SQLITE_OK. |
+ ** Otherwise, return an SQLite error code. |
+ */ |
+ return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); |
+} |
+ |
+ |
+/* |
+** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table |
+** passed as the first argument. This is done as part of the xConnect() |
+** and xCreate() methods. |
+** |
+** If *pRc is non-zero when this function is called, it is a no-op. |
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc |
+** before returning. |
+*/ |
+static void fts3DeclareVtab(int *pRc, Fts3Table *p){ |
+ if( *pRc==SQLITE_OK ){ |
+ int i; /* Iterator variable */ |
+ int rc; /* Return code */ |
+ char *zSql; /* SQL statement passed to declare_vtab() */ |
+ char *zCols; /* List of user defined columns */ |
+ const char *zLanguageid; |
+ |
+ zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); |
+ sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); |
+ |
+ /* Create a list of user columns for the virtual table */ |
+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); |
+ for(i=1; zCols && i<p->nColumn; i++){ |
+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); |
+ } |
+ |
+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */ |
+ zSql = sqlite3_mprintf( |
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", |
+ zCols, p->zName, zLanguageid |
+ ); |
+ if( !zCols || !zSql ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_declare_vtab(p->db, zSql); |
+ } |
+ |
+ sqlite3_free(zSql); |
+ sqlite3_free(zCols); |
+ *pRc = rc; |
+ } |
+} |
+ |
+/* |
+** Create the %_stat table if it does not already exist. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ |
+ fts3DbExec(pRc, p->db, |
+ "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" |
+ "(id INTEGER PRIMARY KEY, value BLOB);", |
+ p->zDb, p->zName |
+ ); |
+ if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; |
+} |
+ |
+/* |
+** Create the backing store tables (%_content, %_segments and %_segdir) |
+** required by the FTS3 table passed as the only argument. This is done |
+** as part of the vtab xCreate() method. |
+** |
+** If the p->bHasDocsize boolean is true (indicating that this is an |
+** FTS4 table, not an FTS3 table) then also create the %_docsize and |
+** %_stat tables required by FTS4. |
+*/ |
+static int fts3CreateTables(Fts3Table *p){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int i; /* Iterator variable */ |
+ sqlite3 *db = p->db; /* The database connection */ |
+ |
+ if( p->zContentTbl==0 ){ |
+ const char *zLanguageid = p->zLanguageid; |
+ char *zContentCols; /* Columns of %_content table */ |
+ |
+ /* Create a list of user columns for the content table */ |
+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); |
+ for(i=0; zContentCols && i<p->nColumn; i++){ |
+ char *z = p->azColumn[i]; |
+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); |
+ } |
+ if( zLanguageid && zContentCols ){ |
+ zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); |
+ } |
+ if( zContentCols==0 ) rc = SQLITE_NOMEM; |
+ |
+ /* Create the content table */ |
+ fts3DbExec(&rc, db, |
+ "CREATE TABLE %Q.'%q_content'(%s)", |
+ p->zDb, p->zName, zContentCols |
+ ); |
+ sqlite3_free(zContentCols); |
+ } |
+ |
+ /* Create other tables */ |
+ fts3DbExec(&rc, db, |
+ "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", |
+ p->zDb, p->zName |
+ ); |
+ fts3DbExec(&rc, db, |
+ "CREATE TABLE %Q.'%q_segdir'(" |
+ "level INTEGER," |
+ "idx INTEGER," |
+ "start_block INTEGER," |
+ "leaves_end_block INTEGER," |
+ "end_block INTEGER," |
+ "root BLOB," |
+ "PRIMARY KEY(level, idx)" |
+ ");", |
+ p->zDb, p->zName |
+ ); |
+ if( p->bHasDocsize ){ |
+ fts3DbExec(&rc, db, |
+ "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", |
+ p->zDb, p->zName |
+ ); |
+ } |
+ assert( p->bHasStat==p->bFts4 ); |
+ if( p->bHasStat ){ |
+ sqlite3Fts3CreateStatTable(&rc, p); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Store the current database page-size in bytes in p->nPgsz. |
+** |
+** If *pRc is non-zero when this function is called, it is a no-op. |
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc |
+** before returning. |
+*/ |
+static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ |
+ if( *pRc==SQLITE_OK ){ |
+ int rc; /* Return code */ |
+ char *zSql; /* SQL text "PRAGMA %Q.page_size" */ |
+ sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ |
+ |
+ zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); |
+ if( !zSql ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_step(pStmt); |
+ p->nPgsz = sqlite3_column_int(pStmt, 0); |
+ rc = sqlite3_finalize(pStmt); |
+ }else if( rc==SQLITE_AUTH ){ |
+ p->nPgsz = 1024; |
+ rc = SQLITE_OK; |
+ } |
+ } |
+ assert( p->nPgsz>0 || rc!=SQLITE_OK ); |
+ sqlite3_free(zSql); |
+ *pRc = rc; |
+ } |
+} |
+ |
+/* |
+** "Special" FTS4 arguments are column specifications of the following form: |
+** |
+** <key> = <value> |
+** |
+** There may not be whitespace surrounding the "=" character. The <value> |
+** term may be quoted, but the <key> may not. |
+*/ |
+static int fts3IsSpecialColumn( |
+ const char *z, |
+ int *pnKey, |
+ char **pzValue |
+){ |
+ char *zValue; |
+ const char *zCsr = z; |
+ |
+ while( *zCsr!='=' ){ |
+ if( *zCsr=='\0' ) return 0; |
+ zCsr++; |
+ } |
+ |
+ *pnKey = (int)(zCsr-z); |
+ zValue = sqlite3_mprintf("%s", &zCsr[1]); |
+ if( zValue ){ |
+ sqlite3Fts3Dequote(zValue); |
+ } |
+ *pzValue = zValue; |
+ return 1; |
+} |
+ |
+/* |
+** Append the output of a printf() style formatting to an existing string. |
+*/ |
+static void fts3Appendf( |
+ int *pRc, /* IN/OUT: Error code */ |
+ char **pz, /* IN/OUT: Pointer to string buffer */ |
+ const char *zFormat, /* Printf format string to append */ |
+ ... /* Arguments for printf format string */ |
+){ |
+ if( *pRc==SQLITE_OK ){ |
+ va_list ap; |
+ char *z; |
+ va_start(ap, zFormat); |
+ z = sqlite3_vmprintf(zFormat, ap); |
+ va_end(ap); |
+ if( z && *pz ){ |
+ char *z2 = sqlite3_mprintf("%s%s", *pz, z); |
+ sqlite3_free(z); |
+ z = z2; |
+ } |
+ if( z==0 ) *pRc = SQLITE_NOMEM; |
+ sqlite3_free(*pz); |
+ *pz = z; |
+ } |
+} |
+ |
+/* |
+** Return a copy of input string zInput enclosed in double-quotes (") and |
+** with all double quote characters escaped. For example: |
+** |
+** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" |
+** |
+** The pointer returned points to memory obtained from sqlite3_malloc(). It |
+** is the callers responsibility to call sqlite3_free() to release this |
+** memory. |
+*/ |
+static char *fts3QuoteId(char const *zInput){ |
+ int nRet; |
+ char *zRet; |
+ nRet = 2 + (int)strlen(zInput)*2 + 1; |
+ zRet = sqlite3_malloc(nRet); |
+ if( zRet ){ |
+ int i; |
+ char *z = zRet; |
+ *(z++) = '"'; |
+ for(i=0; zInput[i]; i++){ |
+ if( zInput[i]=='"' ) *(z++) = '"'; |
+ *(z++) = zInput[i]; |
+ } |
+ *(z++) = '"'; |
+ *(z++) = '\0'; |
+ } |
+ return zRet; |
+} |
+ |
+/* |
+** Return a list of comma separated SQL expressions and a FROM clause that |
+** could be used in a SELECT statement such as the following: |
+** |
+** SELECT <list of expressions> FROM %_content AS x ... |
+** |
+** to return the docid, followed by each column of text data in order |
+** from left to write. If parameter zFunc is not NULL, then instead of |
+** being returned directly each column of text data is passed to an SQL |
+** function named zFunc first. For example, if zFunc is "unzip" and the |
+** table has the three user-defined columns "a", "b", and "c", the following |
+** string is returned: |
+** |
+** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" |
+** |
+** The pointer returned points to a buffer allocated by sqlite3_malloc(). It |
+** is the responsibility of the caller to eventually free it. |
+** |
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and |
+** a NULL pointer is returned). Otherwise, if an OOM error is encountered |
+** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If |
+** no error occurs, *pRc is left unmodified. |
+*/ |
+static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ |
+ char *zRet = 0; |
+ char *zFree = 0; |
+ char *zFunction; |
+ int i; |
+ |
+ if( p->zContentTbl==0 ){ |
+ if( !zFunc ){ |
+ zFunction = ""; |
+ }else{ |
+ zFree = zFunction = fts3QuoteId(zFunc); |
+ } |
+ fts3Appendf(pRc, &zRet, "docid"); |
+ for(i=0; i<p->nColumn; i++){ |
+ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); |
+ } |
+ if( p->zLanguageid ){ |
+ fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); |
+ } |
+ sqlite3_free(zFree); |
+ }else{ |
+ fts3Appendf(pRc, &zRet, "rowid"); |
+ for(i=0; i<p->nColumn; i++){ |
+ fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); |
+ } |
+ if( p->zLanguageid ){ |
+ fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); |
+ } |
+ } |
+ fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", |
+ p->zDb, |
+ (p->zContentTbl ? p->zContentTbl : p->zName), |
+ (p->zContentTbl ? "" : "_content") |
+ ); |
+ return zRet; |
+} |
+ |
+/* |
+** Return a list of N comma separated question marks, where N is the number |
+** of columns in the %_content table (one for the docid plus one for each |
+** user-defined text column). |
+** |
+** If argument zFunc is not NULL, then all but the first question mark |
+** is preceded by zFunc and an open bracket, and followed by a closed |
+** bracket. For example, if zFunc is "zip" and the FTS3 table has three |
+** user-defined text columns, the following string is returned: |
+** |
+** "?, zip(?), zip(?), zip(?)" |
+** |
+** The pointer returned points to a buffer allocated by sqlite3_malloc(). It |
+** is the responsibility of the caller to eventually free it. |
+** |
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and |
+** a NULL pointer is returned). Otherwise, if an OOM error is encountered |
+** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If |
+** no error occurs, *pRc is left unmodified. |
+*/ |
+static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ |
+ char *zRet = 0; |
+ char *zFree = 0; |
+ char *zFunction; |
+ int i; |
+ |
+ if( !zFunc ){ |
+ zFunction = ""; |
+ }else{ |
+ zFree = zFunction = fts3QuoteId(zFunc); |
+ } |
+ fts3Appendf(pRc, &zRet, "?"); |
+ for(i=0; i<p->nColumn; i++){ |
+ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); |
+ } |
+ if( p->zLanguageid ){ |
+ fts3Appendf(pRc, &zRet, ", ?"); |
+ } |
+ sqlite3_free(zFree); |
+ return zRet; |
+} |
+ |
+/* |
+** This function interprets the string at (*pp) as a non-negative integer |
+** value. It reads the integer and sets *pnOut to the value read, then |
+** sets *pp to point to the byte immediately following the last byte of |
+** the integer value. |
+** |
+** Only decimal digits ('0'..'9') may be part of an integer value. |
+** |
+** If *pp does not being with a decimal digit SQLITE_ERROR is returned and |
+** the output value undefined. Otherwise SQLITE_OK is returned. |
+** |
+** This function is used when parsing the "prefix=" FTS4 parameter. |
+*/ |
+static int fts3GobbleInt(const char **pp, int *pnOut){ |
+ const int MAX_NPREFIX = 10000000; |
+ const char *p; /* Iterator pointer */ |
+ int nInt = 0; /* Output value */ |
+ |
+ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ |
+ nInt = nInt * 10 + (p[0] - '0'); |
+ if( nInt>MAX_NPREFIX ){ |
+ nInt = 0; |
+ break; |
+ } |
+ } |
+ if( p==*pp ) return SQLITE_ERROR; |
+ *pnOut = nInt; |
+ *pp = p; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function is called to allocate an array of Fts3Index structures |
+** representing the indexes maintained by the current FTS table. FTS tables |
+** always maintain the main "terms" index, but may also maintain one or |
+** more "prefix" indexes, depending on the value of the "prefix=" parameter |
+** (if any) specified as part of the CREATE VIRTUAL TABLE statement. |
+** |
+** Argument zParam is passed the value of the "prefix=" option if one was |
+** specified, or NULL otherwise. |
+** |
+** If no error occurs, SQLITE_OK is returned and *apIndex set to point to |
+** the allocated array. *pnIndex is set to the number of elements in the |
+** array. If an error does occur, an SQLite error code is returned. |
+** |
+** Regardless of whether or not an error is returned, it is the responsibility |
+** of the caller to call sqlite3_free() on the output array to free it. |
+*/ |
+static int fts3PrefixParameter( |
+ const char *zParam, /* ABC in prefix=ABC parameter to parse */ |
+ int *pnIndex, /* OUT: size of *apIndex[] array */ |
+ struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ |
+){ |
+ struct Fts3Index *aIndex; /* Allocated array */ |
+ int nIndex = 1; /* Number of entries in array */ |
+ |
+ if( zParam && zParam[0] ){ |
+ const char *p; |
+ nIndex++; |
+ for(p=zParam; *p; p++){ |
+ if( *p==',' ) nIndex++; |
+ } |
+ } |
+ |
+ aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); |
+ *apIndex = aIndex; |
+ if( !aIndex ){ |
+ return SQLITE_NOMEM; |
+ } |
+ |
+ memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); |
+ if( zParam ){ |
+ const char *p = zParam; |
+ int i; |
+ for(i=1; i<nIndex; i++){ |
+ int nPrefix = 0; |
+ if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR; |
+ assert( nPrefix>=0 ); |
+ if( nPrefix==0 ){ |
+ nIndex--; |
+ i--; |
+ }else{ |
+ aIndex[i].nPrefix = nPrefix; |
+ } |
+ p++; |
+ } |
+ } |
+ |
+ *pnIndex = nIndex; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function is called when initializing an FTS4 table that uses the |
+** content=xxx option. It determines the number of and names of the columns |
+** of the new FTS4 table. |
+** |
+** The third argument passed to this function is the value passed to the |
+** config=xxx option (i.e. "xxx"). This function queries the database for |
+** a table of that name. If found, the output variables are populated |
+** as follows: |
+** |
+** *pnCol: Set to the number of columns table xxx has, |
+** |
+** *pnStr: Set to the total amount of space required to store a copy |
+** of each columns name, including the nul-terminator. |
+** |
+** *pazCol: Set to point to an array of *pnCol strings. Each string is |
+** the name of the corresponding column in table xxx. The array |
+** and its contents are allocated using a single allocation. It |
+** is the responsibility of the caller to free this allocation |
+** by eventually passing the *pazCol value to sqlite3_free(). |
+** |
+** If the table cannot be found, an error code is returned and the output |
+** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is |
+** returned (and the output variables are undefined). |
+*/ |
+static int fts3ContentColumns( |
+ sqlite3 *db, /* Database handle */ |
+ const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ |
+ const char *zTbl, /* Name of content table */ |
+ const char ***pazCol, /* OUT: Malloc'd array of column names */ |
+ int *pnCol, /* OUT: Size of array *pazCol */ |
+ int *pnStr, /* OUT: Bytes of string content */ |
+ char **pzErr /* OUT: error message */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ char *zSql; /* "SELECT *" statement on zTbl */ |
+ sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ |
+ |
+ zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); |
+ if( !zSql ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); |
+ } |
+ } |
+ sqlite3_free(zSql); |
+ |
+ if( rc==SQLITE_OK ){ |
+ const char **azCol; /* Output array */ |
+ int nStr = 0; /* Size of all column names (incl. 0x00) */ |
+ int nCol; /* Number of table columns */ |
+ int i; /* Used to iterate through columns */ |
+ |
+ /* Loop through the returned columns. Set nStr to the number of bytes of |
+ ** space required to store a copy of each column name, including the |
+ ** nul-terminator byte. */ |
+ nCol = sqlite3_column_count(pStmt); |
+ for(i=0; i<nCol; i++){ |
+ const char *zCol = sqlite3_column_name(pStmt, i); |
+ nStr += (int)strlen(zCol) + 1; |
+ } |
+ |
+ /* Allocate and populate the array to return. */ |
+ azCol = (const char **)sqlite3_malloc(sizeof(char *) * nCol + nStr); |
+ if( azCol==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ char *p = (char *)&azCol[nCol]; |
+ for(i=0; i<nCol; i++){ |
+ const char *zCol = sqlite3_column_name(pStmt, i); |
+ int n = (int)strlen(zCol)+1; |
+ memcpy(p, zCol, n); |
+ azCol[i] = p; |
+ p += n; |
+ } |
+ } |
+ sqlite3_finalize(pStmt); |
+ |
+ /* Set the output variables. */ |
+ *pnCol = nCol; |
+ *pnStr = nStr; |
+ *pazCol = azCol; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This function is the implementation of both the xConnect and xCreate |
+** methods of the FTS3 virtual table. |
+** |
+** The argv[] array contains the following: |
+** |
+** argv[0] -> module name ("fts3" or "fts4") |
+** argv[1] -> database name |
+** argv[2] -> table name |
+** argv[...] -> "column name" and other module argument fields. |
+*/ |
+static int fts3InitVtab( |
+ int isCreate, /* True for xCreate, false for xConnect */ |
+ sqlite3 *db, /* The SQLite database connection */ |
+ void *pAux, /* Hash table containing tokenizers */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ |
+ char **pzErr /* Write any error message here */ |
+){ |
+ Fts3Hash *pHash = (Fts3Hash *)pAux; |
+ Fts3Table *p = 0; /* Pointer to allocated vtab */ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int i; /* Iterator variable */ |
+ int nByte; /* Size of allocation used for *p */ |
+ int iCol; /* Column index */ |
+ int nString = 0; /* Bytes required to hold all column names */ |
+ int nCol = 0; /* Number of columns in the FTS table */ |
+ char *zCsr; /* Space for holding column names */ |
+ int nDb; /* Bytes required to hold database name */ |
+ int nName; /* Bytes required to hold table name */ |
+ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ |
+ const char **aCol; /* Array of column names */ |
+ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ |
+ |
+ int nIndex = 0; /* Size of aIndex[] array */ |
+ struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ |
+ |
+ /* The results of parsing supported FTS4 key=value options: */ |
+ int bNoDocsize = 0; /* True to omit %_docsize table */ |
+ int bDescIdx = 0; /* True to store descending indexes */ |
+ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ |
+ char *zCompress = 0; /* compress=? parameter (or NULL) */ |
+ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ |
+ char *zContent = 0; /* content=? parameter (or NULL) */ |
+ char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ |
+ char **azNotindexed = 0; /* The set of notindexed= columns */ |
+ int nNotindexed = 0; /* Size of azNotindexed[] array */ |
+ |
+ assert( strlen(argv[0])==4 ); |
+ assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) |
+ || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) |
+ ); |
+ |
+ nDb = (int)strlen(argv[1]) + 1; |
+ nName = (int)strlen(argv[2]) + 1; |
+ |
+ nByte = sizeof(const char *) * (argc-2); |
+ aCol = (const char **)sqlite3_malloc(nByte); |
+ if( aCol ){ |
+ memset((void*)aCol, 0, nByte); |
+ azNotindexed = (char **)sqlite3_malloc(nByte); |
+ } |
+ if( azNotindexed ){ |
+ memset(azNotindexed, 0, nByte); |
+ } |
+ if( !aCol || !azNotindexed ){ |
+ rc = SQLITE_NOMEM; |
+ goto fts3_init_out; |
+ } |
+ |
+ /* Loop through all of the arguments passed by the user to the FTS3/4 |
+ ** module (i.e. all the column names and special arguments). This loop |
+ ** does the following: |
+ ** |
+ ** + Figures out the number of columns the FTSX table will have, and |
+ ** the number of bytes of space that must be allocated to store copies |
+ ** of the column names. |
+ ** |
+ ** + If there is a tokenizer specification included in the arguments, |
+ ** initializes the tokenizer pTokenizer. |
+ */ |
+ for(i=3; rc==SQLITE_OK && i<argc; i++){ |
+ char const *z = argv[i]; |
+ int nKey; |
+ char *zVal; |
+ |
+ /* Check if this is a tokenizer specification */ |
+ if( !pTokenizer |
+ && strlen(z)>8 |
+ && 0==sqlite3_strnicmp(z, "tokenize", 8) |
+ && 0==sqlite3Fts3IsIdChar(z[8]) |
+ ){ |
+ rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); |
+ } |
+ |
+ /* Check if it is an FTS4 special argument. */ |
+ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ |
+ struct Fts4Option { |
+ const char *zOpt; |
+ int nOpt; |
+ } aFts4Opt[] = { |
+ { "matchinfo", 9 }, /* 0 -> MATCHINFO */ |
+ { "prefix", 6 }, /* 1 -> PREFIX */ |
+ { "compress", 8 }, /* 2 -> COMPRESS */ |
+ { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ |
+ { "order", 5 }, /* 4 -> ORDER */ |
+ { "content", 7 }, /* 5 -> CONTENT */ |
+ { "languageid", 10 }, /* 6 -> LANGUAGEID */ |
+ { "notindexed", 10 } /* 7 -> NOTINDEXED */ |
+ }; |
+ |
+ int iOpt; |
+ if( !zVal ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){ |
+ struct Fts4Option *pOp = &aFts4Opt[iOpt]; |
+ if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ |
+ break; |
+ } |
+ } |
+ if( iOpt==SizeofArray(aFts4Opt) ){ |
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ switch( iOpt ){ |
+ case 0: /* MATCHINFO */ |
+ if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ |
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); |
+ rc = SQLITE_ERROR; |
+ } |
+ bNoDocsize = 1; |
+ break; |
+ |
+ case 1: /* PREFIX */ |
+ sqlite3_free(zPrefix); |
+ zPrefix = zVal; |
+ zVal = 0; |
+ break; |
+ |
+ case 2: /* COMPRESS */ |
+ sqlite3_free(zCompress); |
+ zCompress = zVal; |
+ zVal = 0; |
+ break; |
+ |
+ case 3: /* UNCOMPRESS */ |
+ sqlite3_free(zUncompress); |
+ zUncompress = zVal; |
+ zVal = 0; |
+ break; |
+ |
+ case 4: /* ORDER */ |
+ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) |
+ && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) |
+ ){ |
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); |
+ rc = SQLITE_ERROR; |
+ } |
+ bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); |
+ break; |
+ |
+ case 5: /* CONTENT */ |
+ sqlite3_free(zContent); |
+ zContent = zVal; |
+ zVal = 0; |
+ break; |
+ |
+ case 6: /* LANGUAGEID */ |
+ assert( iOpt==6 ); |
+ sqlite3_free(zLanguageid); |
+ zLanguageid = zVal; |
+ zVal = 0; |
+ break; |
+ |
+ case 7: /* NOTINDEXED */ |
+ azNotindexed[nNotindexed++] = zVal; |
+ zVal = 0; |
+ break; |
+ } |
+ } |
+ sqlite3_free(zVal); |
+ } |
+ } |
+ |
+ /* Otherwise, the argument is a column name. */ |
+ else { |
+ nString += (int)(strlen(z) + 1); |
+ aCol[nCol++] = z; |
+ } |
+ } |
+ |
+ /* If a content=xxx option was specified, the following: |
+ ** |
+ ** 1. Ignore any compress= and uncompress= options. |
+ ** |
+ ** 2. If no column names were specified as part of the CREATE VIRTUAL |
+ ** TABLE statement, use all columns from the content table. |
+ */ |
+ if( rc==SQLITE_OK && zContent ){ |
+ sqlite3_free(zCompress); |
+ sqlite3_free(zUncompress); |
+ zCompress = 0; |
+ zUncompress = 0; |
+ if( nCol==0 ){ |
+ sqlite3_free((void*)aCol); |
+ aCol = 0; |
+ rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); |
+ |
+ /* If a languageid= option was specified, remove the language id |
+ ** column from the aCol[] array. */ |
+ if( rc==SQLITE_OK && zLanguageid ){ |
+ int j; |
+ for(j=0; j<nCol; j++){ |
+ if( sqlite3_stricmp(zLanguageid, aCol[j])==0 ){ |
+ int k; |
+ for(k=j; k<nCol; k++) aCol[k] = aCol[k+1]; |
+ nCol--; |
+ break; |
+ } |
+ } |
+ } |
+ } |
+ } |
+ if( rc!=SQLITE_OK ) goto fts3_init_out; |
+ |
+ if( nCol==0 ){ |
+ assert( nString==0 ); |
+ aCol[0] = "content"; |
+ nString = 8; |
+ nCol = 1; |
+ } |
+ |
+ if( pTokenizer==0 ){ |
+ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); |
+ if( rc!=SQLITE_OK ) goto fts3_init_out; |
+ } |
+ assert( pTokenizer ); |
+ |
+ rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); |
+ if( rc==SQLITE_ERROR ){ |
+ assert( zPrefix ); |
+ sqlite3Fts3ErrMsg(pzErr, "error parsing prefix parameter: %s", zPrefix); |
+ } |
+ if( rc!=SQLITE_OK ) goto fts3_init_out; |
+ |
+ /* Allocate and populate the Fts3Table structure. */ |
+ nByte = sizeof(Fts3Table) + /* Fts3Table */ |
+ nCol * sizeof(char *) + /* azColumn */ |
+ nIndex * sizeof(struct Fts3Index) + /* aIndex */ |
+ nCol * sizeof(u8) + /* abNotindexed */ |
+ nName + /* zName */ |
+ nDb + /* zDb */ |
+ nString; /* Space for azColumn strings */ |
+ p = (Fts3Table*)sqlite3_malloc(nByte); |
+ if( p==0 ){ |
+ rc = SQLITE_NOMEM; |
+ goto fts3_init_out; |
+ } |
+ memset(p, 0, nByte); |
+ p->db = db; |
+ p->nColumn = nCol; |
+ p->nPendingData = 0; |
+ p->azColumn = (char **)&p[1]; |
+ p->pTokenizer = pTokenizer; |
+ p->nMaxPendingData = FTS3_MAX_PENDING_DATA; |
+ p->bHasDocsize = (isFts4 && bNoDocsize==0); |
+ p->bHasStat = (u8)isFts4; |
+ p->bFts4 = (u8)isFts4; |
+ p->bDescIdx = (u8)bDescIdx; |
+ p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ |
+ p->zContentTbl = zContent; |
+ p->zLanguageid = zLanguageid; |
+ zContent = 0; |
+ zLanguageid = 0; |
+ TESTONLY( p->inTransaction = -1 ); |
+ TESTONLY( p->mxSavepoint = -1 ); |
+ |
+ p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; |
+ memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); |
+ p->nIndex = nIndex; |
+ for(i=0; i<nIndex; i++){ |
+ fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1); |
+ } |
+ p->abNotindexed = (u8 *)&p->aIndex[nIndex]; |
+ |
+ /* Fill in the zName and zDb fields of the vtab structure. */ |
+ zCsr = (char *)&p->abNotindexed[nCol]; |
+ p->zName = zCsr; |
+ memcpy(zCsr, argv[2], nName); |
+ zCsr += nName; |
+ p->zDb = zCsr; |
+ memcpy(zCsr, argv[1], nDb); |
+ zCsr += nDb; |
+ |
+ /* Fill in the azColumn array */ |
+ for(iCol=0; iCol<nCol; iCol++){ |
+ char *z; |
+ int n = 0; |
+ z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n); |
+ memcpy(zCsr, z, n); |
+ zCsr[n] = '\0'; |
+ sqlite3Fts3Dequote(zCsr); |
+ p->azColumn[iCol] = zCsr; |
+ zCsr += n+1; |
+ assert( zCsr <= &((char *)p)[nByte] ); |
+ } |
+ |
+ /* Fill in the abNotindexed array */ |
+ for(iCol=0; iCol<nCol; iCol++){ |
+ int n = (int)strlen(p->azColumn[iCol]); |
+ for(i=0; i<nNotindexed; i++){ |
+ char *zNot = azNotindexed[i]; |
+ if( zNot && n==(int)strlen(zNot) |
+ && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n) |
+ ){ |
+ p->abNotindexed[iCol] = 1; |
+ sqlite3_free(zNot); |
+ azNotindexed[i] = 0; |
+ } |
+ } |
+ } |
+ for(i=0; i<nNotindexed; i++){ |
+ if( azNotindexed[i] ){ |
+ sqlite3Fts3ErrMsg(pzErr, "no such column: %s", azNotindexed[i]); |
+ rc = SQLITE_ERROR; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK && (zCompress==0)!=(zUncompress==0) ){ |
+ char const *zMiss = (zCompress==0 ? "compress" : "uncompress"); |
+ rc = SQLITE_ERROR; |
+ sqlite3Fts3ErrMsg(pzErr, "missing %s parameter in fts4 constructor", zMiss); |
+ } |
+ p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc); |
+ p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); |
+ if( rc!=SQLITE_OK ) goto fts3_init_out; |
+ |
+ /* If this is an xCreate call, create the underlying tables in the |
+ ** database. TODO: For xConnect(), it could verify that said tables exist. |
+ */ |
+ if( isCreate ){ |
+ rc = fts3CreateTables(p); |
+ } |
+ |
+ /* Check to see if a legacy fts3 table has been "upgraded" by the |
+ ** addition of a %_stat table so that it can use incremental merge. |
+ */ |
+ if( !isFts4 && !isCreate ){ |
+ p->bHasStat = 2; |
+ } |
+ |
+ /* Figure out the page-size for the database. This is required in order to |
+ ** estimate the cost of loading large doclists from the database. */ |
+ fts3DatabasePageSize(&rc, p); |
+ p->nNodeSize = p->nPgsz-35; |
+ |
+ /* Declare the table schema to SQLite. */ |
+ fts3DeclareVtab(&rc, p); |
+ |
+fts3_init_out: |
+ sqlite3_free(zPrefix); |
+ sqlite3_free(aIndex); |
+ sqlite3_free(zCompress); |
+ sqlite3_free(zUncompress); |
+ sqlite3_free(zContent); |
+ sqlite3_free(zLanguageid); |
+ for(i=0; i<nNotindexed; i++) sqlite3_free(azNotindexed[i]); |
+ sqlite3_free((void *)aCol); |
+ sqlite3_free((void *)azNotindexed); |
+ if( rc!=SQLITE_OK ){ |
+ if( p ){ |
+ fts3DisconnectMethod((sqlite3_vtab *)p); |
+ }else if( pTokenizer ){ |
+ pTokenizer->pModule->xDestroy(pTokenizer); |
+ } |
+ }else{ |
+ assert( p->pSegments==0 ); |
+ *ppVTab = &p->base; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** The xConnect() and xCreate() methods for the virtual table. All the |
+** work is done in function fts3InitVtab(). |
+*/ |
+static int fts3ConnectMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pAux, /* Pointer to tokenizer hash table */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); |
+} |
+static int fts3CreateMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pAux, /* Pointer to tokenizer hash table */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); |
+} |
+ |
+/* |
+** Set the pIdxInfo->estimatedRows variable to nRow. Unless this |
+** extension is currently being used by a version of SQLite too old to |
+** support estimatedRows. In that case this function is a no-op. |
+*/ |
+static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ |
+#if SQLITE_VERSION_NUMBER>=3008002 |
+ if( sqlite3_libversion_number()>=3008002 ){ |
+ pIdxInfo->estimatedRows = nRow; |
+ } |
+#endif |
+} |
+ |
+/* |
+** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this |
+** extension is currently being used by a version of SQLite too old to |
+** support index-info flags. In that case this function is a no-op. |
+*/ |
+static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ |
+#if SQLITE_VERSION_NUMBER>=3008012 |
+ if( sqlite3_libversion_number()>=3008012 ){ |
+ pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; |
+ } |
+#endif |
+} |
+ |
+/* |
+** Implementation of the xBestIndex method for FTS3 tables. There |
+** are three possible strategies, in order of preference: |
+** |
+** 1. Direct lookup by rowid or docid. |
+** 2. Full-text search using a MATCH operator on a non-docid column. |
+** 3. Linear scan of %_content table. |
+*/ |
+static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ |
+ Fts3Table *p = (Fts3Table *)pVTab; |
+ int i; /* Iterator variable */ |
+ int iCons = -1; /* Index of constraint to use */ |
+ |
+ int iLangidCons = -1; /* Index of langid=x constraint, if present */ |
+ int iDocidGe = -1; /* Index of docid>=x constraint, if present */ |
+ int iDocidLe = -1; /* Index of docid<=x constraint, if present */ |
+ int iIdx; |
+ |
+ /* By default use a full table scan. This is an expensive option, |
+ ** so search through the constraints to see if a more efficient |
+ ** strategy is possible. |
+ */ |
+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; |
+ pInfo->estimatedCost = 5000000; |
+ for(i=0; i<pInfo->nConstraint; i++){ |
+ int bDocid; /* True if this constraint is on docid */ |
+ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; |
+ if( pCons->usable==0 ){ |
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ |
+ /* There exists an unusable MATCH constraint. This means that if |
+ ** the planner does elect to use the results of this call as part |
+ ** of the overall query plan the user will see an "unable to use |
+ ** function MATCH in the requested context" error. To discourage |
+ ** this, return a very high cost here. */ |
+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; |
+ pInfo->estimatedCost = 1e50; |
+ fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); |
+ return SQLITE_OK; |
+ } |
+ continue; |
+ } |
+ |
+ bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); |
+ |
+ /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ |
+ if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ |
+ pInfo->idxNum = FTS3_DOCID_SEARCH; |
+ pInfo->estimatedCost = 1.0; |
+ iCons = i; |
+ } |
+ |
+ /* A MATCH constraint. Use a full-text search. |
+ ** |
+ ** If there is more than one MATCH constraint available, use the first |
+ ** one encountered. If there is both a MATCH constraint and a direct |
+ ** rowid/docid lookup, prefer the MATCH strategy. This is done even |
+ ** though the rowid/docid lookup is faster than a MATCH query, selecting |
+ ** it would lead to an "unable to use function MATCH in the requested |
+ ** context" error. |
+ */ |
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH |
+ && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn |
+ ){ |
+ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; |
+ pInfo->estimatedCost = 2.0; |
+ iCons = i; |
+ } |
+ |
+ /* Equality constraint on the langid column */ |
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ |
+ && pCons->iColumn==p->nColumn + 2 |
+ ){ |
+ iLangidCons = i; |
+ } |
+ |
+ if( bDocid ){ |
+ switch( pCons->op ){ |
+ case SQLITE_INDEX_CONSTRAINT_GE: |
+ case SQLITE_INDEX_CONSTRAINT_GT: |
+ iDocidGe = i; |
+ break; |
+ |
+ case SQLITE_INDEX_CONSTRAINT_LE: |
+ case SQLITE_INDEX_CONSTRAINT_LT: |
+ iDocidLe = i; |
+ break; |
+ } |
+ } |
+ } |
+ |
+ /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ |
+ if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); |
+ |
+ iIdx = 1; |
+ if( iCons>=0 ){ |
+ pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; |
+ pInfo->aConstraintUsage[iCons].omit = 1; |
+ } |
+ if( iLangidCons>=0 ){ |
+ pInfo->idxNum |= FTS3_HAVE_LANGID; |
+ pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; |
+ } |
+ if( iDocidGe>=0 ){ |
+ pInfo->idxNum |= FTS3_HAVE_DOCID_GE; |
+ pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; |
+ } |
+ if( iDocidLe>=0 ){ |
+ pInfo->idxNum |= FTS3_HAVE_DOCID_LE; |
+ pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; |
+ } |
+ |
+ /* Regardless of the strategy selected, FTS can deliver rows in rowid (or |
+ ** docid) order. Both ascending and descending are possible. |
+ */ |
+ if( pInfo->nOrderBy==1 ){ |
+ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; |
+ if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ |
+ if( pOrder->desc ){ |
+ pInfo->idxStr = "DESC"; |
+ }else{ |
+ pInfo->idxStr = "ASC"; |
+ } |
+ pInfo->orderByConsumed = 1; |
+ } |
+ } |
+ |
+ assert( p->pSegments==0 ); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Implementation of xOpen method. |
+*/ |
+static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ |
+ sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ |
+ |
+ UNUSED_PARAMETER(pVTab); |
+ |
+ /* Allocate a buffer large enough for an Fts3Cursor structure. If the |
+ ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, |
+ ** if the allocation fails, return SQLITE_NOMEM. |
+ */ |
+ *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); |
+ if( !pCsr ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pCsr, 0, sizeof(Fts3Cursor)); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Finalize the statement handle at pCsr->pStmt. |
+** |
+** Or, if that statement handle is one created by fts3CursorSeekStmt(), |
+** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement |
+** pointer there instead of finalizing it. |
+*/ |
+static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ |
+ if( pCsr->bSeekStmt ){ |
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; |
+ if( p->pSeekStmt==0 ){ |
+ p->pSeekStmt = pCsr->pStmt; |
+ sqlite3_reset(pCsr->pStmt); |
+ pCsr->pStmt = 0; |
+ } |
+ pCsr->bSeekStmt = 0; |
+ } |
+ sqlite3_finalize(pCsr->pStmt); |
+} |
+ |
+/* |
+** Close the cursor. For additional information see the documentation |
+** on the xClose method of the virtual table interface. |
+*/ |
+static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; |
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); |
+ fts3CursorFinalizeStmt(pCsr); |
+ sqlite3Fts3ExprFree(pCsr->pExpr); |
+ sqlite3Fts3FreeDeferredTokens(pCsr); |
+ sqlite3_free(pCsr->aDoclist); |
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); |
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); |
+ sqlite3_free(pCsr); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then |
+** compose and prepare an SQL statement of the form: |
+** |
+** "SELECT <columns> FROM %_content WHERE rowid = ?" |
+** |
+** (or the equivalent for a content=xxx table) and set pCsr->pStmt to |
+** it. If an error occurs, return an SQLite error code. |
+*/ |
+static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ |
+ int rc = SQLITE_OK; |
+ if( pCsr->pStmt==0 ){ |
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; |
+ char *zSql; |
+ if( p->pSeekStmt ){ |
+ pCsr->pStmt = p->pSeekStmt; |
+ p->pSeekStmt = 0; |
+ }else{ |
+ zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); |
+ if( !zSql ) return SQLITE_NOMEM; |
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); |
+ sqlite3_free(zSql); |
+ } |
+ if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Position the pCsr->pStmt statement so that it is on the row |
+** of the %_content table that contains the last match. Return |
+** SQLITE_OK on success. |
+*/ |
+static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ |
+ int rc = SQLITE_OK; |
+ if( pCsr->isRequireSeek ){ |
+ rc = fts3CursorSeekStmt(pCsr); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); |
+ pCsr->isRequireSeek = 0; |
+ if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ |
+ return SQLITE_OK; |
+ }else{ |
+ rc = sqlite3_reset(pCsr->pStmt); |
+ if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ |
+ /* If no row was found and no error has occurred, then the %_content |
+ ** table is missing a row that is present in the full-text index. |
+ ** The data structures are corrupt. */ |
+ rc = FTS_CORRUPT_VTAB; |
+ pCsr->isEof = 1; |
+ } |
+ } |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK && pContext ){ |
+ sqlite3_result_error_code(pContext, rc); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This function is used to process a single interior node when searching |
+** a b-tree for a term or term prefix. The node data is passed to this |
+** function via the zNode/nNode parameters. The term to search for is |
+** passed in zTerm/nTerm. |
+** |
+** If piFirst is not NULL, then this function sets *piFirst to the blockid |
+** of the child node that heads the sub-tree that may contain the term. |
+** |
+** If piLast is not NULL, then *piLast is set to the right-most child node |
+** that heads a sub-tree that may contain a term for which zTerm/nTerm is |
+** a prefix. |
+** |
+** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. |
+*/ |
+static int fts3ScanInteriorNode( |
+ const char *zTerm, /* Term to select leaves for */ |
+ int nTerm, /* Size of term zTerm in bytes */ |
+ const char *zNode, /* Buffer containing segment interior node */ |
+ int nNode, /* Size of buffer at zNode */ |
+ sqlite3_int64 *piFirst, /* OUT: Selected child node */ |
+ sqlite3_int64 *piLast /* OUT: Selected child node */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ const char *zCsr = zNode; /* Cursor to iterate through node */ |
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ |
+ char *zBuffer = 0; /* Buffer to load terms into */ |
+ int nAlloc = 0; /* Size of allocated buffer */ |
+ int isFirstTerm = 1; /* True when processing first term on page */ |
+ sqlite3_int64 iChild; /* Block id of child node to descend to */ |
+ |
+ /* Skip over the 'height' varint that occurs at the start of every |
+ ** interior node. Then load the blockid of the left-child of the b-tree |
+ ** node into variable iChild. |
+ ** |
+ ** Even if the data structure on disk is corrupted, this (reading two |
+ ** varints from the buffer) does not risk an overread. If zNode is a |
+ ** root node, then the buffer comes from a SELECT statement. SQLite does |
+ ** not make this guarantee explicitly, but in practice there are always |
+ ** either more than 20 bytes of allocated space following the nNode bytes of |
+ ** contents, or two zero bytes. Or, if the node is read from the %_segments |
+ ** table, then there are always 20 bytes of zeroed padding following the |
+ ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). |
+ */ |
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); |
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); |
+ if( zCsr>zEnd ){ |
+ return FTS_CORRUPT_VTAB; |
+ } |
+ |
+ while( zCsr<zEnd && (piFirst || piLast) ){ |
+ int cmp; /* memcmp() result */ |
+ int nSuffix; /* Size of term suffix */ |
+ int nPrefix = 0; /* Size of term prefix */ |
+ int nBuffer; /* Total term size */ |
+ |
+ /* Load the next term on the node into zBuffer. Use realloc() to expand |
+ ** the size of zBuffer if required. */ |
+ if( !isFirstTerm ){ |
+ zCsr += fts3GetVarint32(zCsr, &nPrefix); |
+ } |
+ isFirstTerm = 0; |
+ zCsr += fts3GetVarint32(zCsr, &nSuffix); |
+ |
+ /* NOTE(shess): Previous code checked for negative nPrefix and |
+ ** nSuffix and suffix overrunning zEnd. Additionally corrupt if |
+ ** the prefix is longer than the previous term, or if the suffix |
+ ** causes overflow. |
+ */ |
+ if( nPrefix<0 || nSuffix<0 /* || nPrefix>nBuffer */ |
+ || &zCsr[nSuffix]<zCsr || &zCsr[nSuffix]>zEnd ){ |
+ rc = FTS_CORRUPT_VTAB; |
+ goto finish_scan; |
+ } |
+ if( nPrefix+nSuffix>nAlloc ){ |
+ char *zNew; |
+ nAlloc = (nPrefix+nSuffix) * 2; |
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); |
+ if( !zNew ){ |
+ rc = SQLITE_NOMEM; |
+ goto finish_scan; |
+ } |
+ zBuffer = zNew; |
+ } |
+ assert( zBuffer ); |
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix); |
+ nBuffer = nPrefix + nSuffix; |
+ zCsr += nSuffix; |
+ |
+ /* Compare the term we are searching for with the term just loaded from |
+ ** the interior node. If the specified term is greater than or equal |
+ ** to the term from the interior node, then all terms on the sub-tree |
+ ** headed by node iChild are smaller than zTerm. No need to search |
+ ** iChild. |
+ ** |
+ ** If the interior node term is larger than the specified term, then |
+ ** the tree headed by iChild may contain the specified term. |
+ */ |
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); |
+ if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ |
+ *piFirst = iChild; |
+ piFirst = 0; |
+ } |
+ |
+ if( piLast && cmp<0 ){ |
+ *piLast = iChild; |
+ piLast = 0; |
+ } |
+ |
+ iChild++; |
+ }; |
+ |
+ if( piFirst ) *piFirst = iChild; |
+ if( piLast ) *piLast = iChild; |
+ |
+ finish_scan: |
+ sqlite3_free(zBuffer); |
+ return rc; |
+} |
+ |
+ |
+/* |
+** The buffer pointed to by argument zNode (size nNode bytes) contains an |
+** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) |
+** contains a term. This function searches the sub-tree headed by the zNode |
+** node for the range of leaf nodes that may contain the specified term |
+** or terms for which the specified term is a prefix. |
+** |
+** If piLeaf is not NULL, then *piLeaf is set to the blockid of the |
+** left-most leaf node in the tree that may contain the specified term. |
+** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the |
+** right-most leaf node that may contain a term for which the specified |
+** term is a prefix. |
+** |
+** It is possible that the range of returned leaf nodes does not contain |
+** the specified term or any terms for which it is a prefix. However, if the |
+** segment does contain any such terms, they are stored within the identified |
+** range. Because this function only inspects interior segment nodes (and |
+** never loads leaf nodes into memory), it is not possible to be sure. |
+** |
+** If an error occurs, an error code other than SQLITE_OK is returned. |
+*/ |
+static int fts3SelectLeaf( |
+ Fts3Table *p, /* Virtual table handle */ |
+ const char *zTerm, /* Term to select leaves for */ |
+ int nTerm, /* Size of term zTerm in bytes */ |
+ const char *zNode, /* Buffer containing segment interior node */ |
+ int nNode, /* Size of buffer at zNode */ |
+ sqlite3_int64 *piLeaf, /* Selected leaf node */ |
+ sqlite3_int64 *piLeaf2 /* Selected leaf node */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int iHeight; /* Height of this node in tree */ |
+ |
+ assert( piLeaf || piLeaf2 ); |
+ |
+ fts3GetVarint32(zNode, &iHeight); |
+ rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); |
+ assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); |
+ |
+ if( rc==SQLITE_OK && iHeight>1 ){ |
+ char *zBlob = 0; /* Blob read from %_segments table */ |
+ int nBlob = 0; /* Size of zBlob in bytes */ |
+ |
+ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ |
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); |
+ } |
+ sqlite3_free(zBlob); |
+ piLeaf = 0; |
+ zBlob = 0; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); |
+ } |
+ sqlite3_free(zBlob); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This function is used to create delta-encoded serialized lists of FTS3 |
+** varints. Each call to this function appends a single varint to a list. |
+*/ |
+static void fts3PutDeltaVarint( |
+ char **pp, /* IN/OUT: Output pointer */ |
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ |
+ sqlite3_int64 iVal /* Write this value to the list */ |
+){ |
+ assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); |
+ *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); |
+ *piPrev = iVal; |
+} |
+ |
+/* |
+** When this function is called, *ppPoslist is assumed to point to the |
+** start of a position-list. After it returns, *ppPoslist points to the |
+** first byte after the position-list. |
+** |
+** A position list is list of positions (delta encoded) and columns for |
+** a single document record of a doclist. So, in other words, this |
+** routine advances *ppPoslist so that it points to the next docid in |
+** the doclist, or to the first byte past the end of the doclist. |
+** |
+** If pp is not NULL, then the contents of the position list are copied |
+** to *pp. *pp is set to point to the first byte past the last byte copied |
+** before this function returns. |
+*/ |
+static void fts3PoslistCopy(char **pp, char **ppPoslist){ |
+ char *pEnd = *ppPoslist; |
+ char c = 0; |
+ |
+ /* The end of a position list is marked by a zero encoded as an FTS3 |
+ ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by |
+ ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail |
+ ** of some other, multi-byte, value. |
+ ** |
+ ** The following while-loop moves pEnd to point to the first byte that is not |
+ ** immediately preceded by a byte with the 0x80 bit set. Then increments |
+ ** pEnd once more so that it points to the byte immediately following the |
+ ** last byte in the position-list. |
+ */ |
+ while( *pEnd | c ){ |
+ c = *pEnd++ & 0x80; |
+ testcase( c!=0 && (*pEnd)==0 ); |
+ } |
+ pEnd++; /* Advance past the POS_END terminator byte */ |
+ |
+ if( pp ){ |
+ int n = (int)(pEnd - *ppPoslist); |
+ char *p = *pp; |
+ memcpy(p, *ppPoslist, n); |
+ p += n; |
+ *pp = p; |
+ } |
+ *ppPoslist = pEnd; |
+} |
+ |
+/* |
+** When this function is called, *ppPoslist is assumed to point to the |
+** start of a column-list. After it returns, *ppPoslist points to the |
+** to the terminator (POS_COLUMN or POS_END) byte of the column-list. |
+** |
+** A column-list is list of delta-encoded positions for a single column |
+** within a single document within a doclist. |
+** |
+** The column-list is terminated either by a POS_COLUMN varint (1) or |
+** a POS_END varint (0). This routine leaves *ppPoslist pointing to |
+** the POS_COLUMN or POS_END that terminates the column-list. |
+** |
+** If pp is not NULL, then the contents of the column-list are copied |
+** to *pp. *pp is set to point to the first byte past the last byte copied |
+** before this function returns. The POS_COLUMN or POS_END terminator |
+** is not copied into *pp. |
+*/ |
+static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ |
+ char *pEnd = *ppPoslist; |
+ char c = 0; |
+ |
+ /* A column-list is terminated by either a 0x01 or 0x00 byte that is |
+ ** not part of a multi-byte varint. |
+ */ |
+ while( 0xFE & (*pEnd | c) ){ |
+ c = *pEnd++ & 0x80; |
+ testcase( c!=0 && ((*pEnd)&0xfe)==0 ); |
+ } |
+ if( pp ){ |
+ int n = (int)(pEnd - *ppPoslist); |
+ char *p = *pp; |
+ memcpy(p, *ppPoslist, n); |
+ p += n; |
+ *pp = p; |
+ } |
+ *ppPoslist = pEnd; |
+} |
+ |
+/* |
+** Value used to signify the end of an position-list. This is safe because |
+** it is not possible to have a document with 2^31 terms. |
+*/ |
+#define POSITION_LIST_END 0x7fffffff |
+ |
+/* |
+** This function is used to help parse position-lists. When this function is |
+** called, *pp may point to the start of the next varint in the position-list |
+** being parsed, or it may point to 1 byte past the end of the position-list |
+** (in which case **pp will be a terminator bytes POS_END (0) or |
+** (1)). |
+** |
+** If *pp points past the end of the current position-list, set *pi to |
+** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, |
+** increment the current value of *pi by the value read, and set *pp to |
+** point to the next value before returning. |
+** |
+** Before calling this routine *pi must be initialized to the value of |
+** the previous position, or zero if we are reading the first position |
+** in the position-list. Because positions are delta-encoded, the value |
+** of the previous position is needed in order to compute the value of |
+** the next position. |
+*/ |
+static void fts3ReadNextPos( |
+ char **pp, /* IN/OUT: Pointer into position-list buffer */ |
+ sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ |
+){ |
+ if( (**pp)&0xFE ){ |
+ fts3GetDeltaVarint(pp, pi); |
+ *pi -= 2; |
+ }else{ |
+ *pi = POSITION_LIST_END; |
+ } |
+} |
+ |
+/* |
+** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by |
+** the value of iCol encoded as a varint to *pp. This will start a new |
+** column list. |
+** |
+** Set *pp to point to the byte just after the last byte written before |
+** returning (do not modify it if iCol==0). Return the total number of bytes |
+** written (0 if iCol==0). |
+*/ |
+static int fts3PutColNumber(char **pp, int iCol){ |
+ int n = 0; /* Number of bytes written */ |
+ if( iCol ){ |
+ char *p = *pp; /* Output pointer */ |
+ n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); |
+ *p = 0x01; |
+ *pp = &p[n]; |
+ } |
+ return n; |
+} |
+ |
+/* |
+** Compute the union of two position lists. The output written |
+** into *pp contains all positions of both *pp1 and *pp2 in sorted |
+** order and with any duplicates removed. All pointers are |
+** updated appropriately. The caller is responsible for insuring |
+** that there is enough space in *pp to hold the complete output. |
+*/ |
+static void fts3PoslistMerge( |
+ char **pp, /* Output buffer */ |
+ char **pp1, /* Left input list */ |
+ char **pp2 /* Right input list */ |
+){ |
+ char *p = *pp; |
+ char *p1 = *pp1; |
+ char *p2 = *pp2; |
+ |
+ while( *p1 || *p2 ){ |
+ int iCol1; /* The current column index in pp1 */ |
+ int iCol2; /* The current column index in pp2 */ |
+ |
+ if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); |
+ else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; |
+ else iCol1 = 0; |
+ |
+ if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); |
+ else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; |
+ else iCol2 = 0; |
+ |
+ if( iCol1==iCol2 ){ |
+ sqlite3_int64 i1 = 0; /* Last position from pp1 */ |
+ sqlite3_int64 i2 = 0; /* Last position from pp2 */ |
+ sqlite3_int64 iPrev = 0; |
+ int n = fts3PutColNumber(&p, iCol1); |
+ p1 += n; |
+ p2 += n; |
+ |
+ /* At this point, both p1 and p2 point to the start of column-lists |
+ ** for the same column (the column with index iCol1 and iCol2). |
+ ** A column-list is a list of non-negative delta-encoded varints, each |
+ ** incremented by 2 before being stored. Each list is terminated by a |
+ ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists |
+ ** and writes the results to buffer p. p is left pointing to the byte |
+ ** after the list written. No terminator (POS_END or POS_COLUMN) is |
+ ** written to the output. |
+ */ |
+ fts3GetDeltaVarint(&p1, &i1); |
+ fts3GetDeltaVarint(&p2, &i2); |
+ do { |
+ fts3PutDeltaVarint(&p, &iPrev, (i1<i2) ? i1 : i2); |
+ iPrev -= 2; |
+ if( i1==i2 ){ |
+ fts3ReadNextPos(&p1, &i1); |
+ fts3ReadNextPos(&p2, &i2); |
+ }else if( i1<i2 ){ |
+ fts3ReadNextPos(&p1, &i1); |
+ }else{ |
+ fts3ReadNextPos(&p2, &i2); |
+ } |
+ }while( i1!=POSITION_LIST_END || i2!=POSITION_LIST_END ); |
+ }else if( iCol1<iCol2 ){ |
+ p1 += fts3PutColNumber(&p, iCol1); |
+ fts3ColumnlistCopy(&p, &p1); |
+ }else{ |
+ p2 += fts3PutColNumber(&p, iCol2); |
+ fts3ColumnlistCopy(&p, &p2); |
+ } |
+ } |
+ |
+ *p++ = POS_END; |
+ *pp = p; |
+ *pp1 = p1 + 1; |
+ *pp2 = p2 + 1; |
+} |
+ |
+/* |
+** This function is used to merge two position lists into one. When it is |
+** called, *pp1 and *pp2 must both point to position lists. A position-list is |
+** the part of a doclist that follows each document id. For example, if a row |
+** contains: |
+** |
+** 'a b c'|'x y z'|'a b b a' |
+** |
+** Then the position list for this row for token 'b' would consist of: |
+** |
+** 0x02 0x01 0x02 0x03 0x03 0x00 |
+** |
+** When this function returns, both *pp1 and *pp2 are left pointing to the |
+** byte following the 0x00 terminator of their respective position lists. |
+** |
+** If isSaveLeft is 0, an entry is added to the output position list for |
+** each position in *pp2 for which there exists one or more positions in |
+** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. |
+** when the *pp1 token appears before the *pp2 token, but not more than nToken |
+** slots before it. |
+** |
+** e.g. nToken==1 searches for adjacent positions. |
+*/ |
+static int fts3PoslistPhraseMerge( |
+ char **pp, /* IN/OUT: Preallocated output buffer */ |
+ int nToken, /* Maximum difference in token positions */ |
+ int isSaveLeft, /* Save the left position */ |
+ int isExact, /* If *pp1 is exactly nTokens before *pp2 */ |
+ char **pp1, /* IN/OUT: Left input list */ |
+ char **pp2 /* IN/OUT: Right input list */ |
+){ |
+ char *p = *pp; |
+ char *p1 = *pp1; |
+ char *p2 = *pp2; |
+ int iCol1 = 0; |
+ int iCol2 = 0; |
+ |
+ /* Never set both isSaveLeft and isExact for the same invocation. */ |
+ assert( isSaveLeft==0 || isExact==0 ); |
+ |
+ assert( p!=0 && *p1!=0 && *p2!=0 ); |
+ if( *p1==POS_COLUMN ){ |
+ p1++; |
+ p1 += fts3GetVarint32(p1, &iCol1); |
+ } |
+ if( *p2==POS_COLUMN ){ |
+ p2++; |
+ p2 += fts3GetVarint32(p2, &iCol2); |
+ } |
+ |
+ while( 1 ){ |
+ if( iCol1==iCol2 ){ |
+ char *pSave = p; |
+ sqlite3_int64 iPrev = 0; |
+ sqlite3_int64 iPos1 = 0; |
+ sqlite3_int64 iPos2 = 0; |
+ |
+ if( iCol1 ){ |
+ *p++ = POS_COLUMN; |
+ p += sqlite3Fts3PutVarint(p, iCol1); |
+ } |
+ |
+ assert( *p1!=POS_END && *p1!=POS_COLUMN ); |
+ assert( *p2!=POS_END && *p2!=POS_COLUMN ); |
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; |
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; |
+ |
+ while( 1 ){ |
+ if( iPos2==iPos1+nToken |
+ || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) |
+ ){ |
+ sqlite3_int64 iSave; |
+ iSave = isSaveLeft ? iPos1 : iPos2; |
+ fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; |
+ pSave = 0; |
+ assert( p ); |
+ } |
+ if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ |
+ if( (*p2&0xFE)==0 ) break; |
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; |
+ }else{ |
+ if( (*p1&0xFE)==0 ) break; |
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; |
+ } |
+ } |
+ |
+ if( pSave ){ |
+ assert( pp && p ); |
+ p = pSave; |
+ } |
+ |
+ fts3ColumnlistCopy(0, &p1); |
+ fts3ColumnlistCopy(0, &p2); |
+ assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); |
+ if( 0==*p1 || 0==*p2 ) break; |
+ |
+ p1++; |
+ p1 += fts3GetVarint32(p1, &iCol1); |
+ p2++; |
+ p2 += fts3GetVarint32(p2, &iCol2); |
+ } |
+ |
+ /* Advance pointer p1 or p2 (whichever corresponds to the smaller of |
+ ** iCol1 and iCol2) so that it points to either the 0x00 that marks the |
+ ** end of the position list, or the 0x01 that precedes the next |
+ ** column-number in the position list. |
+ */ |
+ else if( iCol1<iCol2 ){ |
+ fts3ColumnlistCopy(0, &p1); |
+ if( 0==*p1 ) break; |
+ p1++; |
+ p1 += fts3GetVarint32(p1, &iCol1); |
+ }else{ |
+ fts3ColumnlistCopy(0, &p2); |
+ if( 0==*p2 ) break; |
+ p2++; |
+ p2 += fts3GetVarint32(p2, &iCol2); |
+ } |
+ } |
+ |
+ fts3PoslistCopy(0, &p2); |
+ fts3PoslistCopy(0, &p1); |
+ *pp1 = p1; |
+ *pp2 = p2; |
+ if( *pp==p ){ |
+ return 0; |
+ } |
+ *p++ = 0x00; |
+ *pp = p; |
+ return 1; |
+} |
+ |
+/* |
+** Merge two position-lists as required by the NEAR operator. The argument |
+** position lists correspond to the left and right phrases of an expression |
+** like: |
+** |
+** "phrase 1" NEAR "phrase number 2" |
+** |
+** Position list *pp1 corresponds to the left-hand side of the NEAR |
+** expression and *pp2 to the right. As usual, the indexes in the position |
+** lists are the offsets of the last token in each phrase (tokens "1" and "2" |
+** in the example above). |
+** |
+** The output position list - written to *pp - is a copy of *pp2 with those |
+** entries that are not sufficiently NEAR entries in *pp1 removed. |
+*/ |
+static int fts3PoslistNearMerge( |
+ char **pp, /* Output buffer */ |
+ char *aTmp, /* Temporary buffer space */ |
+ int nRight, /* Maximum difference in token positions */ |
+ int nLeft, /* Maximum difference in token positions */ |
+ char **pp1, /* IN/OUT: Left input list */ |
+ char **pp2 /* IN/OUT: Right input list */ |
+){ |
+ char *p1 = *pp1; |
+ char *p2 = *pp2; |
+ |
+ char *pTmp1 = aTmp; |
+ char *pTmp2; |
+ char *aTmp2; |
+ int res = 1; |
+ |
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); |
+ aTmp2 = pTmp2 = pTmp1; |
+ *pp1 = p1; |
+ *pp2 = p2; |
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); |
+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ |
+ fts3PoslistMerge(pp, &aTmp, &aTmp2); |
+ }else if( pTmp1!=aTmp ){ |
+ fts3PoslistCopy(pp, &aTmp); |
+ }else if( pTmp2!=aTmp2 ){ |
+ fts3PoslistCopy(pp, &aTmp2); |
+ }else{ |
+ res = 0; |
+ } |
+ |
+ return res; |
+} |
+ |
+/* |
+** An instance of this function is used to merge together the (potentially |
+** large number of) doclists for each term that matches a prefix query. |
+** See function fts3TermSelectMerge() for details. |
+*/ |
+typedef struct TermSelect TermSelect; |
+struct TermSelect { |
+ char *aaOutput[16]; /* Malloc'd output buffers */ |
+ int anOutput[16]; /* Size each output buffer in bytes */ |
+}; |
+ |
+/* |
+** This function is used to read a single varint from a buffer. Parameter |
+** pEnd points 1 byte past the end of the buffer. When this function is |
+** called, if *pp points to pEnd or greater, then the end of the buffer |
+** has been reached. In this case *pp is set to 0 and the function returns. |
+** |
+** If *pp does not point to or past pEnd, then a single varint is read |
+** from *pp. *pp is then set to point 1 byte past the end of the read varint. |
+** |
+** If bDescIdx is false, the value read is added to *pVal before returning. |
+** If it is true, the value read is subtracted from *pVal before this |
+** function returns. |
+*/ |
+static void fts3GetDeltaVarint3( |
+ char **pp, /* IN/OUT: Point to read varint from */ |
+ char *pEnd, /* End of buffer */ |
+ int bDescIdx, /* True if docids are descending */ |
+ sqlite3_int64 *pVal /* IN/OUT: Integer value */ |
+){ |
+ if( *pp>=pEnd ){ |
+ *pp = 0; |
+ }else{ |
+ sqlite3_int64 iVal; |
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal); |
+ if( bDescIdx ){ |
+ *pVal -= iVal; |
+ }else{ |
+ *pVal += iVal; |
+ } |
+ } |
+} |
+ |
+/* |
+** This function is used to write a single varint to a buffer. The varint |
+** is written to *pp. Before returning, *pp is set to point 1 byte past the |
+** end of the value written. |
+** |
+** If *pbFirst is zero when this function is called, the value written to |
+** the buffer is that of parameter iVal. |
+** |
+** If *pbFirst is non-zero when this function is called, then the value |
+** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) |
+** (if bDescIdx is non-zero). |
+** |
+** Before returning, this function always sets *pbFirst to 1 and *piPrev |
+** to the value of parameter iVal. |
+*/ |
+static void fts3PutDeltaVarint3( |
+ char **pp, /* IN/OUT: Output pointer */ |
+ int bDescIdx, /* True for descending docids */ |
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ |
+ int *pbFirst, /* IN/OUT: True after first int written */ |
+ sqlite3_int64 iVal /* Write this value to the list */ |
+){ |
+ sqlite3_int64 iWrite; |
+ if( bDescIdx==0 || *pbFirst==0 ){ |
+ iWrite = iVal - *piPrev; |
+ }else{ |
+ iWrite = *piPrev - iVal; |
+ } |
+ assert( *pbFirst || *piPrev==0 ); |
+ assert( *pbFirst==0 || iWrite>0 ); |
+ *pp += sqlite3Fts3PutVarint(*pp, iWrite); |
+ *piPrev = iVal; |
+ *pbFirst = 1; |
+} |
+ |
+ |
+/* |
+** This macro is used by various functions that merge doclists. The two |
+** arguments are 64-bit docid values. If the value of the stack variable |
+** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). |
+** Otherwise, (i2-i1). |
+** |
+** Using this makes it easier to write code that can merge doclists that are |
+** sorted in either ascending or descending order. |
+*/ |
+#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) |
+ |
+/* |
+** This function does an "OR" merge of two doclists (output contains all |
+** positions contained in either argument doclist). If the docids in the |
+** input doclists are sorted in ascending order, parameter bDescDoclist |
+** should be false. If they are sorted in ascending order, it should be |
+** passed a non-zero value. |
+** |
+** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer |
+** containing the output doclist and SQLITE_OK is returned. In this case |
+** *pnOut is set to the number of bytes in the output doclist. |
+** |
+** If an error occurs, an SQLite error code is returned. The output values |
+** are undefined in this case. |
+*/ |
+static int fts3DoclistOrMerge( |
+ int bDescDoclist, /* True if arguments are desc */ |
+ char *a1, int n1, /* First doclist */ |
+ char *a2, int n2, /* Second doclist */ |
+ char **paOut, int *pnOut /* OUT: Malloc'd doclist */ |
+){ |
+ sqlite3_int64 i1 = 0; |
+ sqlite3_int64 i2 = 0; |
+ sqlite3_int64 iPrev = 0; |
+ char *pEnd1 = &a1[n1]; |
+ char *pEnd2 = &a2[n2]; |
+ char *p1 = a1; |
+ char *p2 = a2; |
+ char *p; |
+ char *aOut; |
+ int bFirstOut = 0; |
+ |
+ *paOut = 0; |
+ *pnOut = 0; |
+ |
+ /* Allocate space for the output. Both the input and output doclists |
+ ** are delta encoded. If they are in ascending order (bDescDoclist==0), |
+ ** then the first docid in each list is simply encoded as a varint. For |
+ ** each subsequent docid, the varint stored is the difference between the |
+ ** current and previous docid (a positive number - since the list is in |
+ ** ascending order). |
+ ** |
+ ** The first docid written to the output is therefore encoded using the |
+ ** same number of bytes as it is in whichever of the input lists it is |
+ ** read from. And each subsequent docid read from the same input list |
+ ** consumes either the same or less bytes as it did in the input (since |
+ ** the difference between it and the previous value in the output must |
+ ** be a positive value less than or equal to the delta value read from |
+ ** the input list). The same argument applies to all but the first docid |
+ ** read from the 'other' list. And to the contents of all position lists |
+ ** that will be copied and merged from the input to the output. |
+ ** |
+ ** However, if the first docid copied to the output is a negative number, |
+ ** then the encoding of the first docid from the 'other' input list may |
+ ** be larger in the output than it was in the input (since the delta value |
+ ** may be a larger positive integer than the actual docid). |
+ ** |
+ ** The space required to store the output is therefore the sum of the |
+ ** sizes of the two inputs, plus enough space for exactly one of the input |
+ ** docids to grow. |
+ ** |
+ ** A symetric argument may be made if the doclists are in descending |
+ ** order. |
+ */ |
+ aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); |
+ if( !aOut ) return SQLITE_NOMEM; |
+ |
+ p = aOut; |
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); |
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); |
+ while( p1 || p2 ){ |
+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2); |
+ |
+ if( p2 && p1 && iDiff==0 ){ |
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); |
+ fts3PoslistMerge(&p, &p1, &p2); |
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); |
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); |
+ }else if( !p2 || (p1 && iDiff<0) ){ |
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); |
+ fts3PoslistCopy(&p, &p1); |
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); |
+ }else{ |
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); |
+ fts3PoslistCopy(&p, &p2); |
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); |
+ } |
+ } |
+ |
+ *paOut = aOut; |
+ *pnOut = (int)(p-aOut); |
+ assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function does a "phrase" merge of two doclists. In a phrase merge, |
+** the output contains a copy of each position from the right-hand input |
+** doclist for which there is a position in the left-hand input doclist |
+** exactly nDist tokens before it. |
+** |
+** If the docids in the input doclists are sorted in ascending order, |
+** parameter bDescDoclist should be false. If they are sorted in ascending |
+** order, it should be passed a non-zero value. |
+** |
+** The right-hand input doclist is overwritten by this function. |
+*/ |
+static int fts3DoclistPhraseMerge( |
+ int bDescDoclist, /* True if arguments are desc */ |
+ int nDist, /* Distance from left to right (1=adjacent) */ |
+ char *aLeft, int nLeft, /* Left doclist */ |
+ char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ |
+){ |
+ sqlite3_int64 i1 = 0; |
+ sqlite3_int64 i2 = 0; |
+ sqlite3_int64 iPrev = 0; |
+ char *aRight = *paRight; |
+ char *pEnd1 = &aLeft[nLeft]; |
+ char *pEnd2 = &aRight[*pnRight]; |
+ char *p1 = aLeft; |
+ char *p2 = aRight; |
+ char *p; |
+ int bFirstOut = 0; |
+ char *aOut; |
+ |
+ assert( nDist>0 ); |
+ if( bDescDoclist ){ |
+ aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); |
+ if( aOut==0 ) return SQLITE_NOMEM; |
+ }else{ |
+ aOut = aRight; |
+ } |
+ p = aOut; |
+ |
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); |
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); |
+ |
+ while( p1 && p2 ){ |
+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2); |
+ if( iDiff==0 ){ |
+ char *pSave = p; |
+ sqlite3_int64 iPrevSave = iPrev; |
+ int bFirstOutSave = bFirstOut; |
+ |
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); |
+ if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ |
+ p = pSave; |
+ iPrev = iPrevSave; |
+ bFirstOut = bFirstOutSave; |
+ } |
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); |
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); |
+ }else if( iDiff<0 ){ |
+ fts3PoslistCopy(0, &p1); |
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); |
+ }else{ |
+ fts3PoslistCopy(0, &p2); |
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); |
+ } |
+ } |
+ |
+ *pnRight = (int)(p - aOut); |
+ if( bDescDoclist ){ |
+ sqlite3_free(aRight); |
+ *paRight = aOut; |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Argument pList points to a position list nList bytes in size. This |
+** function checks to see if the position list contains any entries for |
+** a token in position 0 (of any column). If so, it writes argument iDelta |
+** to the output buffer pOut, followed by a position list consisting only |
+** of the entries from pList at position 0, and terminated by an 0x00 byte. |
+** The value returned is the number of bytes written to pOut (if any). |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3FirstFilter( |
+ sqlite3_int64 iDelta, /* Varint that may be written to pOut */ |
+ char *pList, /* Position list (no 0x00 term) */ |
+ int nList, /* Size of pList in bytes */ |
+ char *pOut /* Write output here */ |
+){ |
+ int nOut = 0; |
+ int bWritten = 0; /* True once iDelta has been written */ |
+ char *p = pList; |
+ char *pEnd = &pList[nList]; |
+ |
+ if( *p!=0x01 ){ |
+ if( *p==0x02 ){ |
+ nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); |
+ pOut[nOut++] = 0x02; |
+ bWritten = 1; |
+ } |
+ fts3ColumnlistCopy(0, &p); |
+ } |
+ |
+ while( p<pEnd && *p==0x01 ){ |
+ sqlite3_int64 iCol; |
+ p++; |
+ p += sqlite3Fts3GetVarint(p, &iCol); |
+ if( *p==0x02 ){ |
+ if( bWritten==0 ){ |
+ nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); |
+ bWritten = 1; |
+ } |
+ pOut[nOut++] = 0x01; |
+ nOut += sqlite3Fts3PutVarint(&pOut[nOut], iCol); |
+ pOut[nOut++] = 0x02; |
+ } |
+ fts3ColumnlistCopy(0, &p); |
+ } |
+ if( bWritten ){ |
+ pOut[nOut++] = 0x00; |
+ } |
+ |
+ return nOut; |
+} |
+ |
+ |
+/* |
+** Merge all doclists in the TermSelect.aaOutput[] array into a single |
+** doclist stored in TermSelect.aaOutput[0]. If successful, delete all |
+** other doclists (except the aaOutput[0] one) and return SQLITE_OK. |
+** |
+** If an OOM error occurs, return SQLITE_NOMEM. In this case it is |
+** the responsibility of the caller to free any doclists left in the |
+** TermSelect.aaOutput[] array. |
+*/ |
+static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ |
+ char *aOut = 0; |
+ int nOut = 0; |
+ int i; |
+ |
+ /* Loop through the doclists in the aaOutput[] array. Merge them all |
+ ** into a single doclist. |
+ */ |
+ for(i=0; i<SizeofArray(pTS->aaOutput); i++){ |
+ if( pTS->aaOutput[i] ){ |
+ if( !aOut ){ |
+ aOut = pTS->aaOutput[i]; |
+ nOut = pTS->anOutput[i]; |
+ pTS->aaOutput[i] = 0; |
+ }else{ |
+ int nNew; |
+ char *aNew; |
+ |
+ int rc = fts3DoclistOrMerge(p->bDescIdx, |
+ pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew |
+ ); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_free(aOut); |
+ return rc; |
+ } |
+ |
+ sqlite3_free(pTS->aaOutput[i]); |
+ sqlite3_free(aOut); |
+ pTS->aaOutput[i] = 0; |
+ aOut = aNew; |
+ nOut = nNew; |
+ } |
+ } |
+ } |
+ |
+ pTS->aaOutput[0] = aOut; |
+ pTS->anOutput[0] = nOut; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Merge the doclist aDoclist/nDoclist into the TermSelect object passed |
+** as the first argument. The merge is an "OR" merge (see function |
+** fts3DoclistOrMerge() for details). |
+** |
+** This function is called with the doclist for each term that matches |
+** a queried prefix. It merges all these doclists into one, the doclist |
+** for the specified prefix. Since there can be a very large number of |
+** doclists to merge, the merging is done pair-wise using the TermSelect |
+** object. |
+** |
+** This function returns SQLITE_OK if the merge is successful, or an |
+** SQLite error code (SQLITE_NOMEM) if an error occurs. |
+*/ |
+static int fts3TermSelectMerge( |
+ Fts3Table *p, /* FTS table handle */ |
+ TermSelect *pTS, /* TermSelect object to merge into */ |
+ char *aDoclist, /* Pointer to doclist */ |
+ int nDoclist /* Size of aDoclist in bytes */ |
+){ |
+ if( pTS->aaOutput[0]==0 ){ |
+ /* If this is the first term selected, copy the doclist to the output |
+ ** buffer using memcpy(). |
+ ** |
+ ** Add FTS3_VARINT_MAX bytes of unused space to the end of the |
+ ** allocation. This is so as to ensure that the buffer is big enough |
+ ** to hold the current doclist AND'd with any other doclist. If the |
+ ** doclists are stored in order=ASC order, this padding would not be |
+ ** required (since the size of [doclistA AND doclistB] is always less |
+ ** than or equal to the size of [doclistA] in that case). But this is |
+ ** not true for order=DESC. For example, a doclist containing (1, -1) |
+ ** may be smaller than (-1), as in the first example the -1 may be stored |
+ ** as a single-byte delta, whereas in the second it must be stored as a |
+ ** FTS3_VARINT_MAX byte varint. |
+ ** |
+ ** Similar padding is added in the fts3DoclistOrMerge() function. |
+ */ |
+ pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); |
+ pTS->anOutput[0] = nDoclist; |
+ if( pTS->aaOutput[0] ){ |
+ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); |
+ }else{ |
+ return SQLITE_NOMEM; |
+ } |
+ }else{ |
+ char *aMerge = aDoclist; |
+ int nMerge = nDoclist; |
+ int iOut; |
+ |
+ for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){ |
+ if( pTS->aaOutput[iOut]==0 ){ |
+ assert( iOut>0 ); |
+ pTS->aaOutput[iOut] = aMerge; |
+ pTS->anOutput[iOut] = nMerge; |
+ break; |
+ }else{ |
+ char *aNew; |
+ int nNew; |
+ |
+ int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, |
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew |
+ ); |
+ if( rc!=SQLITE_OK ){ |
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge); |
+ return rc; |
+ } |
+ |
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge); |
+ sqlite3_free(pTS->aaOutput[iOut]); |
+ pTS->aaOutput[iOut] = 0; |
+ |
+ aMerge = aNew; |
+ nMerge = nNew; |
+ if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ |
+ pTS->aaOutput[iOut] = aMerge; |
+ pTS->anOutput[iOut] = nMerge; |
+ } |
+ } |
+ } |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Append SegReader object pNew to the end of the pCsr->apSegment[] array. |
+*/ |
+static int fts3SegReaderCursorAppend( |
+ Fts3MultiSegReader *pCsr, |
+ Fts3SegReader *pNew |
+){ |
+ if( (pCsr->nSegment%16)==0 ){ |
+ Fts3SegReader **apNew; |
+ int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); |
+ apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); |
+ if( !apNew ){ |
+ sqlite3Fts3SegReaderFree(pNew); |
+ return SQLITE_NOMEM; |
+ } |
+ pCsr->apSegment = apNew; |
+ } |
+ pCsr->apSegment[pCsr->nSegment++] = pNew; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Add seg-reader objects to the Fts3MultiSegReader object passed as the |
+** 8th argument. |
+** |
+** This function returns SQLITE_OK if successful, or an SQLite error code |
+** otherwise. |
+*/ |
+static int fts3SegReaderCursor( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ int iLangid, /* Language id */ |
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ |
+ int iLevel, /* Level of segments to scan */ |
+ const char *zTerm, /* Term to query for */ |
+ int nTerm, /* Size of zTerm in bytes */ |
+ int isPrefix, /* True for a prefix search */ |
+ int isScan, /* True to scan from zTerm to EOF */ |
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */ |
+){ |
+ int rc = SQLITE_OK; /* Error code */ |
+ sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ |
+ int rc2; /* Result of sqlite3_reset() */ |
+ |
+ /* If iLevel is less than 0 and this is not a scan, include a seg-reader |
+ ** for the pending-terms. If this is a scan, then this call must be being |
+ ** made by an fts4aux module, not an FTS table. In this case calling |
+ ** Fts3SegReaderPending might segfault, as the data structures used by |
+ ** fts4aux are not completely populated. So it's easiest to filter these |
+ ** calls out here. */ |
+ if( iLevel<0 && p->aIndex ){ |
+ Fts3SegReader *pSeg = 0; |
+ rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); |
+ if( rc==SQLITE_OK && pSeg ){ |
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg); |
+ } |
+ } |
+ |
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); |
+ } |
+ |
+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ |
+ Fts3SegReader *pSeg = 0; |
+ |
+ /* Read the values returned by the SELECT into local variables. */ |
+ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); |
+ sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); |
+ sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); |
+ int nRoot = sqlite3_column_bytes(pStmt, 4); |
+ char const *zRoot = sqlite3_column_blob(pStmt, 4); |
+ |
+ /* If zTerm is not NULL, and this segment is not stored entirely on its |
+ ** root node, the range of leaves scanned can be reduced. Do this. */ |
+ if( iStartBlock && zTerm ){ |
+ sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); |
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); |
+ if( rc!=SQLITE_OK ) goto finished; |
+ if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; |
+ } |
+ |
+ rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, |
+ (isPrefix==0 && isScan==0), |
+ iStartBlock, iLeavesEndBlock, |
+ iEndBlock, zRoot, nRoot, &pSeg |
+ ); |
+ if( rc!=SQLITE_OK ) goto finished; |
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg); |
+ } |
+ } |
+ |
+ finished: |
+ rc2 = sqlite3_reset(pStmt); |
+ if( rc==SQLITE_DONE ) rc = rc2; |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Set up a cursor object for iterating through a full-text index or a |
+** single level therein. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ int iLangid, /* Language-id to search */ |
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ |
+ int iLevel, /* Level of segments to scan */ |
+ const char *zTerm, /* Term to query for */ |
+ int nTerm, /* Size of zTerm in bytes */ |
+ int isPrefix, /* True for a prefix search */ |
+ int isScan, /* True to scan from zTerm to EOF */ |
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */ |
+){ |
+ assert( iIndex>=0 && iIndex<p->nIndex ); |
+ assert( iLevel==FTS3_SEGCURSOR_ALL |
+ || iLevel==FTS3_SEGCURSOR_PENDING |
+ || iLevel>=0 |
+ ); |
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); |
+ assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); |
+ assert( isPrefix==0 || isScan==0 ); |
+ |
+ memset(pCsr, 0, sizeof(Fts3MultiSegReader)); |
+ return fts3SegReaderCursor( |
+ p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr |
+ ); |
+} |
+ |
+/* |
+** In addition to its current configuration, have the Fts3MultiSegReader |
+** passed as the 4th argument also scan the doclist for term zTerm/nTerm. |
+** |
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. |
+*/ |
+static int fts3SegReaderCursorAddZero( |
+ Fts3Table *p, /* FTS virtual table handle */ |
+ int iLangid, |
+ const char *zTerm, /* Term to scan doclist of */ |
+ int nTerm, /* Number of bytes in zTerm */ |
+ Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ |
+){ |
+ return fts3SegReaderCursor(p, |
+ iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr |
+ ); |
+} |
+ |
+/* |
+** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, |
+** if isPrefix is true, to scan the doclist for all terms for which |
+** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write |
+** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return |
+** an SQLite error code. |
+** |
+** It is the responsibility of the caller to free this object by eventually |
+** passing it to fts3SegReaderCursorFree() |
+** |
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. |
+** Output parameter *ppSegcsr is set to 0 if an error occurs. |
+*/ |
+static int fts3TermSegReaderCursor( |
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */ |
+ const char *zTerm, /* Term to query for */ |
+ int nTerm, /* Size of zTerm in bytes */ |
+ int isPrefix, /* True for a prefix search */ |
+ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ |
+){ |
+ Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ |
+ int rc = SQLITE_NOMEM; /* Return code */ |
+ |
+ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); |
+ if( pSegcsr ){ |
+ int i; |
+ int bFound = 0; /* True once an index has been found */ |
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; |
+ |
+ if( isPrefix ){ |
+ for(i=1; bFound==0 && i<p->nIndex; i++){ |
+ if( p->aIndex[i].nPrefix==nTerm ){ |
+ bFound = 1; |
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, |
+ i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr |
+ ); |
+ pSegcsr->bLookup = 1; |
+ } |
+ } |
+ |
+ for(i=1; bFound==0 && i<p->nIndex; i++){ |
+ if( p->aIndex[i].nPrefix==nTerm+1 ){ |
+ bFound = 1; |
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, |
+ i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr |
+ ); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3SegReaderCursorAddZero( |
+ p, pCsr->iLangid, zTerm, nTerm, pSegcsr |
+ ); |
+ } |
+ } |
+ } |
+ } |
+ |
+ if( bFound==0 ){ |
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, |
+ 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr |
+ ); |
+ pSegcsr->bLookup = !isPrefix; |
+ } |
+ } |
+ |
+ *ppSegcsr = pSegcsr; |
+ return rc; |
+} |
+ |
+/* |
+** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). |
+*/ |
+static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ |
+ sqlite3Fts3SegReaderFinish(pSegcsr); |
+ sqlite3_free(pSegcsr); |
+} |
+ |
+/* |
+** This function retrieves the doclist for the specified term (or term |
+** prefix) from the database. |
+*/ |
+static int fts3TermSelect( |
+ Fts3Table *p, /* Virtual table handle */ |
+ Fts3PhraseToken *pTok, /* Token to query for */ |
+ int iColumn, /* Column to query (or -ve for all columns) */ |
+ int *pnOut, /* OUT: Size of buffer at *ppOut */ |
+ char **ppOut /* OUT: Malloced result buffer */ |
+){ |
+ int rc; /* Return code */ |
+ Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ |
+ TermSelect tsc; /* Object for pair-wise doclist merging */ |
+ Fts3SegFilter filter; /* Segment term filter configuration */ |
+ |
+ pSegcsr = pTok->pSegcsr; |
+ memset(&tsc, 0, sizeof(TermSelect)); |
+ |
+ filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS |
+ | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) |
+ | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) |
+ | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); |
+ filter.iCol = iColumn; |
+ filter.zTerm = pTok->z; |
+ filter.nTerm = pTok->n; |
+ |
+ rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); |
+ while( SQLITE_OK==rc |
+ && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) |
+ ){ |
+ rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3TermSelectFinishMerge(p, &tsc); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ *ppOut = tsc.aaOutput[0]; |
+ *pnOut = tsc.anOutput[0]; |
+ }else{ |
+ int i; |
+ for(i=0; i<SizeofArray(tsc.aaOutput); i++){ |
+ sqlite3_free(tsc.aaOutput[i]); |
+ } |
+ } |
+ |
+ fts3SegReaderCursorFree(pSegcsr); |
+ pTok->pSegcsr = 0; |
+ return rc; |
+} |
+ |
+/* |
+** This function counts the total number of docids in the doclist stored |
+** in buffer aList[], size nList bytes. |
+** |
+** If the isPoslist argument is true, then it is assumed that the doclist |
+** contains a position-list following each docid. Otherwise, it is assumed |
+** that the doclist is simply a list of docids stored as delta encoded |
+** varints. |
+*/ |
+static int fts3DoclistCountDocids(char *aList, int nList){ |
+ int nDoc = 0; /* Return value */ |
+ if( aList ){ |
+ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ |
+ char *p = aList; /* Cursor */ |
+ while( p<aEnd ){ |
+ nDoc++; |
+ while( (*p++)&0x80 ); /* Skip docid varint */ |
+ fts3PoslistCopy(0, &p); /* Skip over position list */ |
+ } |
+ } |
+ |
+ return nDoc; |
+} |
+ |
+/* |
+** Advance the cursor to the next row in the %_content table that |
+** matches the search criteria. For a MATCH search, this will be |
+** the next row that matches. For a full-table scan, this will be |
+** simply the next row in the %_content table. For a docid lookup, |
+** this routine simply sets the EOF flag. |
+** |
+** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned |
+** even if we reach end-of-file. The fts3EofMethod() will be called |
+** subsequently to determine whether or not an EOF was hit. |
+*/ |
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ |
+ int rc; |
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; |
+ if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ |
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ |
+ pCsr->isEof = 1; |
+ rc = sqlite3_reset(pCsr->pStmt); |
+ }else{ |
+ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); |
+ rc = SQLITE_OK; |
+ } |
+ }else{ |
+ rc = fts3EvalNext((Fts3Cursor *)pCursor); |
+ } |
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); |
+ return rc; |
+} |
+ |
+/* |
+** The following are copied from sqliteInt.h. |
+** |
+** Constants for the largest and smallest possible 64-bit signed integers. |
+** These macros are designed to work correctly on both 32-bit and 64-bit |
+** compilers. |
+*/ |
+#ifndef SQLITE_AMALGAMATION |
+# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) |
+# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) |
+#endif |
+ |
+/* |
+** If the numeric type of argument pVal is "integer", then return it |
+** converted to a 64-bit signed integer. Otherwise, return a copy of |
+** the second parameter, iDefault. |
+*/ |
+static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ |
+ if( pVal ){ |
+ int eType = sqlite3_value_numeric_type(pVal); |
+ if( eType==SQLITE_INTEGER ){ |
+ return sqlite3_value_int64(pVal); |
+ } |
+ } |
+ return iDefault; |
+} |
+ |
+/* |
+** This is the xFilter interface for the virtual table. See |
+** the virtual table xFilter method documentation for additional |
+** information. |
+** |
+** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against |
+** the %_content table. |
+** |
+** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry |
+** in the %_content table. |
+** |
+** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The |
+** column on the left-hand side of the MATCH operator is column |
+** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand |
+** side of the MATCH operator. |
+*/ |
+static int fts3FilterMethod( |
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ |
+ int idxNum, /* Strategy index */ |
+ const char *idxStr, /* Unused */ |
+ int nVal, /* Number of elements in apVal */ |
+ sqlite3_value **apVal /* Arguments for the indexing scheme */ |
+){ |
+ int rc = SQLITE_OK; |
+ char *zSql; /* SQL statement used to access %_content */ |
+ int eSearch; |
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab; |
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; |
+ |
+ sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ |
+ sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ |
+ sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ |
+ sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ |
+ int iIdx; |
+ |
+ UNUSED_PARAMETER(idxStr); |
+ UNUSED_PARAMETER(nVal); |
+ |
+ eSearch = (idxNum & 0x0000FFFF); |
+ assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); |
+ assert( p->pSegments==0 ); |
+ |
+ /* Collect arguments into local variables */ |
+ iIdx = 0; |
+ if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; |
+ if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; |
+ if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; |
+ if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; |
+ assert( iIdx==nVal ); |
+ |
+ /* In case the cursor has been used before, clear it now. */ |
+ fts3CursorFinalizeStmt(pCsr); |
+ sqlite3_free(pCsr->aDoclist); |
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); |
+ sqlite3Fts3ExprFree(pCsr->pExpr); |
+ memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); |
+ |
+ /* Set the lower and upper bounds on docids to return */ |
+ pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); |
+ pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); |
+ |
+ if( idxStr ){ |
+ pCsr->bDesc = (idxStr[0]=='D'); |
+ }else{ |
+ pCsr->bDesc = p->bDescIdx; |
+ } |
+ pCsr->eSearch = (i16)eSearch; |
+ |
+ if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ |
+ int iCol = eSearch-FTS3_FULLTEXT_SEARCH; |
+ const char *zQuery = (const char *)sqlite3_value_text(pCons); |
+ |
+ if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ |
+ return SQLITE_NOMEM; |
+ } |
+ |
+ pCsr->iLangid = 0; |
+ if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); |
+ |
+ assert( p->base.zErrMsg==0 ); |
+ rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, |
+ p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, |
+ &p->base.zErrMsg |
+ ); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ rc = fts3EvalStart(pCsr); |
+ sqlite3Fts3SegmentsClose(p); |
+ if( rc!=SQLITE_OK ) return rc; |
+ pCsr->pNextId = pCsr->aDoclist; |
+ pCsr->iPrevId = 0; |
+ } |
+ |
+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the |
+ ** statement loops through all rows of the %_content table. For a |
+ ** full-text query or docid lookup, the statement retrieves a single |
+ ** row by docid. |
+ */ |
+ if( eSearch==FTS3_FULLSCAN_SEARCH ){ |
+ if( pDocidGe || pDocidLe ){ |
+ zSql = sqlite3_mprintf( |
+ "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", |
+ p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, |
+ (pCsr->bDesc ? "DESC" : "ASC") |
+ ); |
+ }else{ |
+ zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", |
+ p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") |
+ ); |
+ } |
+ if( zSql ){ |
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); |
+ sqlite3_free(zSql); |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ }else if( eSearch==FTS3_DOCID_SEARCH ){ |
+ rc = fts3CursorSeekStmt(pCsr); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); |
+ } |
+ } |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ return fts3NextMethod(pCursor); |
+} |
+ |
+/* |
+** This is the xEof method of the virtual table. SQLite calls this |
+** routine to find out if it has reached the end of a result set. |
+*/ |
+static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ |
+ return ((Fts3Cursor *)pCursor)->isEof; |
+} |
+ |
+/* |
+** This is the xRowid method. The SQLite core calls this routine to |
+** retrieve the rowid for the current row of the result set. fts3 |
+** exposes %_content.docid as the rowid for the virtual table. The |
+** rowid should be written to *pRowid. |
+*/ |
+static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ |
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; |
+ *pRowid = pCsr->iPrevId; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This is the xColumn method, called by SQLite to request a value from |
+** the row that the supplied cursor currently points to. |
+** |
+** If: |
+** |
+** (iCol < p->nColumn) -> The value of the iCol'th user column. |
+** (iCol == p->nColumn) -> Magic column with the same name as the table. |
+** (iCol == p->nColumn+1) -> Docid column |
+** (iCol == p->nColumn+2) -> Langid column |
+*/ |
+static int fts3ColumnMethod( |
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ |
+ int iCol /* Index of column to read value from */ |
+){ |
+ int rc = SQLITE_OK; /* Return Code */ |
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; |
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab; |
+ |
+ /* The column value supplied by SQLite must be in range. */ |
+ assert( iCol>=0 && iCol<=p->nColumn+2 ); |
+ |
+ if( iCol==p->nColumn+1 ){ |
+ /* This call is a request for the "docid" column. Since "docid" is an |
+ ** alias for "rowid", use the xRowid() method to obtain the value. |
+ */ |
+ sqlite3_result_int64(pCtx, pCsr->iPrevId); |
+ }else if( iCol==p->nColumn ){ |
+ /* The extra column whose name is the same as the table. |
+ ** Return a blob which is a pointer to the cursor. */ |
+ sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); |
+ }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ |
+ sqlite3_result_int64(pCtx, pCsr->iLangid); |
+ }else{ |
+ /* The requested column is either a user column (one that contains |
+ ** indexed data), or the language-id column. */ |
+ rc = fts3CursorSeek(0, pCsr); |
+ |
+ if( rc==SQLITE_OK ){ |
+ if( iCol==p->nColumn+2 ){ |
+ int iLangid = 0; |
+ if( p->zLanguageid ){ |
+ iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); |
+ } |
+ sqlite3_result_int(pCtx, iLangid); |
+ }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ |
+ sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); |
+ } |
+ } |
+ } |
+ |
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); |
+ return rc; |
+} |
+ |
+/* |
+** This function is the implementation of the xUpdate callback used by |
+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be |
+** inserted, updated or deleted. |
+*/ |
+static int fts3UpdateMethod( |
+ sqlite3_vtab *pVtab, /* Virtual table handle */ |
+ int nArg, /* Size of argument array */ |
+ sqlite3_value **apVal, /* Array of arguments */ |
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ |
+){ |
+ return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); |
+} |
+ |
+/* |
+** Implementation of xSync() method. Flush the contents of the pending-terms |
+** hash-table to the database. |
+*/ |
+static int fts3SyncMethod(sqlite3_vtab *pVtab){ |
+ |
+ /* Following an incremental-merge operation, assuming that the input |
+ ** segments are not completely consumed (the usual case), they are updated |
+ ** in place to remove the entries that have already been merged. This |
+ ** involves updating the leaf block that contains the smallest unmerged |
+ ** entry and each block (if any) between the leaf and the root node. So |
+ ** if the height of the input segment b-trees is N, and input segments |
+ ** are merged eight at a time, updating the input segments at the end |
+ ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually |
+ ** small - often between 0 and 2. So the overhead of the incremental |
+ ** merge is somewhere between 8 and 24 blocks. To avoid this overhead |
+ ** dwarfing the actual productive work accomplished, the incremental merge |
+ ** is only attempted if it will write at least 64 leaf blocks. Hence |
+ ** nMinMerge. |
+ ** |
+ ** Of course, updating the input segments also involves deleting a bunch |
+ ** of blocks from the segments table. But this is not considered overhead |
+ ** as it would also be required by a crisis-merge that used the same input |
+ ** segments. |
+ */ |
+ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ |
+ |
+ Fts3Table *p = (Fts3Table*)pVtab; |
+ int rc = sqlite3Fts3PendingTermsFlush(p); |
+ |
+ if( rc==SQLITE_OK |
+ && p->nLeafAdd>(nMinMerge/16) |
+ && p->nAutoincrmerge && p->nAutoincrmerge!=0xff |
+ ){ |
+ int mxLevel = 0; /* Maximum relative level value in db */ |
+ int A; /* Incr-merge parameter A */ |
+ |
+ rc = sqlite3Fts3MaxLevel(p, &mxLevel); |
+ assert( rc==SQLITE_OK || mxLevel==0 ); |
+ A = p->nLeafAdd * mxLevel; |
+ A += (A/2); |
+ if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); |
+ } |
+ sqlite3Fts3SegmentsClose(p); |
+ return rc; |
+} |
+ |
+/* |
+** If it is currently unknown whether or not the FTS table has an %_stat |
+** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat |
+** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code |
+** if an error occurs. |
+*/ |
+static int fts3SetHasStat(Fts3Table *p){ |
+ int rc = SQLITE_OK; |
+ if( p->bHasStat==2 ){ |
+ const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; |
+ char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); |
+ if( zSql ){ |
+ sqlite3_stmt *pStmt = 0; |
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); |
+ rc = sqlite3_finalize(pStmt); |
+ if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat; |
+ } |
+ sqlite3_free(zSql); |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of xBegin() method. |
+*/ |
+static int fts3BeginMethod(sqlite3_vtab *pVtab){ |
+ Fts3Table *p = (Fts3Table*)pVtab; |
+ UNUSED_PARAMETER(pVtab); |
+ assert( p->pSegments==0 ); |
+ assert( p->nPendingData==0 ); |
+ assert( p->inTransaction!=1 ); |
+ TESTONLY( p->inTransaction = 1 ); |
+ TESTONLY( p->mxSavepoint = -1; ); |
+ p->nLeafAdd = 0; |
+ return fts3SetHasStat(p); |
+} |
+ |
+/* |
+** Implementation of xCommit() method. This is a no-op. The contents of |
+** the pending-terms hash-table have already been flushed into the database |
+** by fts3SyncMethod(). |
+*/ |
+static int fts3CommitMethod(sqlite3_vtab *pVtab){ |
+ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); |
+ UNUSED_PARAMETER(pVtab); |
+ assert( p->nPendingData==0 ); |
+ assert( p->inTransaction!=0 ); |
+ assert( p->pSegments==0 ); |
+ TESTONLY( p->inTransaction = 0 ); |
+ TESTONLY( p->mxSavepoint = -1; ); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Implementation of xRollback(). Discard the contents of the pending-terms |
+** hash-table. Any changes made to the database are reverted by SQLite. |
+*/ |
+static int fts3RollbackMethod(sqlite3_vtab *pVtab){ |
+ Fts3Table *p = (Fts3Table*)pVtab; |
+ sqlite3Fts3PendingTermsClear(p); |
+ assert( p->inTransaction!=0 ); |
+ TESTONLY( p->inTransaction = 0 ); |
+ TESTONLY( p->mxSavepoint = -1; ); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** When called, *ppPoslist must point to the byte immediately following the |
+** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function |
+** moves *ppPoslist so that it instead points to the first byte of the |
+** same position list. |
+*/ |
+static void fts3ReversePoslist(char *pStart, char **ppPoslist){ |
+ char *p = &(*ppPoslist)[-2]; |
+ char c = 0; |
+ |
+ /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ |
+ while( p>pStart && (c=*p--)==0 ); |
+ |
+ /* Search backwards for a varint with value zero (the end of the previous |
+ ** poslist). This is an 0x00 byte preceded by some byte that does not |
+ ** have the 0x80 bit set. */ |
+ while( p>pStart && (*p & 0x80) | c ){ |
+ c = *p--; |
+ } |
+ assert( p==pStart || c==0 ); |
+ |
+ /* At this point p points to that preceding byte without the 0x80 bit |
+ ** set. So to find the start of the poslist, skip forward 2 bytes then |
+ ** over a varint. |
+ ** |
+ ** Normally. The other case is that p==pStart and the poslist to return |
+ ** is the first in the doclist. In this case do not skip forward 2 bytes. |
+ ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) |
+ ** is required for cases where the first byte of a doclist and the |
+ ** doclist is empty. For example, if the first docid is 10, a doclist |
+ ** that begins with: |
+ ** |
+ ** 0x0A 0x00 <next docid delta varint> |
+ */ |
+ if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } |
+ while( *p++&0x80 ); |
+ *ppPoslist = p; |
+} |
+ |
+/* |
+** Helper function used by the implementation of the overloaded snippet(), |
+** offsets() and optimize() SQL functions. |
+** |
+** If the value passed as the third argument is a blob of size |
+** sizeof(Fts3Cursor*), then the blob contents are copied to the |
+** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error |
+** message is written to context pContext and SQLITE_ERROR returned. The |
+** string passed via zFunc is used as part of the error message. |
+*/ |
+static int fts3FunctionArg( |
+ sqlite3_context *pContext, /* SQL function call context */ |
+ const char *zFunc, /* Function name */ |
+ sqlite3_value *pVal, /* argv[0] passed to function */ |
+ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ |
+){ |
+ Fts3Cursor *pRet; |
+ if( sqlite3_value_type(pVal)!=SQLITE_BLOB |
+ || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) |
+ ){ |
+ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); |
+ sqlite3_result_error(pContext, zErr, -1); |
+ sqlite3_free(zErr); |
+ return SQLITE_ERROR; |
+ } |
+ memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); |
+ *ppCsr = pRet; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Implementation of the snippet() function for FTS3 |
+*/ |
+static void fts3SnippetFunc( |
+ sqlite3_context *pContext, /* SQLite function call context */ |
+ int nVal, /* Size of apVal[] array */ |
+ sqlite3_value **apVal /* Array of arguments */ |
+){ |
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ |
+ const char *zStart = "<b>"; |
+ const char *zEnd = "</b>"; |
+ const char *zEllipsis = "<b>...</b>"; |
+ int iCol = -1; |
+ int nToken = 15; /* Default number of tokens in snippet */ |
+ |
+ /* There must be at least one argument passed to this function (otherwise |
+ ** the non-overloaded version would have been called instead of this one). |
+ */ |
+ assert( nVal>=1 ); |
+ |
+ if( nVal>6 ){ |
+ sqlite3_result_error(pContext, |
+ "wrong number of arguments to function snippet()", -1); |
+ return; |
+ } |
+ if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; |
+ |
+ switch( nVal ){ |
+ case 6: nToken = sqlite3_value_int(apVal[5]); |
+ case 5: iCol = sqlite3_value_int(apVal[4]); |
+ case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); |
+ case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); |
+ case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); |
+ } |
+ if( !zEllipsis || !zEnd || !zStart ){ |
+ sqlite3_result_error_nomem(pContext); |
+ }else if( nToken==0 ){ |
+ sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); |
+ }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ |
+ sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); |
+ } |
+} |
+ |
+/* |
+** Implementation of the offsets() function for FTS3 |
+*/ |
+static void fts3OffsetsFunc( |
+ sqlite3_context *pContext, /* SQLite function call context */ |
+ int nVal, /* Size of argument array */ |
+ sqlite3_value **apVal /* Array of arguments */ |
+){ |
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ |
+ |
+ UNUSED_PARAMETER(nVal); |
+ |
+ assert( nVal==1 ); |
+ if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; |
+ assert( pCsr ); |
+ if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ |
+ sqlite3Fts3Offsets(pContext, pCsr); |
+ } |
+} |
+ |
+/* |
+** Implementation of the special optimize() function for FTS3. This |
+** function merges all segments in the database to a single segment. |
+** Example usage is: |
+** |
+** SELECT optimize(t) FROM t LIMIT 1; |
+** |
+** where 't' is the name of an FTS3 table. |
+*/ |
+static void fts3OptimizeFunc( |
+ sqlite3_context *pContext, /* SQLite function call context */ |
+ int nVal, /* Size of argument array */ |
+ sqlite3_value **apVal /* Array of arguments */ |
+){ |
+ int rc; /* Return code */ |
+ Fts3Table *p; /* Virtual table handle */ |
+ Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ |
+ |
+ UNUSED_PARAMETER(nVal); |
+ |
+ assert( nVal==1 ); |
+ if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; |
+ p = (Fts3Table *)pCursor->base.pVtab; |
+ assert( p ); |
+ |
+ rc = sqlite3Fts3Optimize(p); |
+ |
+ switch( rc ){ |
+ case SQLITE_OK: |
+ sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); |
+ break; |
+ case SQLITE_DONE: |
+ sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); |
+ break; |
+ default: |
+ sqlite3_result_error_code(pContext, rc); |
+ break; |
+ } |
+} |
+ |
+/* |
+** Implementation of the matchinfo() function for FTS3 |
+*/ |
+static void fts3MatchinfoFunc( |
+ sqlite3_context *pContext, /* SQLite function call context */ |
+ int nVal, /* Size of argument array */ |
+ sqlite3_value **apVal /* Array of arguments */ |
+){ |
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ |
+ assert( nVal==1 || nVal==2 ); |
+ if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ |
+ const char *zArg = 0; |
+ if( nVal>1 ){ |
+ zArg = (const char *)sqlite3_value_text(apVal[1]); |
+ } |
+ sqlite3Fts3Matchinfo(pContext, pCsr, zArg); |
+ } |
+} |
+ |
+/* |
+** This routine implements the xFindFunction method for the FTS3 |
+** virtual table. |
+*/ |
+static int fts3FindFunctionMethod( |
+ sqlite3_vtab *pVtab, /* Virtual table handle */ |
+ int nArg, /* Number of SQL function arguments */ |
+ const char *zName, /* Name of SQL function */ |
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ |
+ void **ppArg /* Unused */ |
+){ |
+ struct Overloaded { |
+ const char *zName; |
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); |
+ } aOverload[] = { |
+ { "snippet", fts3SnippetFunc }, |
+ { "offsets", fts3OffsetsFunc }, |
+ { "optimize", fts3OptimizeFunc }, |
+ { "matchinfo", fts3MatchinfoFunc }, |
+ }; |
+ int i; /* Iterator variable */ |
+ |
+ UNUSED_PARAMETER(pVtab); |
+ UNUSED_PARAMETER(nArg); |
+ UNUSED_PARAMETER(ppArg); |
+ |
+ for(i=0; i<SizeofArray(aOverload); i++){ |
+ if( strcmp(zName, aOverload[i].zName)==0 ){ |
+ *pxFunc = aOverload[i].xFunc; |
+ return 1; |
+ } |
+ } |
+ |
+ /* No function of the specified name was found. Return 0. */ |
+ return 0; |
+} |
+ |
+/* |
+** Implementation of FTS3 xRename method. Rename an fts3 table. |
+*/ |
+static int fts3RenameMethod( |
+ sqlite3_vtab *pVtab, /* Virtual table handle */ |
+ const char *zName /* New name of table */ |
+){ |
+ Fts3Table *p = (Fts3Table *)pVtab; |
+ sqlite3 *db = p->db; /* Database connection */ |
+ int rc; /* Return Code */ |
+ |
+ /* At this point it must be known if the %_stat table exists or not. |
+ ** So bHasStat may not be 2. */ |
+ rc = fts3SetHasStat(p); |
+ |
+ /* As it happens, the pending terms table is always empty here. This is |
+ ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction |
+ ** always opens a savepoint transaction. And the xSavepoint() method |
+ ** flushes the pending terms table. But leave the (no-op) call to |
+ ** PendingTermsFlush() in in case that changes. |
+ */ |
+ assert( p->nPendingData==0 ); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3PendingTermsFlush(p); |
+ } |
+ |
+ if( p->zContentTbl==0 ){ |
+ fts3DbExec(&rc, db, |
+ "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", |
+ p->zDb, p->zName, zName |
+ ); |
+ } |
+ |
+ if( p->bHasDocsize ){ |
+ fts3DbExec(&rc, db, |
+ "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", |
+ p->zDb, p->zName, zName |
+ ); |
+ } |
+ if( p->bHasStat ){ |
+ fts3DbExec(&rc, db, |
+ "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", |
+ p->zDb, p->zName, zName |
+ ); |
+ } |
+ fts3DbExec(&rc, db, |
+ "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", |
+ p->zDb, p->zName, zName |
+ ); |
+ fts3DbExec(&rc, db, |
+ "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", |
+ p->zDb, p->zName, zName |
+ ); |
+ return rc; |
+} |
+ |
+/* |
+** The xSavepoint() method. |
+** |
+** Flush the contents of the pending-terms table to disk. |
+*/ |
+static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
+ int rc = SQLITE_OK; |
+ UNUSED_PARAMETER(iSavepoint); |
+ assert( ((Fts3Table *)pVtab)->inTransaction ); |
+ assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); |
+ TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); |
+ if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ |
+ rc = fts3SyncMethod(pVtab); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** The xRelease() method. |
+** |
+** This is a no-op. |
+*/ |
+static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
+ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); |
+ UNUSED_PARAMETER(iSavepoint); |
+ UNUSED_PARAMETER(pVtab); |
+ assert( p->inTransaction ); |
+ assert( p->mxSavepoint >= iSavepoint ); |
+ TESTONLY( p->mxSavepoint = iSavepoint-1 ); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The xRollbackTo() method. |
+** |
+** Discard the contents of the pending terms table. |
+*/ |
+static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
+ Fts3Table *p = (Fts3Table*)pVtab; |
+ UNUSED_PARAMETER(iSavepoint); |
+ assert( p->inTransaction ); |
+ assert( p->mxSavepoint >= iSavepoint ); |
+ TESTONLY( p->mxSavepoint = iSavepoint ); |
+ sqlite3Fts3PendingTermsClear(p); |
+ return SQLITE_OK; |
+} |
+ |
+static const sqlite3_module fts3Module = { |
+ /* iVersion */ 2, |
+ /* xCreate */ fts3CreateMethod, |
+ /* xConnect */ fts3ConnectMethod, |
+ /* xBestIndex */ fts3BestIndexMethod, |
+ /* xDisconnect */ fts3DisconnectMethod, |
+ /* xDestroy */ fts3DestroyMethod, |
+ /* xOpen */ fts3OpenMethod, |
+ /* xClose */ fts3CloseMethod, |
+ /* xFilter */ fts3FilterMethod, |
+ /* xNext */ fts3NextMethod, |
+ /* xEof */ fts3EofMethod, |
+ /* xColumn */ fts3ColumnMethod, |
+ /* xRowid */ fts3RowidMethod, |
+ /* xUpdate */ fts3UpdateMethod, |
+ /* xBegin */ fts3BeginMethod, |
+ /* xSync */ fts3SyncMethod, |
+ /* xCommit */ fts3CommitMethod, |
+ /* xRollback */ fts3RollbackMethod, |
+ /* xFindFunction */ fts3FindFunctionMethod, |
+ /* xRename */ fts3RenameMethod, |
+ /* xSavepoint */ fts3SavepointMethod, |
+ /* xRelease */ fts3ReleaseMethod, |
+ /* xRollbackTo */ fts3RollbackToMethod, |
+}; |
+ |
+/* |
+** This function is registered as the module destructor (called when an |
+** FTS3 enabled database connection is closed). It frees the memory |
+** allocated for the tokenizer hash table. |
+*/ |
+static void hashDestroy(void *p){ |
+ Fts3Hash *pHash = (Fts3Hash *)p; |
+ sqlite3Fts3HashClear(pHash); |
+ sqlite3_free(pHash); |
+} |
+ |
+/* |
+** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are |
+** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c |
+** respectively. The following three forward declarations are for functions |
+** declared in these files used to retrieve the respective implementations. |
+** |
+** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed |
+** to by the argument to point to the "simple" tokenizer implementation. |
+** And so on. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); |
+SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); |
+#ifndef SQLITE_DISABLE_FTS3_UNICODE |
+SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); |
+#endif |
+#ifdef SQLITE_ENABLE_ICU |
+SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); |
+#endif |
+ |
+/* |
+** Initialize the fts3 extension. If this extension is built as part |
+** of the sqlite library, then this function is called directly by |
+** SQLite. If fts3 is built as a dynamically loadable extension, this |
+** function is called by the sqlite3_extension_init() entry point. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ |
+ int rc = SQLITE_OK; |
+ Fts3Hash *pHash = 0; |
+ const sqlite3_tokenizer_module *pSimple = 0; |
+ const sqlite3_tokenizer_module *pPorter = 0; |
+#ifndef SQLITE_DISABLE_FTS3_UNICODE |
+ const sqlite3_tokenizer_module *pUnicode = 0; |
+#endif |
+ |
+#ifdef SQLITE_ENABLE_ICU |
+ const sqlite3_tokenizer_module *pIcu = 0; |
+ sqlite3Fts3IcuTokenizerModule(&pIcu); |
+#endif |
+ |
+#ifndef SQLITE_DISABLE_FTS3_UNICODE |
+ sqlite3Fts3UnicodeTokenizer(&pUnicode); |
+#endif |
+ |
+#ifdef SQLITE_TEST |
+ rc = sqlite3Fts3InitTerm(db); |
+ if( rc!=SQLITE_OK ) return rc; |
+#endif |
+ |
+ rc = sqlite3Fts3InitAux(db); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ sqlite3Fts3SimpleTokenizerModule(&pSimple); |
+ sqlite3Fts3PorterTokenizerModule(&pPorter); |
+ |
+ /* Allocate and initialize the hash-table used to store tokenizers. */ |
+ pHash = sqlite3_malloc(sizeof(Fts3Hash)); |
+ if( !pHash ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); |
+ } |
+ |
+ /* Load the built-in tokenizers into the hash table */ |
+ if( rc==SQLITE_OK ){ |
+ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) |
+ || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) |
+ |
+#ifndef SQLITE_DISABLE_FTS3_UNICODE |
+ || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) |
+#endif |
+#ifdef SQLITE_ENABLE_ICU |
+ || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) |
+#endif |
+ ){ |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ |
+#ifdef SQLITE_TEST |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3ExprInitTestInterface(db); |
+ } |
+#endif |
+ |
+ /* Create the virtual table wrapper around the hash-table and overload |
+ ** the two scalar functions. If this is successful, register the |
+ ** module with sqlite. |
+ */ |
+ if( SQLITE_OK==rc |
+#if CHROMIUM_FTS3_CHANGES && !SQLITE_TEST |
+ /* fts3_tokenizer() disabled for security reasons. */ |
+#else |
+ && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) |
+#endif |
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) |
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) |
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) |
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) |
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) |
+ ){ |
+ rc = sqlite3_create_module_v2( |
+ db, "fts3", &fts3Module, (void *)pHash, hashDestroy |
+ ); |
+#if CHROMIUM_FTS3_CHANGES && !SQLITE_TEST |
+ /* Disable fts4 and tokenizer vtab pending review. */ |
+#else |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_create_module_v2( |
+ db, "fts4", &fts3Module, (void *)pHash, 0 |
+ ); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3InitTok(db, (void *)pHash); |
+ } |
+#endif |
+ return rc; |
+ } |
+ |
+ |
+ /* An error has occurred. Delete the hash table and return the error code. */ |
+ assert( rc!=SQLITE_OK ); |
+ if( pHash ){ |
+ sqlite3Fts3HashClear(pHash); |
+ sqlite3_free(pHash); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Allocate an Fts3MultiSegReader for each token in the expression headed |
+** by pExpr. |
+** |
+** An Fts3SegReader object is a cursor that can seek or scan a range of |
+** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple |
+** Fts3SegReader objects internally to provide an interface to seek or scan |
+** within the union of all segments of a b-tree. Hence the name. |
+** |
+** If the allocated Fts3MultiSegReader just seeks to a single entry in a |
+** segment b-tree (if the term is not a prefix or it is a prefix for which |
+** there exists prefix b-tree of the right length) then it may be traversed |
+** and merged incrementally. Otherwise, it has to be merged into an in-memory |
+** doclist and then traversed. |
+*/ |
+static void fts3EvalAllocateReaders( |
+ Fts3Cursor *pCsr, /* FTS cursor handle */ |
+ Fts3Expr *pExpr, /* Allocate readers for this expression */ |
+ int *pnToken, /* OUT: Total number of tokens in phrase. */ |
+ int *pnOr, /* OUT: Total number of OR nodes in expr. */ |
+ int *pRc /* IN/OUT: Error code */ |
+){ |
+ if( pExpr && SQLITE_OK==*pRc ){ |
+ if( pExpr->eType==FTSQUERY_PHRASE ){ |
+ int i; |
+ int nToken = pExpr->pPhrase->nToken; |
+ *pnToken += nToken; |
+ for(i=0; i<nToken; i++){ |
+ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; |
+ int rc = fts3TermSegReaderCursor(pCsr, |
+ pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr |
+ ); |
+ if( rc!=SQLITE_OK ){ |
+ *pRc = rc; |
+ return; |
+ } |
+ } |
+ assert( pExpr->pPhrase->iDoclistToken==0 ); |
+ pExpr->pPhrase->iDoclistToken = -1; |
+ }else{ |
+ *pnOr += (pExpr->eType==FTSQUERY_OR); |
+ fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); |
+ fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); |
+ } |
+ } |
+} |
+ |
+/* |
+** Arguments pList/nList contain the doclist for token iToken of phrase p. |
+** It is merged into the main doclist stored in p->doclist.aAll/nAll. |
+** |
+** This function assumes that pList points to a buffer allocated using |
+** sqlite3_malloc(). This function takes responsibility for eventually |
+** freeing the buffer. |
+** |
+** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. |
+*/ |
+static int fts3EvalPhraseMergeToken( |
+ Fts3Table *pTab, /* FTS Table pointer */ |
+ Fts3Phrase *p, /* Phrase to merge pList/nList into */ |
+ int iToken, /* Token pList/nList corresponds to */ |
+ char *pList, /* Pointer to doclist */ |
+ int nList /* Number of bytes in pList */ |
+){ |
+ int rc = SQLITE_OK; |
+ assert( iToken!=p->iDoclistToken ); |
+ |
+ if( pList==0 ){ |
+ sqlite3_free(p->doclist.aAll); |
+ p->doclist.aAll = 0; |
+ p->doclist.nAll = 0; |
+ } |
+ |
+ else if( p->iDoclistToken<0 ){ |
+ p->doclist.aAll = pList; |
+ p->doclist.nAll = nList; |
+ } |
+ |
+ else if( p->doclist.aAll==0 ){ |
+ sqlite3_free(pList); |
+ } |
+ |
+ else { |
+ char *pLeft; |
+ char *pRight; |
+ int nLeft; |
+ int nRight; |
+ int nDiff; |
+ |
+ if( p->iDoclistToken<iToken ){ |
+ pLeft = p->doclist.aAll; |
+ nLeft = p->doclist.nAll; |
+ pRight = pList; |
+ nRight = nList; |
+ nDiff = iToken - p->iDoclistToken; |
+ }else{ |
+ pRight = p->doclist.aAll; |
+ nRight = p->doclist.nAll; |
+ pLeft = pList; |
+ nLeft = nList; |
+ nDiff = p->iDoclistToken - iToken; |
+ } |
+ |
+ rc = fts3DoclistPhraseMerge( |
+ pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight |
+ ); |
+ sqlite3_free(pLeft); |
+ p->doclist.aAll = pRight; |
+ p->doclist.nAll = nRight; |
+ } |
+ |
+ if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; |
+ return rc; |
+} |
+ |
+/* |
+** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist |
+** does not take deferred tokens into account. |
+** |
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. |
+*/ |
+static int fts3EvalPhraseLoad( |
+ Fts3Cursor *pCsr, /* FTS Cursor handle */ |
+ Fts3Phrase *p /* Phrase object */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int iToken; |
+ int rc = SQLITE_OK; |
+ |
+ for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){ |
+ Fts3PhraseToken *pToken = &p->aToken[iToken]; |
+ assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); |
+ |
+ if( pToken->pSegcsr ){ |
+ int nThis = 0; |
+ char *pThis = 0; |
+ rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); |
+ } |
+ } |
+ assert( pToken->pSegcsr==0 ); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This function is called on each phrase after the position lists for |
+** any deferred tokens have been loaded into memory. It updates the phrases |
+** current position list to include only those positions that are really |
+** instances of the phrase (after considering deferred tokens). If this |
+** means that the phrase does not appear in the current row, doclist.pList |
+** and doclist.nList are both zeroed. |
+** |
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. |
+*/ |
+static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ |
+ int iToken; /* Used to iterate through phrase tokens */ |
+ char *aPoslist = 0; /* Position list for deferred tokens */ |
+ int nPoslist = 0; /* Number of bytes in aPoslist */ |
+ int iPrev = -1; /* Token number of previous deferred token */ |
+ |
+ assert( pPhrase->doclist.bFreeList==0 ); |
+ |
+ for(iToken=0; iToken<pPhrase->nToken; iToken++){ |
+ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; |
+ Fts3DeferredToken *pDeferred = pToken->pDeferred; |
+ |
+ if( pDeferred ){ |
+ char *pList = 0; |
+ int nList = 0; |
+ int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ if( pList==0 ){ |
+ sqlite3_free(aPoslist); |
+ pPhrase->doclist.pList = 0; |
+ pPhrase->doclist.nList = 0; |
+ return SQLITE_OK; |
+ |
+ }else if( aPoslist==0 ){ |
+ aPoslist = pList; |
+ nPoslist = nList; |
+ |
+ }else{ |
+ char *aOut = pList; |
+ char *p1 = aPoslist; |
+ char *p2 = aOut; |
+ |
+ assert( iPrev>=0 ); |
+ fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); |
+ sqlite3_free(aPoslist); |
+ aPoslist = pList; |
+ nPoslist = (int)(aOut - aPoslist); |
+ if( nPoslist==0 ){ |
+ sqlite3_free(aPoslist); |
+ pPhrase->doclist.pList = 0; |
+ pPhrase->doclist.nList = 0; |
+ return SQLITE_OK; |
+ } |
+ } |
+ iPrev = iToken; |
+ } |
+ } |
+ |
+ if( iPrev>=0 ){ |
+ int nMaxUndeferred = pPhrase->iDoclistToken; |
+ if( nMaxUndeferred<0 ){ |
+ pPhrase->doclist.pList = aPoslist; |
+ pPhrase->doclist.nList = nPoslist; |
+ pPhrase->doclist.iDocid = pCsr->iPrevId; |
+ pPhrase->doclist.bFreeList = 1; |
+ }else{ |
+ int nDistance; |
+ char *p1; |
+ char *p2; |
+ char *aOut; |
+ |
+ if( nMaxUndeferred>iPrev ){ |
+ p1 = aPoslist; |
+ p2 = pPhrase->doclist.pList; |
+ nDistance = nMaxUndeferred - iPrev; |
+ }else{ |
+ p1 = pPhrase->doclist.pList; |
+ p2 = aPoslist; |
+ nDistance = iPrev - nMaxUndeferred; |
+ } |
+ |
+ aOut = (char *)sqlite3_malloc(nPoslist+8); |
+ if( !aOut ){ |
+ sqlite3_free(aPoslist); |
+ return SQLITE_NOMEM; |
+ } |
+ |
+ pPhrase->doclist.pList = aOut; |
+ if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ |
+ pPhrase->doclist.bFreeList = 1; |
+ pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); |
+ }else{ |
+ sqlite3_free(aOut); |
+ pPhrase->doclist.pList = 0; |
+ pPhrase->doclist.nList = 0; |
+ } |
+ sqlite3_free(aPoslist); |
+ } |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Maximum number of tokens a phrase may have to be considered for the |
+** incremental doclists strategy. |
+*/ |
+#define MAX_INCR_PHRASE_TOKENS 4 |
+ |
+/* |
+** This function is called for each Fts3Phrase in a full-text query |
+** expression to initialize the mechanism for returning rows. Once this |
+** function has been called successfully on an Fts3Phrase, it may be |
+** used with fts3EvalPhraseNext() to iterate through the matching docids. |
+** |
+** If parameter bOptOk is true, then the phrase may (or may not) use the |
+** incremental loading strategy. Otherwise, the entire doclist is loaded into |
+** memory within this call. |
+** |
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. |
+*/ |
+static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int rc = SQLITE_OK; /* Error code */ |
+ int i; |
+ |
+ /* Determine if doclists may be loaded from disk incrementally. This is |
+ ** possible if the bOptOk argument is true, the FTS doclists will be |
+ ** scanned in forward order, and the phrase consists of |
+ ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" |
+ ** tokens or prefix tokens that cannot use a prefix-index. */ |
+ int bHaveIncr = 0; |
+ int bIncrOk = (bOptOk |
+ && pCsr->bDesc==pTab->bDescIdx |
+ && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 |
+#ifdef SQLITE_TEST |
+ && pTab->bNoIncrDoclist==0 |
+#endif |
+ ); |
+ for(i=0; bIncrOk==1 && i<p->nToken; i++){ |
+ Fts3PhraseToken *pToken = &p->aToken[i]; |
+ if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ |
+ bIncrOk = 0; |
+ } |
+ if( pToken->pSegcsr ) bHaveIncr = 1; |
+ } |
+ |
+ if( bIncrOk && bHaveIncr ){ |
+ /* Use the incremental approach. */ |
+ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); |
+ for(i=0; rc==SQLITE_OK && i<p->nToken; i++){ |
+ Fts3PhraseToken *pToken = &p->aToken[i]; |
+ Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; |
+ if( pSegcsr ){ |
+ rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); |
+ } |
+ } |
+ p->bIncr = 1; |
+ }else{ |
+ /* Load the full doclist for the phrase into memory. */ |
+ rc = fts3EvalPhraseLoad(pCsr, p); |
+ p->bIncr = 0; |
+ } |
+ |
+ assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); |
+ return rc; |
+} |
+ |
+/* |
+** This function is used to iterate backwards (from the end to start) |
+** through doclists. It is used by this module to iterate through phrase |
+** doclists in reverse and by the fts3_write.c module to iterate through |
+** pending-terms lists when writing to databases with "order=desc". |
+** |
+** The doclist may be sorted in ascending (parameter bDescIdx==0) or |
+** descending (parameter bDescIdx==1) order of docid. Regardless, this |
+** function iterates from the end of the doclist to the beginning. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( |
+ int bDescIdx, /* True if the doclist is desc */ |
+ char *aDoclist, /* Pointer to entire doclist */ |
+ int nDoclist, /* Length of aDoclist in bytes */ |
+ char **ppIter, /* IN/OUT: Iterator pointer */ |
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ |
+ int *pnList, /* OUT: List length pointer */ |
+ u8 *pbEof /* OUT: End-of-file flag */ |
+){ |
+ char *p = *ppIter; |
+ |
+ assert( nDoclist>0 ); |
+ assert( *pbEof==0 ); |
+ assert( p || *piDocid==0 ); |
+ assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); |
+ |
+ if( p==0 ){ |
+ sqlite3_int64 iDocid = 0; |
+ char *pNext = 0; |
+ char *pDocid = aDoclist; |
+ char *pEnd = &aDoclist[nDoclist]; |
+ int iMul = 1; |
+ |
+ while( pDocid<pEnd ){ |
+ sqlite3_int64 iDelta; |
+ pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta); |
+ iDocid += (iMul * iDelta); |
+ pNext = pDocid; |
+ fts3PoslistCopy(0, &pDocid); |
+ while( pDocid<pEnd && *pDocid==0 ) pDocid++; |
+ iMul = (bDescIdx ? -1 : 1); |
+ } |
+ |
+ *pnList = (int)(pEnd - pNext); |
+ *ppIter = pNext; |
+ *piDocid = iDocid; |
+ }else{ |
+ int iMul = (bDescIdx ? -1 : 1); |
+ sqlite3_int64 iDelta; |
+ fts3GetReverseVarint(&p, aDoclist, &iDelta); |
+ *piDocid -= (iMul * iDelta); |
+ |
+ if( p==aDoclist ){ |
+ *pbEof = 1; |
+ }else{ |
+ char *pSave = p; |
+ fts3ReversePoslist(aDoclist, &p); |
+ *pnList = (int)(pSave - p); |
+ } |
+ *ppIter = p; |
+ } |
+} |
+ |
+/* |
+** Iterate forwards through a doclist. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3DoclistNext( |
+ int bDescIdx, /* True if the doclist is desc */ |
+ char *aDoclist, /* Pointer to entire doclist */ |
+ int nDoclist, /* Length of aDoclist in bytes */ |
+ char **ppIter, /* IN/OUT: Iterator pointer */ |
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ |
+ u8 *pbEof /* OUT: End-of-file flag */ |
+){ |
+ char *p = *ppIter; |
+ |
+ assert( nDoclist>0 ); |
+ assert( *pbEof==0 ); |
+ assert( p || *piDocid==0 ); |
+ assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); |
+ |
+ if( p==0 ){ |
+ p = aDoclist; |
+ p += sqlite3Fts3GetVarint(p, piDocid); |
+ }else{ |
+ fts3PoslistCopy(0, &p); |
+ while( p<&aDoclist[nDoclist] && *p==0 ) p++; |
+ if( p>=&aDoclist[nDoclist] ){ |
+ *pbEof = 1; |
+ }else{ |
+ sqlite3_int64 iVar; |
+ p += sqlite3Fts3GetVarint(p, &iVar); |
+ *piDocid += ((bDescIdx ? -1 : 1) * iVar); |
+ } |
+ } |
+ |
+ *ppIter = p; |
+} |
+ |
+/* |
+** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof |
+** to true if EOF is reached. |
+*/ |
+static void fts3EvalDlPhraseNext( |
+ Fts3Table *pTab, |
+ Fts3Doclist *pDL, |
+ u8 *pbEof |
+){ |
+ char *pIter; /* Used to iterate through aAll */ |
+ char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ |
+ |
+ if( pDL->pNextDocid ){ |
+ pIter = pDL->pNextDocid; |
+ }else{ |
+ pIter = pDL->aAll; |
+ } |
+ |
+ if( pIter>=pEnd ){ |
+ /* We have already reached the end of this doclist. EOF. */ |
+ *pbEof = 1; |
+ }else{ |
+ sqlite3_int64 iDelta; |
+ pIter += sqlite3Fts3GetVarint(pIter, &iDelta); |
+ if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ |
+ pDL->iDocid += iDelta; |
+ }else{ |
+ pDL->iDocid -= iDelta; |
+ } |
+ pDL->pList = pIter; |
+ fts3PoslistCopy(0, &pIter); |
+ pDL->nList = (int)(pIter - pDL->pList); |
+ |
+ /* pIter now points just past the 0x00 that terminates the position- |
+ ** list for document pDL->iDocid. However, if this position-list was |
+ ** edited in place by fts3EvalNearTrim(), then pIter may not actually |
+ ** point to the start of the next docid value. The following line deals |
+ ** with this case by advancing pIter past the zero-padding added by |
+ ** fts3EvalNearTrim(). */ |
+ while( pIter<pEnd && *pIter==0 ) pIter++; |
+ |
+ pDL->pNextDocid = pIter; |
+ assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); |
+ *pbEof = 0; |
+ } |
+} |
+ |
+/* |
+** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). |
+*/ |
+typedef struct TokenDoclist TokenDoclist; |
+struct TokenDoclist { |
+ int bIgnore; |
+ sqlite3_int64 iDocid; |
+ char *pList; |
+ int nList; |
+}; |
+ |
+/* |
+** Token pToken is an incrementally loaded token that is part of a |
+** multi-token phrase. Advance it to the next matching document in the |
+** database and populate output variable *p with the details of the new |
+** entry. Or, if the iterator has reached EOF, set *pbEof to true. |
+** |
+** If an error occurs, return an SQLite error code. Otherwise, return |
+** SQLITE_OK. |
+*/ |
+static int incrPhraseTokenNext( |
+ Fts3Table *pTab, /* Virtual table handle */ |
+ Fts3Phrase *pPhrase, /* Phrase to advance token of */ |
+ int iToken, /* Specific token to advance */ |
+ TokenDoclist *p, /* OUT: Docid and doclist for new entry */ |
+ u8 *pbEof /* OUT: True if iterator is at EOF */ |
+){ |
+ int rc = SQLITE_OK; |
+ |
+ if( pPhrase->iDoclistToken==iToken ){ |
+ assert( p->bIgnore==0 ); |
+ assert( pPhrase->aToken[iToken].pSegcsr==0 ); |
+ fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); |
+ p->pList = pPhrase->doclist.pList; |
+ p->nList = pPhrase->doclist.nList; |
+ p->iDocid = pPhrase->doclist.iDocid; |
+ }else{ |
+ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; |
+ assert( pToken->pDeferred==0 ); |
+ assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); |
+ if( pToken->pSegcsr ){ |
+ assert( p->bIgnore==0 ); |
+ rc = sqlite3Fts3MsrIncrNext( |
+ pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList |
+ ); |
+ if( p->pList==0 ) *pbEof = 1; |
+ }else{ |
+ p->bIgnore = 1; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+/* |
+** The phrase iterator passed as the second argument: |
+** |
+** * features at least one token that uses an incremental doclist, and |
+** |
+** * does not contain any deferred tokens. |
+** |
+** Advance it to the next matching documnent in the database and populate |
+** the Fts3Doclist.pList and nList fields. |
+** |
+** If there is no "next" entry and no error occurs, then *pbEof is set to |
+** 1 before returning. Otherwise, if no error occurs and the iterator is |
+** successfully advanced, *pbEof is set to 0. |
+** |
+** If an error occurs, return an SQLite error code. Otherwise, return |
+** SQLITE_OK. |
+*/ |
+static int fts3EvalIncrPhraseNext( |
+ Fts3Cursor *pCsr, /* FTS Cursor handle */ |
+ Fts3Phrase *p, /* Phrase object to advance to next docid */ |
+ u8 *pbEof /* OUT: Set to 1 if EOF */ |
+){ |
+ int rc = SQLITE_OK; |
+ Fts3Doclist *pDL = &p->doclist; |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ u8 bEof = 0; |
+ |
+ /* This is only called if it is guaranteed that the phrase has at least |
+ ** one incremental token. In which case the bIncr flag is set. */ |
+ assert( p->bIncr==1 ); |
+ |
+ if( p->nToken==1 && p->bIncr ){ |
+ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, |
+ &pDL->iDocid, &pDL->pList, &pDL->nList |
+ ); |
+ if( pDL->pList==0 ) bEof = 1; |
+ }else{ |
+ int bDescDoclist = pCsr->bDesc; |
+ struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; |
+ |
+ memset(a, 0, sizeof(a)); |
+ assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); |
+ assert( p->iDoclistToken<MAX_INCR_PHRASE_TOKENS ); |
+ |
+ while( bEof==0 ){ |
+ int bMaxSet = 0; |
+ sqlite3_int64 iMax = 0; /* Largest docid for all iterators */ |
+ int i; /* Used to iterate through tokens */ |
+ |
+ /* Advance the iterator for each token in the phrase once. */ |
+ for(i=0; rc==SQLITE_OK && i<p->nToken && bEof==0; i++){ |
+ rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); |
+ if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ |
+ iMax = a[i].iDocid; |
+ bMaxSet = 1; |
+ } |
+ } |
+ assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); |
+ assert( rc!=SQLITE_OK || bMaxSet ); |
+ |
+ /* Keep advancing iterators until they all point to the same document */ |
+ for(i=0; i<p->nToken; i++){ |
+ while( rc==SQLITE_OK && bEof==0 |
+ && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 |
+ ){ |
+ rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); |
+ if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ |
+ iMax = a[i].iDocid; |
+ i = 0; |
+ } |
+ } |
+ } |
+ |
+ /* Check if the current entries really are a phrase match */ |
+ if( bEof==0 ){ |
+ int nList = 0; |
+ int nByte = a[p->nToken-1].nList; |
+ char *aDoclist = sqlite3_malloc(nByte+1); |
+ if( !aDoclist ) return SQLITE_NOMEM; |
+ memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); |
+ |
+ for(i=0; i<(p->nToken-1); i++){ |
+ if( a[i].bIgnore==0 ){ |
+ char *pL = a[i].pList; |
+ char *pR = aDoclist; |
+ char *pOut = aDoclist; |
+ int nDist = p->nToken-1-i; |
+ int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); |
+ if( res==0 ) break; |
+ nList = (int)(pOut - aDoclist); |
+ } |
+ } |
+ if( i==(p->nToken-1) ){ |
+ pDL->iDocid = iMax; |
+ pDL->pList = aDoclist; |
+ pDL->nList = nList; |
+ pDL->bFreeList = 1; |
+ break; |
+ } |
+ sqlite3_free(aDoclist); |
+ } |
+ } |
+ } |
+ |
+ *pbEof = bEof; |
+ return rc; |
+} |
+ |
+/* |
+** Attempt to move the phrase iterator to point to the next matching docid. |
+** If an error occurs, return an SQLite error code. Otherwise, return |
+** SQLITE_OK. |
+** |
+** If there is no "next" entry and no error occurs, then *pbEof is set to |
+** 1 before returning. Otherwise, if no error occurs and the iterator is |
+** successfully advanced, *pbEof is set to 0. |
+*/ |
+static int fts3EvalPhraseNext( |
+ Fts3Cursor *pCsr, /* FTS Cursor handle */ |
+ Fts3Phrase *p, /* Phrase object to advance to next docid */ |
+ u8 *pbEof /* OUT: Set to 1 if EOF */ |
+){ |
+ int rc = SQLITE_OK; |
+ Fts3Doclist *pDL = &p->doclist; |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ |
+ if( p->bIncr ){ |
+ rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); |
+ }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ |
+ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, |
+ &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof |
+ ); |
+ pDL->pList = pDL->pNextDocid; |
+ }else{ |
+ fts3EvalDlPhraseNext(pTab, pDL, pbEof); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** |
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op. |
+** Otherwise, fts3EvalPhraseStart() is called on all phrases within the |
+** expression. Also the Fts3Expr.bDeferred variable is set to true for any |
+** expressions for which all descendent tokens are deferred. |
+** |
+** If parameter bOptOk is zero, then it is guaranteed that the |
+** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for |
+** each phrase in the expression (subject to deferred token processing). |
+** Or, if bOptOk is non-zero, then one or more tokens within the expression |
+** may be loaded incrementally, meaning doclist.aAll/nAll is not available. |
+** |
+** If an error occurs within this function, *pRc is set to an SQLite error |
+** code before returning. |
+*/ |
+static void fts3EvalStartReaders( |
+ Fts3Cursor *pCsr, /* FTS Cursor handle */ |
+ Fts3Expr *pExpr, /* Expression to initialize phrases in */ |
+ int *pRc /* IN/OUT: Error code */ |
+){ |
+ if( pExpr && SQLITE_OK==*pRc ){ |
+ if( pExpr->eType==FTSQUERY_PHRASE ){ |
+ int nToken = pExpr->pPhrase->nToken; |
+ if( nToken ){ |
+ int i; |
+ for(i=0; i<nToken; i++){ |
+ if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break; |
+ } |
+ pExpr->bDeferred = (i==nToken); |
+ } |
+ *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); |
+ }else{ |
+ fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); |
+ fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); |
+ pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); |
+ } |
+ } |
+} |
+ |
+/* |
+** An array of the following structures is assembled as part of the process |
+** of selecting tokens to defer before the query starts executing (as part |
+** of the xFilter() method). There is one element in the array for each |
+** token in the FTS expression. |
+** |
+** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong |
+** to phrases that are connected only by AND and NEAR operators (not OR or |
+** NOT). When determining tokens to defer, each AND/NEAR cluster is considered |
+** separately. The root of a tokens AND/NEAR cluster is stored in |
+** Fts3TokenAndCost.pRoot. |
+*/ |
+typedef struct Fts3TokenAndCost Fts3TokenAndCost; |
+struct Fts3TokenAndCost { |
+ Fts3Phrase *pPhrase; /* The phrase the token belongs to */ |
+ int iToken; /* Position of token in phrase */ |
+ Fts3PhraseToken *pToken; /* The token itself */ |
+ Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ |
+ int nOvfl; /* Number of overflow pages to load doclist */ |
+ int iCol; /* The column the token must match */ |
+}; |
+ |
+/* |
+** This function is used to populate an allocated Fts3TokenAndCost array. |
+** |
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op. |
+** Otherwise, if an error occurs during execution, *pRc is set to an |
+** SQLite error code. |
+*/ |
+static void fts3EvalTokenCosts( |
+ Fts3Cursor *pCsr, /* FTS Cursor handle */ |
+ Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ |
+ Fts3Expr *pExpr, /* Expression to consider */ |
+ Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ |
+ Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ |
+ int *pRc /* IN/OUT: Error code */ |
+){ |
+ if( *pRc==SQLITE_OK ){ |
+ if( pExpr->eType==FTSQUERY_PHRASE ){ |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ int i; |
+ for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){ |
+ Fts3TokenAndCost *pTC = (*ppTC)++; |
+ pTC->pPhrase = pPhrase; |
+ pTC->iToken = i; |
+ pTC->pRoot = pRoot; |
+ pTC->pToken = &pPhrase->aToken[i]; |
+ pTC->iCol = pPhrase->iColumn; |
+ *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); |
+ } |
+ }else if( pExpr->eType!=FTSQUERY_NOT ){ |
+ assert( pExpr->eType==FTSQUERY_OR |
+ || pExpr->eType==FTSQUERY_AND |
+ || pExpr->eType==FTSQUERY_NEAR |
+ ); |
+ assert( pExpr->pLeft && pExpr->pRight ); |
+ if( pExpr->eType==FTSQUERY_OR ){ |
+ pRoot = pExpr->pLeft; |
+ **ppOr = pRoot; |
+ (*ppOr)++; |
+ } |
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); |
+ if( pExpr->eType==FTSQUERY_OR ){ |
+ pRoot = pExpr->pRight; |
+ **ppOr = pRoot; |
+ (*ppOr)++; |
+ } |
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); |
+ } |
+ } |
+} |
+ |
+/* |
+** Determine the average document (row) size in pages. If successful, |
+** write this value to *pnPage and return SQLITE_OK. Otherwise, return |
+** an SQLite error code. |
+** |
+** The average document size in pages is calculated by first calculating |
+** determining the average size in bytes, B. If B is less than the amount |
+** of data that will fit on a single leaf page of an intkey table in |
+** this database, then the average docsize is 1. Otherwise, it is 1 plus |
+** the number of overflow pages consumed by a record B bytes in size. |
+*/ |
+static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ |
+ if( pCsr->nRowAvg==0 ){ |
+ /* The average document size, which is required to calculate the cost |
+ ** of each doclist, has not yet been determined. Read the required |
+ ** data from the %_stat table to calculate it. |
+ ** |
+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 |
+ ** varints, where nCol is the number of columns in the FTS3 table. |
+ ** The first varint is the number of documents currently stored in |
+ ** the table. The following nCol varints contain the total amount of |
+ ** data stored in all rows of each column of the table, from left |
+ ** to right. |
+ */ |
+ int rc; |
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; |
+ sqlite3_stmt *pStmt; |
+ sqlite3_int64 nDoc = 0; |
+ sqlite3_int64 nByte = 0; |
+ const char *pEnd; |
+ const char *a; |
+ |
+ rc = sqlite3Fts3SelectDoctotal(p, &pStmt); |
+ if( rc!=SQLITE_OK ) return rc; |
+ a = sqlite3_column_blob(pStmt, 0); |
+ assert( a ); |
+ |
+ pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; |
+ a += sqlite3Fts3GetVarint(a, &nDoc); |
+ while( a<pEnd ){ |
+ a += sqlite3Fts3GetVarint(a, &nByte); |
+ } |
+ if( nDoc==0 || nByte==0 ){ |
+ sqlite3_reset(pStmt); |
+ return FTS_CORRUPT_VTAB; |
+ } |
+ |
+ pCsr->nDoc = nDoc; |
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); |
+ assert( pCsr->nRowAvg>0 ); |
+ rc = sqlite3_reset(pStmt); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ |
+ *pnPage = pCsr->nRowAvg; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function is called to select the tokens (if any) that will be |
+** deferred. The array aTC[] has already been populated when this is |
+** called. |
+** |
+** This function is called once for each AND/NEAR cluster in the |
+** expression. Each invocation determines which tokens to defer within |
+** the cluster with root node pRoot. See comments above the definition |
+** of struct Fts3TokenAndCost for more details. |
+** |
+** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() |
+** called on each token to defer. Otherwise, an SQLite error code is |
+** returned. |
+*/ |
+static int fts3EvalSelectDeferred( |
+ Fts3Cursor *pCsr, /* FTS Cursor handle */ |
+ Fts3Expr *pRoot, /* Consider tokens with this root node */ |
+ Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ |
+ int nTC /* Number of entries in aTC[] */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int nDocSize = 0; /* Number of pages per doc loaded */ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int ii; /* Iterator variable for various purposes */ |
+ int nOvfl = 0; /* Total overflow pages used by doclists */ |
+ int nToken = 0; /* Total number of tokens in cluster */ |
+ |
+ int nMinEst = 0; /* The minimum count for any phrase so far. */ |
+ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ |
+ |
+ /* Tokens are never deferred for FTS tables created using the content=xxx |
+ ** option. The reason being that it is not guaranteed that the content |
+ ** table actually contains the same data as the index. To prevent this from |
+ ** causing any problems, the deferred token optimization is completely |
+ ** disabled for content=xxx tables. */ |
+ if( pTab->zContentTbl ){ |
+ return SQLITE_OK; |
+ } |
+ |
+ /* Count the tokens in this AND/NEAR cluster. If none of the doclists |
+ ** associated with the tokens spill onto overflow pages, or if there is |
+ ** only 1 token, exit early. No tokens to defer in this case. */ |
+ for(ii=0; ii<nTC; ii++){ |
+ if( aTC[ii].pRoot==pRoot ){ |
+ nOvfl += aTC[ii].nOvfl; |
+ nToken++; |
+ } |
+ } |
+ if( nOvfl==0 || nToken<2 ) return SQLITE_OK; |
+ |
+ /* Obtain the average docsize (in pages). */ |
+ rc = fts3EvalAverageDocsize(pCsr, &nDocSize); |
+ assert( rc!=SQLITE_OK || nDocSize>0 ); |
+ |
+ |
+ /* Iterate through all tokens in this AND/NEAR cluster, in ascending order |
+ ** of the number of overflow pages that will be loaded by the pager layer |
+ ** to retrieve the entire doclist for the token from the full-text index. |
+ ** Load the doclists for tokens that are either: |
+ ** |
+ ** a. The cheapest token in the entire query (i.e. the one visited by the |
+ ** first iteration of this loop), or |
+ ** |
+ ** b. Part of a multi-token phrase. |
+ ** |
+ ** After each token doclist is loaded, merge it with the others from the |
+ ** same phrase and count the number of documents that the merged doclist |
+ ** contains. Set variable "nMinEst" to the smallest number of documents in |
+ ** any phrase doclist for which 1 or more token doclists have been loaded. |
+ ** Let nOther be the number of other phrases for which it is certain that |
+ ** one or more tokens will not be deferred. |
+ ** |
+ ** Then, for each token, defer it if loading the doclist would result in |
+ ** loading N or more overflow pages into memory, where N is computed as: |
+ ** |
+ ** (nMinEst + 4^nOther - 1) / (4^nOther) |
+ */ |
+ for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){ |
+ int iTC; /* Used to iterate through aTC[] array. */ |
+ Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */ |
+ |
+ /* Set pTC to point to the cheapest remaining token. */ |
+ for(iTC=0; iTC<nTC; iTC++){ |
+ if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot |
+ && (!pTC || aTC[iTC].nOvfl<pTC->nOvfl) |
+ ){ |
+ pTC = &aTC[iTC]; |
+ } |
+ } |
+ assert( pTC ); |
+ |
+ if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ |
+ /* The number of overflow pages to load for this (and therefore all |
+ ** subsequent) tokens is greater than the estimated number of pages |
+ ** that will be loaded if all subsequent tokens are deferred. |
+ */ |
+ Fts3PhraseToken *pToken = pTC->pToken; |
+ rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); |
+ fts3SegReaderCursorFree(pToken->pSegcsr); |
+ pToken->pSegcsr = 0; |
+ }else{ |
+ /* Set nLoad4 to the value of (4^nOther) for the next iteration of the |
+ ** for-loop. Except, limit the value to 2^24 to prevent it from |
+ ** overflowing the 32-bit integer it is stored in. */ |
+ if( ii<12 ) nLoad4 = nLoad4*4; |
+ |
+ if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ |
+ /* Either this is the cheapest token in the entire query, or it is |
+ ** part of a multi-token phrase. Either way, the entire doclist will |
+ ** (eventually) be loaded into memory. It may as well be now. */ |
+ Fts3PhraseToken *pToken = pTC->pToken; |
+ int nList = 0; |
+ char *pList = 0; |
+ rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); |
+ assert( rc==SQLITE_OK || pList==0 ); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3EvalPhraseMergeToken( |
+ pTab, pTC->pPhrase, pTC->iToken,pList,nList |
+ ); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ int nCount; |
+ nCount = fts3DoclistCountDocids( |
+ pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll |
+ ); |
+ if( ii==0 || nCount<nMinEst ) nMinEst = nCount; |
+ } |
+ } |
+ } |
+ pTC->pToken = 0; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This function is called from within the xFilter method. It initializes |
+** the full-text query currently stored in pCsr->pExpr. To iterate through |
+** the results of a query, the caller does: |
+** |
+** fts3EvalStart(pCsr); |
+** while( 1 ){ |
+** fts3EvalNext(pCsr); |
+** if( pCsr->bEof ) break; |
+** ... return row pCsr->iPrevId to the caller ... |
+** } |
+*/ |
+static int fts3EvalStart(Fts3Cursor *pCsr){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int rc = SQLITE_OK; |
+ int nToken = 0; |
+ int nOr = 0; |
+ |
+ /* Allocate a MultiSegReader for each token in the expression. */ |
+ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); |
+ |
+ /* Determine which, if any, tokens in the expression should be deferred. */ |
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED |
+ if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ |
+ Fts3TokenAndCost *aTC; |
+ Fts3Expr **apOr; |
+ aTC = (Fts3TokenAndCost *)sqlite3_malloc( |
+ sizeof(Fts3TokenAndCost) * nToken |
+ + sizeof(Fts3Expr *) * nOr * 2 |
+ ); |
+ apOr = (Fts3Expr **)&aTC[nToken]; |
+ |
+ if( !aTC ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ int ii; |
+ Fts3TokenAndCost *pTC = aTC; |
+ Fts3Expr **ppOr = apOr; |
+ |
+ fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); |
+ nToken = (int)(pTC-aTC); |
+ nOr = (int)(ppOr-apOr); |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); |
+ for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){ |
+ rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken); |
+ } |
+ } |
+ |
+ sqlite3_free(aTC); |
+ } |
+ } |
+#endif |
+ |
+ fts3EvalStartReaders(pCsr, pCsr->pExpr, &rc); |
+ return rc; |
+} |
+ |
+/* |
+** Invalidate the current position list for phrase pPhrase. |
+*/ |
+static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ |
+ if( pPhrase->doclist.bFreeList ){ |
+ sqlite3_free(pPhrase->doclist.pList); |
+ } |
+ pPhrase->doclist.pList = 0; |
+ pPhrase->doclist.nList = 0; |
+ pPhrase->doclist.bFreeList = 0; |
+} |
+ |
+/* |
+** This function is called to edit the position list associated with |
+** the phrase object passed as the fifth argument according to a NEAR |
+** condition. For example: |
+** |
+** abc NEAR/5 "def ghi" |
+** |
+** Parameter nNear is passed the NEAR distance of the expression (5 in |
+** the example above). When this function is called, *paPoslist points to |
+** the position list, and *pnToken is the number of phrase tokens in, the |
+** phrase on the other side of the NEAR operator to pPhrase. For example, |
+** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to |
+** the position list associated with phrase "abc". |
+** |
+** All positions in the pPhrase position list that are not sufficiently |
+** close to a position in the *paPoslist position list are removed. If this |
+** leaves 0 positions, zero is returned. Otherwise, non-zero. |
+** |
+** Before returning, *paPoslist is set to point to the position lsit |
+** associated with pPhrase. And *pnToken is set to the number of tokens in |
+** pPhrase. |
+*/ |
+static int fts3EvalNearTrim( |
+ int nNear, /* NEAR distance. As in "NEAR/nNear". */ |
+ char *aTmp, /* Temporary space to use */ |
+ char **paPoslist, /* IN/OUT: Position list */ |
+ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ |
+ Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ |
+){ |
+ int nParam1 = nNear + pPhrase->nToken; |
+ int nParam2 = nNear + *pnToken; |
+ int nNew; |
+ char *p2; |
+ char *pOut; |
+ int res; |
+ |
+ assert( pPhrase->doclist.pList ); |
+ |
+ p2 = pOut = pPhrase->doclist.pList; |
+ res = fts3PoslistNearMerge( |
+ &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 |
+ ); |
+ if( res ){ |
+ nNew = (int)(pOut - pPhrase->doclist.pList) - 1; |
+ assert( pPhrase->doclist.pList[nNew]=='\0' ); |
+ assert( nNew<=pPhrase->doclist.nList && nNew>0 ); |
+ memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); |
+ pPhrase->doclist.nList = nNew; |
+ *paPoslist = pPhrase->doclist.pList; |
+ *pnToken = pPhrase->nToken; |
+ } |
+ |
+ return res; |
+} |
+ |
+/* |
+** This function is a no-op if *pRc is other than SQLITE_OK when it is called. |
+** Otherwise, it advances the expression passed as the second argument to |
+** point to the next matching row in the database. Expressions iterate through |
+** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, |
+** or descending if it is non-zero. |
+** |
+** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if |
+** successful, the following variables in pExpr are set: |
+** |
+** Fts3Expr.bEof (non-zero if EOF - there is no next row) |
+** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) |
+** |
+** If the expression is of type FTSQUERY_PHRASE, and the expression is not |
+** at EOF, then the following variables are populated with the position list |
+** for the phrase for the visited row: |
+** |
+** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) |
+** FTs3Expr.pPhrase->doclist.pList (pointer to position list) |
+** |
+** It says above that this function advances the expression to the next |
+** matching row. This is usually true, but there are the following exceptions: |
+** |
+** 1. Deferred tokens are not taken into account. If a phrase consists |
+** entirely of deferred tokens, it is assumed to match every row in |
+** the db. In this case the position-list is not populated at all. |
+** |
+** Or, if a phrase contains one or more deferred tokens and one or |
+** more non-deferred tokens, then the expression is advanced to the |
+** next possible match, considering only non-deferred tokens. In other |
+** words, if the phrase is "A B C", and "B" is deferred, the expression |
+** is advanced to the next row that contains an instance of "A * C", |
+** where "*" may match any single token. The position list in this case |
+** is populated as for "A * C" before returning. |
+** |
+** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is |
+** advanced to point to the next row that matches "x AND y". |
+** |
+** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is |
+** really a match, taking into account deferred tokens and NEAR operators. |
+*/ |
+static void fts3EvalNextRow( |
+ Fts3Cursor *pCsr, /* FTS Cursor handle */ |
+ Fts3Expr *pExpr, /* Expr. to advance to next matching row */ |
+ int *pRc /* IN/OUT: Error code */ |
+){ |
+ if( *pRc==SQLITE_OK ){ |
+ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ |
+ assert( pExpr->bEof==0 ); |
+ pExpr->bStart = 1; |
+ |
+ switch( pExpr->eType ){ |
+ case FTSQUERY_NEAR: |
+ case FTSQUERY_AND: { |
+ Fts3Expr *pLeft = pExpr->pLeft; |
+ Fts3Expr *pRight = pExpr->pRight; |
+ assert( !pLeft->bDeferred || !pRight->bDeferred ); |
+ |
+ if( pLeft->bDeferred ){ |
+ /* LHS is entirely deferred. So we assume it matches every row. |
+ ** Advance the RHS iterator to find the next row visited. */ |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ pExpr->iDocid = pRight->iDocid; |
+ pExpr->bEof = pRight->bEof; |
+ }else if( pRight->bDeferred ){ |
+ /* RHS is entirely deferred. So we assume it matches every row. |
+ ** Advance the LHS iterator to find the next row visited. */ |
+ fts3EvalNextRow(pCsr, pLeft, pRc); |
+ pExpr->iDocid = pLeft->iDocid; |
+ pExpr->bEof = pLeft->bEof; |
+ }else{ |
+ /* Neither the RHS or LHS are deferred. */ |
+ fts3EvalNextRow(pCsr, pLeft, pRc); |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ |
+ sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); |
+ if( iDiff==0 ) break; |
+ if( iDiff<0 ){ |
+ fts3EvalNextRow(pCsr, pLeft, pRc); |
+ }else{ |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ } |
+ } |
+ pExpr->iDocid = pLeft->iDocid; |
+ pExpr->bEof = (pLeft->bEof || pRight->bEof); |
+ if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ |
+ if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ |
+ Fts3Doclist *pDl = &pRight->pPhrase->doclist; |
+ while( *pRc==SQLITE_OK && pRight->bEof==0 ){ |
+ memset(pDl->pList, 0, pDl->nList); |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ } |
+ } |
+ if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ |
+ Fts3Doclist *pDl = &pLeft->pPhrase->doclist; |
+ while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ |
+ memset(pDl->pList, 0, pDl->nList); |
+ fts3EvalNextRow(pCsr, pLeft, pRc); |
+ } |
+ } |
+ } |
+ } |
+ break; |
+ } |
+ |
+ case FTSQUERY_OR: { |
+ Fts3Expr *pLeft = pExpr->pLeft; |
+ Fts3Expr *pRight = pExpr->pRight; |
+ sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); |
+ |
+ assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); |
+ assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); |
+ |
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ |
+ fts3EvalNextRow(pCsr, pLeft, pRc); |
+ }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ }else{ |
+ fts3EvalNextRow(pCsr, pLeft, pRc); |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ } |
+ |
+ pExpr->bEof = (pLeft->bEof && pRight->bEof); |
+ iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); |
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ |
+ pExpr->iDocid = pLeft->iDocid; |
+ }else{ |
+ pExpr->iDocid = pRight->iDocid; |
+ } |
+ |
+ break; |
+ } |
+ |
+ case FTSQUERY_NOT: { |
+ Fts3Expr *pLeft = pExpr->pLeft; |
+ Fts3Expr *pRight = pExpr->pRight; |
+ |
+ if( pRight->bStart==0 ){ |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ assert( *pRc!=SQLITE_OK || pRight->bStart ); |
+ } |
+ |
+ fts3EvalNextRow(pCsr, pLeft, pRc); |
+ if( pLeft->bEof==0 ){ |
+ while( !*pRc |
+ && !pRight->bEof |
+ && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 |
+ ){ |
+ fts3EvalNextRow(pCsr, pRight, pRc); |
+ } |
+ } |
+ pExpr->iDocid = pLeft->iDocid; |
+ pExpr->bEof = pLeft->bEof; |
+ break; |
+ } |
+ |
+ default: { |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ fts3EvalInvalidatePoslist(pPhrase); |
+ *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); |
+ pExpr->iDocid = pPhrase->doclist.iDocid; |
+ break; |
+ } |
+ } |
+ } |
+} |
+ |
+/* |
+** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR |
+** cluster, then this function returns 1 immediately. |
+** |
+** Otherwise, it checks if the current row really does match the NEAR |
+** expression, using the data currently stored in the position lists |
+** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. |
+** |
+** If the current row is a match, the position list associated with each |
+** phrase in the NEAR expression is edited in place to contain only those |
+** phrase instances sufficiently close to their peers to satisfy all NEAR |
+** constraints. In this case it returns 1. If the NEAR expression does not |
+** match the current row, 0 is returned. The position lists may or may not |
+** be edited if 0 is returned. |
+*/ |
+static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ |
+ int res = 1; |
+ |
+ /* The following block runs if pExpr is the root of a NEAR query. |
+ ** For example, the query: |
+ ** |
+ ** "w" NEAR "x" NEAR "y" NEAR "z" |
+ ** |
+ ** which is represented in tree form as: |
+ ** |
+ ** | |
+ ** +--NEAR--+ <-- root of NEAR query |
+ ** | | |
+ ** +--NEAR--+ "z" |
+ ** | | |
+ ** +--NEAR--+ "y" |
+ ** | | |
+ ** "w" "x" |
+ ** |
+ ** The right-hand child of a NEAR node is always a phrase. The |
+ ** left-hand child may be either a phrase or a NEAR node. There are |
+ ** no exceptions to this - it's the way the parser in fts3_expr.c works. |
+ */ |
+ if( *pRc==SQLITE_OK |
+ && pExpr->eType==FTSQUERY_NEAR |
+ && pExpr->bEof==0 |
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) |
+ ){ |
+ Fts3Expr *p; |
+ int nTmp = 0; /* Bytes of temp space */ |
+ char *aTmp; /* Temp space for PoslistNearMerge() */ |
+ |
+ /* Allocate temporary working space. */ |
+ for(p=pExpr; p->pLeft; p=p->pLeft){ |
+ nTmp += p->pRight->pPhrase->doclist.nList; |
+ } |
+ nTmp += p->pPhrase->doclist.nList; |
+ if( nTmp==0 ){ |
+ res = 0; |
+ }else{ |
+ aTmp = sqlite3_malloc(nTmp*2); |
+ if( !aTmp ){ |
+ *pRc = SQLITE_NOMEM; |
+ res = 0; |
+ }else{ |
+ char *aPoslist = p->pPhrase->doclist.pList; |
+ int nToken = p->pPhrase->nToken; |
+ |
+ for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ |
+ Fts3Phrase *pPhrase = p->pRight->pPhrase; |
+ int nNear = p->nNear; |
+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); |
+ } |
+ |
+ aPoslist = pExpr->pRight->pPhrase->doclist.pList; |
+ nToken = pExpr->pRight->pPhrase->nToken; |
+ for(p=pExpr->pLeft; p && res; p=p->pLeft){ |
+ int nNear; |
+ Fts3Phrase *pPhrase; |
+ assert( p->pParent && p->pParent->pLeft==p ); |
+ nNear = p->pParent->nNear; |
+ pPhrase = ( |
+ p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase |
+ ); |
+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); |
+ } |
+ } |
+ |
+ sqlite3_free(aTmp); |
+ } |
+ } |
+ |
+ return res; |
+} |
+ |
+/* |
+** This function is a helper function for sqlite3Fts3EvalTestDeferred(). |
+** Assuming no error occurs or has occurred, It returns non-zero if the |
+** expression passed as the second argument matches the row that pCsr |
+** currently points to, or zero if it does not. |
+** |
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op. |
+** If an error occurs during execution of this function, *pRc is set to |
+** the appropriate SQLite error code. In this case the returned value is |
+** undefined. |
+*/ |
+static int fts3EvalTestExpr( |
+ Fts3Cursor *pCsr, /* FTS cursor handle */ |
+ Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ |
+ int *pRc /* IN/OUT: Error code */ |
+){ |
+ int bHit = 1; /* Return value */ |
+ if( *pRc==SQLITE_OK ){ |
+ switch( pExpr->eType ){ |
+ case FTSQUERY_NEAR: |
+ case FTSQUERY_AND: |
+ bHit = ( |
+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) |
+ && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) |
+ && fts3EvalNearTest(pExpr, pRc) |
+ ); |
+ |
+ /* If the NEAR expression does not match any rows, zero the doclist for |
+ ** all phrases involved in the NEAR. This is because the snippet(), |
+ ** offsets() and matchinfo() functions are not supposed to recognize |
+ ** any instances of phrases that are part of unmatched NEAR queries. |
+ ** For example if this expression: |
+ ** |
+ ** ... MATCH 'a OR (b NEAR c)' |
+ ** |
+ ** is matched against a row containing: |
+ ** |
+ ** 'a b d e' |
+ ** |
+ ** then any snippet() should ony highlight the "a" term, not the "b" |
+ ** (as "b" is part of a non-matching NEAR clause). |
+ */ |
+ if( bHit==0 |
+ && pExpr->eType==FTSQUERY_NEAR |
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) |
+ ){ |
+ Fts3Expr *p; |
+ for(p=pExpr; p->pPhrase==0; p=p->pLeft){ |
+ if( p->pRight->iDocid==pCsr->iPrevId ){ |
+ fts3EvalInvalidatePoslist(p->pRight->pPhrase); |
+ } |
+ } |
+ if( p->iDocid==pCsr->iPrevId ){ |
+ fts3EvalInvalidatePoslist(p->pPhrase); |
+ } |
+ } |
+ |
+ break; |
+ |
+ case FTSQUERY_OR: { |
+ int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); |
+ int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); |
+ bHit = bHit1 || bHit2; |
+ break; |
+ } |
+ |
+ case FTSQUERY_NOT: |
+ bHit = ( |
+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) |
+ && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) |
+ ); |
+ break; |
+ |
+ default: { |
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED |
+ if( pCsr->pDeferred |
+ && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) |
+ ){ |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); |
+ if( pExpr->bDeferred ){ |
+ fts3EvalInvalidatePoslist(pPhrase); |
+ } |
+ *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); |
+ bHit = (pPhrase->doclist.pList!=0); |
+ pExpr->iDocid = pCsr->iPrevId; |
+ }else |
+#endif |
+ { |
+ bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); |
+ } |
+ break; |
+ } |
+ } |
+ } |
+ return bHit; |
+} |
+ |
+/* |
+** This function is called as the second part of each xNext operation when |
+** iterating through the results of a full-text query. At this point the |
+** cursor points to a row that matches the query expression, with the |
+** following caveats: |
+** |
+** * Up until this point, "NEAR" operators in the expression have been |
+** treated as "AND". |
+** |
+** * Deferred tokens have not yet been considered. |
+** |
+** If *pRc is not SQLITE_OK when this function is called, it immediately |
+** returns 0. Otherwise, it tests whether or not after considering NEAR |
+** operators and deferred tokens the current row is still a match for the |
+** expression. It returns 1 if both of the following are true: |
+** |
+** 1. *pRc is SQLITE_OK when this function returns, and |
+** |
+** 2. After scanning the current FTS table row for the deferred tokens, |
+** it is determined that the row does *not* match the query. |
+** |
+** Or, if no error occurs and it seems the current row does match the FTS |
+** query, return 0. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ |
+ int rc = *pRc; |
+ int bMiss = 0; |
+ if( rc==SQLITE_OK ){ |
+ |
+ /* If there are one or more deferred tokens, load the current row into |
+ ** memory and scan it to determine the position list for each deferred |
+ ** token. Then, see if this row is really a match, considering deferred |
+ ** tokens and NEAR operators (neither of which were taken into account |
+ ** earlier, by fts3EvalNextRow()). |
+ */ |
+ if( pCsr->pDeferred ){ |
+ rc = fts3CursorSeek(0, pCsr); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3CacheDeferredDoclists(pCsr); |
+ } |
+ } |
+ bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); |
+ |
+ /* Free the position-lists accumulated for each deferred token above. */ |
+ sqlite3Fts3FreeDeferredDoclists(pCsr); |
+ *pRc = rc; |
+ } |
+ return (rc==SQLITE_OK && bMiss); |
+} |
+ |
+/* |
+** Advance to the next document that matches the FTS expression in |
+** Fts3Cursor.pExpr. |
+*/ |
+static int fts3EvalNext(Fts3Cursor *pCsr){ |
+ int rc = SQLITE_OK; /* Return Code */ |
+ Fts3Expr *pExpr = pCsr->pExpr; |
+ assert( pCsr->isEof==0 ); |
+ if( pExpr==0 ){ |
+ pCsr->isEof = 1; |
+ }else{ |
+ do { |
+ if( pCsr->isRequireSeek==0 ){ |
+ sqlite3_reset(pCsr->pStmt); |
+ } |
+ assert( sqlite3_data_count(pCsr->pStmt)==0 ); |
+ fts3EvalNextRow(pCsr, pExpr, &rc); |
+ pCsr->isEof = pExpr->bEof; |
+ pCsr->isRequireSeek = 1; |
+ pCsr->isMatchinfoNeeded = 1; |
+ pCsr->iPrevId = pExpr->iDocid; |
+ }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); |
+ } |
+ |
+ /* Check if the cursor is past the end of the docid range specified |
+ ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ |
+ if( rc==SQLITE_OK && ( |
+ (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) |
+ || (pCsr->bDesc!=0 && pCsr->iPrevId<pCsr->iMinDocid) |
+ )){ |
+ pCsr->isEof = 1; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Restart interation for expression pExpr so that the next call to |
+** fts3EvalNext() visits the first row. Do not allow incremental |
+** loading or merging of phrase doclists for this iteration. |
+** |
+** If *pRc is other than SQLITE_OK when this function is called, it is |
+** a no-op. If an error occurs within this function, *pRc is set to an |
+** SQLite error code before returning. |
+*/ |
+static void fts3EvalRestart( |
+ Fts3Cursor *pCsr, |
+ Fts3Expr *pExpr, |
+ int *pRc |
+){ |
+ if( pExpr && *pRc==SQLITE_OK ){ |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ |
+ if( pPhrase ){ |
+ fts3EvalInvalidatePoslist(pPhrase); |
+ if( pPhrase->bIncr ){ |
+ int i; |
+ for(i=0; i<pPhrase->nToken; i++){ |
+ Fts3PhraseToken *pToken = &pPhrase->aToken[i]; |
+ assert( pToken->pDeferred==0 ); |
+ if( pToken->pSegcsr ){ |
+ sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); |
+ } |
+ } |
+ *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); |
+ } |
+ pPhrase->doclist.pNextDocid = 0; |
+ pPhrase->doclist.iDocid = 0; |
+ pPhrase->pOrPoslist = 0; |
+ } |
+ |
+ pExpr->iDocid = 0; |
+ pExpr->bEof = 0; |
+ pExpr->bStart = 0; |
+ |
+ fts3EvalRestart(pCsr, pExpr->pLeft, pRc); |
+ fts3EvalRestart(pCsr, pExpr->pRight, pRc); |
+ } |
+} |
+ |
+/* |
+** After allocating the Fts3Expr.aMI[] array for each phrase in the |
+** expression rooted at pExpr, the cursor iterates through all rows matched |
+** by pExpr, calling this function for each row. This function increments |
+** the values in Fts3Expr.aMI[] according to the position-list currently |
+** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase |
+** expression nodes. |
+*/ |
+static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ |
+ if( pExpr ){ |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ if( pPhrase && pPhrase->doclist.pList ){ |
+ int iCol = 0; |
+ char *p = pPhrase->doclist.pList; |
+ |
+ assert( *p ); |
+ while( 1 ){ |
+ u8 c = 0; |
+ int iCnt = 0; |
+ while( 0xFE & (*p | c) ){ |
+ if( (c&0x80)==0 ) iCnt++; |
+ c = *p++ & 0x80; |
+ } |
+ |
+ /* aMI[iCol*3 + 1] = Number of occurrences |
+ ** aMI[iCol*3 + 2] = Number of rows containing at least one instance |
+ */ |
+ pExpr->aMI[iCol*3 + 1] += iCnt; |
+ pExpr->aMI[iCol*3 + 2] += (iCnt>0); |
+ if( *p==0x00 ) break; |
+ p++; |
+ p += fts3GetVarint32(p, &iCol); |
+ } |
+ } |
+ |
+ fts3EvalUpdateCounts(pExpr->pLeft); |
+ fts3EvalUpdateCounts(pExpr->pRight); |
+ } |
+} |
+ |
+/* |
+** Expression pExpr must be of type FTSQUERY_PHRASE. |
+** |
+** If it is not already allocated and populated, this function allocates and |
+** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part |
+** of a NEAR expression, then it also allocates and populates the same array |
+** for all other phrases that are part of the NEAR expression. |
+** |
+** SQLITE_OK is returned if the aMI[] array is successfully allocated and |
+** populated. Otherwise, if an error occurs, an SQLite error code is returned. |
+*/ |
+static int fts3EvalGatherStats( |
+ Fts3Cursor *pCsr, /* Cursor object */ |
+ Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ |
+ assert( pExpr->eType==FTSQUERY_PHRASE ); |
+ if( pExpr->aMI==0 ){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ Fts3Expr *pRoot; /* Root of NEAR expression */ |
+ Fts3Expr *p; /* Iterator used for several purposes */ |
+ |
+ sqlite3_int64 iPrevId = pCsr->iPrevId; |
+ sqlite3_int64 iDocid; |
+ u8 bEof; |
+ |
+ /* Find the root of the NEAR expression */ |
+ pRoot = pExpr; |
+ while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ |
+ pRoot = pRoot->pParent; |
+ } |
+ iDocid = pRoot->iDocid; |
+ bEof = pRoot->bEof; |
+ assert( pRoot->bStart ); |
+ |
+ /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ |
+ for(p=pRoot; p; p=p->pLeft){ |
+ Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); |
+ assert( pE->aMI==0 ); |
+ pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); |
+ if( !pE->aMI ) return SQLITE_NOMEM; |
+ memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); |
+ } |
+ |
+ fts3EvalRestart(pCsr, pRoot, &rc); |
+ |
+ while( pCsr->isEof==0 && rc==SQLITE_OK ){ |
+ |
+ do { |
+ /* Ensure the %_content statement is reset. */ |
+ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); |
+ assert( sqlite3_data_count(pCsr->pStmt)==0 ); |
+ |
+ /* Advance to the next document */ |
+ fts3EvalNextRow(pCsr, pRoot, &rc); |
+ pCsr->isEof = pRoot->bEof; |
+ pCsr->isRequireSeek = 1; |
+ pCsr->isMatchinfoNeeded = 1; |
+ pCsr->iPrevId = pRoot->iDocid; |
+ }while( pCsr->isEof==0 |
+ && pRoot->eType==FTSQUERY_NEAR |
+ && sqlite3Fts3EvalTestDeferred(pCsr, &rc) |
+ ); |
+ |
+ if( rc==SQLITE_OK && pCsr->isEof==0 ){ |
+ fts3EvalUpdateCounts(pRoot); |
+ } |
+ } |
+ |
+ pCsr->isEof = 0; |
+ pCsr->iPrevId = iPrevId; |
+ |
+ if( bEof ){ |
+ pRoot->bEof = bEof; |
+ }else{ |
+ /* Caution: pRoot may iterate through docids in ascending or descending |
+ ** order. For this reason, even though it seems more defensive, the |
+ ** do loop can not be written: |
+ ** |
+ ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK ); |
+ */ |
+ fts3EvalRestart(pCsr, pRoot, &rc); |
+ do { |
+ fts3EvalNextRow(pCsr, pRoot, &rc); |
+ assert( pRoot->bEof==0 ); |
+ }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This function is used by the matchinfo() module to query a phrase |
+** expression node for the following information: |
+** |
+** 1. The total number of occurrences of the phrase in each column of |
+** the FTS table (considering all rows), and |
+** |
+** 2. For each column, the number of rows in the table for which the |
+** column contains at least one instance of the phrase. |
+** |
+** If no error occurs, SQLITE_OK is returned and the values for each column |
+** written into the array aiOut as follows: |
+** |
+** aiOut[iCol*3 + 1] = Number of occurrences |
+** aiOut[iCol*3 + 2] = Number of rows containing at least one instance |
+** |
+** Caveats: |
+** |
+** * If a phrase consists entirely of deferred tokens, then all output |
+** values are set to the number of documents in the table. In other |
+** words we assume that very common tokens occur exactly once in each |
+** column of each row of the table. |
+** |
+** * If a phrase contains some deferred tokens (and some non-deferred |
+** tokens), count the potential occurrence identified by considering |
+** the non-deferred tokens instead of actual phrase occurrences. |
+** |
+** * If the phrase is part of a NEAR expression, then only phrase instances |
+** that meet the NEAR constraint are included in the counts. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( |
+ Fts3Cursor *pCsr, /* FTS cursor handle */ |
+ Fts3Expr *pExpr, /* Phrase expression */ |
+ u32 *aiOut /* Array to write results into (see above) */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int rc = SQLITE_OK; |
+ int iCol; |
+ |
+ if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ |
+ assert( pCsr->nDoc>0 ); |
+ for(iCol=0; iCol<pTab->nColumn; iCol++){ |
+ aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; |
+ aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; |
+ } |
+ }else{ |
+ rc = fts3EvalGatherStats(pCsr, pExpr); |
+ if( rc==SQLITE_OK ){ |
+ assert( pExpr->aMI ); |
+ for(iCol=0; iCol<pTab->nColumn; iCol++){ |
+ aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; |
+ aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** The expression pExpr passed as the second argument to this function |
+** must be of type FTSQUERY_PHRASE. |
+** |
+** The returned value is either NULL or a pointer to a buffer containing |
+** a position-list indicating the occurrences of the phrase in column iCol |
+** of the current row. |
+** |
+** More specifically, the returned buffer contains 1 varint for each |
+** occurrence of the phrase in the column, stored using the normal (delta+2) |
+** compression and is terminated by either an 0x01 or 0x00 byte. For example, |
+** if the requested column contains "a b X c d X X" and the position-list |
+** for 'X' is requested, the buffer returned may contain: |
+** |
+** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 |
+** |
+** This function works regardless of whether or not the phrase is deferred, |
+** incremental, or neither. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( |
+ Fts3Cursor *pCsr, /* FTS3 cursor object */ |
+ Fts3Expr *pExpr, /* Phrase to return doclist for */ |
+ int iCol, /* Column to return position list for */ |
+ char **ppOut /* OUT: Pointer to position list */ |
+){ |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ char *pIter; |
+ int iThis; |
+ sqlite3_int64 iDocid; |
+ |
+ /* If this phrase is applies specifically to some column other than |
+ ** column iCol, return a NULL pointer. */ |
+ *ppOut = 0; |
+ assert( iCol>=0 && iCol<pTab->nColumn ); |
+ if( (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) ){ |
+ return SQLITE_OK; |
+ } |
+ |
+ iDocid = pExpr->iDocid; |
+ pIter = pPhrase->doclist.pList; |
+ if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ |
+ int rc = SQLITE_OK; |
+ int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ |
+ int bOr = 0; |
+ u8 bTreeEof = 0; |
+ Fts3Expr *p; /* Used to iterate from pExpr to root */ |
+ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ |
+ int bMatch; |
+ |
+ /* Check if this phrase descends from an OR expression node. If not, |
+ ** return NULL. Otherwise, the entry that corresponds to docid |
+ ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the |
+ ** tree that the node is part of has been marked as EOF, but the node |
+ ** itself is not EOF, then it may point to an earlier entry. */ |
+ pNear = pExpr; |
+ for(p=pExpr->pParent; p; p=p->pParent){ |
+ if( p->eType==FTSQUERY_OR ) bOr = 1; |
+ if( p->eType==FTSQUERY_NEAR ) pNear = p; |
+ if( p->bEof ) bTreeEof = 1; |
+ } |
+ if( bOr==0 ) return SQLITE_OK; |
+ |
+ /* This is the descendent of an OR node. In this case we cannot use |
+ ** an incremental phrase. Load the entire doclist for the phrase |
+ ** into memory in this case. */ |
+ if( pPhrase->bIncr ){ |
+ int bEofSave = pNear->bEof; |
+ fts3EvalRestart(pCsr, pNear, &rc); |
+ while( rc==SQLITE_OK && !pNear->bEof ){ |
+ fts3EvalNextRow(pCsr, pNear, &rc); |
+ if( bEofSave==0 && pNear->iDocid==iDocid ) break; |
+ } |
+ assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); |
+ } |
+ if( bTreeEof ){ |
+ while( rc==SQLITE_OK && !pNear->bEof ){ |
+ fts3EvalNextRow(pCsr, pNear, &rc); |
+ } |
+ } |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ bMatch = 1; |
+ for(p=pNear; p; p=p->pLeft){ |
+ u8 bEof = 0; |
+ Fts3Expr *pTest = p; |
+ Fts3Phrase *pPh; |
+ assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); |
+ if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; |
+ assert( pTest->eType==FTSQUERY_PHRASE ); |
+ pPh = pTest->pPhrase; |
+ |
+ pIter = pPh->pOrPoslist; |
+ iDocid = pPh->iOrDocid; |
+ if( pCsr->bDesc==bDescDoclist ){ |
+ bEof = !pPh->doclist.nAll || |
+ (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); |
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ |
+ sqlite3Fts3DoclistNext( |
+ bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, |
+ &pIter, &iDocid, &bEof |
+ ); |
+ } |
+ }else{ |
+ bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); |
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ |
+ int dummy; |
+ sqlite3Fts3DoclistPrev( |
+ bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, |
+ &pIter, &iDocid, &dummy, &bEof |
+ ); |
+ } |
+ } |
+ pPh->pOrPoslist = pIter; |
+ pPh->iOrDocid = iDocid; |
+ if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; |
+ } |
+ |
+ if( bMatch ){ |
+ pIter = pPhrase->pOrPoslist; |
+ }else{ |
+ pIter = 0; |
+ } |
+ } |
+ if( pIter==0 ) return SQLITE_OK; |
+ |
+ if( *pIter==0x01 ){ |
+ pIter++; |
+ pIter += fts3GetVarint32(pIter, &iThis); |
+ }else{ |
+ iThis = 0; |
+ } |
+ while( iThis<iCol ){ |
+ fts3ColumnlistCopy(0, &pIter); |
+ if( *pIter==0x00 ) return SQLITE_OK; |
+ pIter++; |
+ pIter += fts3GetVarint32(pIter, &iThis); |
+ } |
+ if( *pIter==0x00 ){ |
+ pIter = 0; |
+ } |
+ |
+ *ppOut = ((iCol==iThis)?pIter:0); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Free all components of the Fts3Phrase structure that were allocated by |
+** the eval module. Specifically, this means to free: |
+** |
+** * the contents of pPhrase->doclist, and |
+** * any Fts3MultiSegReader objects held by phrase tokens. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ |
+ if( pPhrase ){ |
+ int i; |
+ sqlite3_free(pPhrase->doclist.aAll); |
+ fts3EvalInvalidatePoslist(pPhrase); |
+ memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); |
+ for(i=0; i<pPhrase->nToken; i++){ |
+ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); |
+ pPhrase->aToken[i].pSegcsr = 0; |
+ } |
+ } |
+} |
+ |
+ |
+/* |
+** Return SQLITE_CORRUPT_VTAB. |
+*/ |
+#ifdef SQLITE_DEBUG |
+SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ |
+ return SQLITE_CORRUPT_VTAB; |
+} |
+#endif |
+ |
+#if !SQLITE_CORE |
+/* |
+** Initialize API pointer table, if required. |
+*/ |
+#ifdef _WIN32 |
+__declspec(dllexport) |
+#endif |
+SQLITE_API int sqlite3_fts3_init( |
+ sqlite3 *db, |
+ char **pzErrMsg, |
+ const sqlite3_api_routines *pApi |
+){ |
+ SQLITE_EXTENSION_INIT2(pApi) |
+ return sqlite3Fts3Init(db); |
+} |
+#endif |
+ |
+#endif |
+ |
+/************** End of fts3.c ************************************************/ |
+/************** Begin file fts3_aux.c ****************************************/ |
+/* |
+** 2011 Jan 27 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <string.h> */ |
+/* #include <assert.h> */ |
+ |
+typedef struct Fts3auxTable Fts3auxTable; |
+typedef struct Fts3auxCursor Fts3auxCursor; |
+ |
+struct Fts3auxTable { |
+ sqlite3_vtab base; /* Base class used by SQLite core */ |
+ Fts3Table *pFts3Tab; |
+}; |
+ |
+struct Fts3auxCursor { |
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */ |
+ Fts3MultiSegReader csr; /* Must be right after "base" */ |
+ Fts3SegFilter filter; |
+ char *zStop; |
+ int nStop; /* Byte-length of string zStop */ |
+ int iLangid; /* Language id to query */ |
+ int isEof; /* True if cursor is at EOF */ |
+ sqlite3_int64 iRowid; /* Current rowid */ |
+ |
+ int iCol; /* Current value of 'col' column */ |
+ int nStat; /* Size of aStat[] array */ |
+ struct Fts3auxColstats { |
+ sqlite3_int64 nDoc; /* 'documents' values for current csr row */ |
+ sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ |
+ } *aStat; |
+}; |
+ |
+/* |
+** Schema of the terms table. |
+*/ |
+#define FTS3_AUX_SCHEMA \ |
+ "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" |
+ |
+/* |
+** This function does all the work for both the xConnect and xCreate methods. |
+** These tables have no persistent representation of their own, so xConnect |
+** and xCreate are identical operations. |
+*/ |
+static int fts3auxConnectMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pUnused, /* Unused */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ char const *zDb; /* Name of database (e.g. "main") */ |
+ char const *zFts3; /* Name of fts3 table */ |
+ int nDb; /* Result of strlen(zDb) */ |
+ int nFts3; /* Result of strlen(zFts3) */ |
+ int nByte; /* Bytes of space to allocate here */ |
+ int rc; /* value returned by declare_vtab() */ |
+ Fts3auxTable *p; /* Virtual table object to return */ |
+ |
+ UNUSED_PARAMETER(pUnused); |
+ |
+ /* The user should invoke this in one of two forms: |
+ ** |
+ ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); |
+ ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); |
+ */ |
+ if( argc!=4 && argc!=5 ) goto bad_args; |
+ |
+ zDb = argv[1]; |
+ nDb = (int)strlen(zDb); |
+ if( argc==5 ){ |
+ if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ |
+ zDb = argv[3]; |
+ nDb = (int)strlen(zDb); |
+ zFts3 = argv[4]; |
+ }else{ |
+ goto bad_args; |
+ } |
+ }else{ |
+ zFts3 = argv[3]; |
+ } |
+ nFts3 = (int)strlen(zFts3); |
+ |
+ rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; |
+ p = (Fts3auxTable *)sqlite3_malloc(nByte); |
+ if( !p ) return SQLITE_NOMEM; |
+ memset(p, 0, nByte); |
+ |
+ p->pFts3Tab = (Fts3Table *)&p[1]; |
+ p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; |
+ p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; |
+ p->pFts3Tab->db = db; |
+ p->pFts3Tab->nIndex = 1; |
+ |
+ memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); |
+ memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); |
+ sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); |
+ |
+ *ppVtab = (sqlite3_vtab *)p; |
+ return SQLITE_OK; |
+ |
+ bad_args: |
+ sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); |
+ return SQLITE_ERROR; |
+} |
+ |
+/* |
+** This function does the work for both the xDisconnect and xDestroy methods. |
+** These tables have no persistent representation of their own, so xDisconnect |
+** and xDestroy are identical operations. |
+*/ |
+static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ |
+ Fts3auxTable *p = (Fts3auxTable *)pVtab; |
+ Fts3Table *pFts3 = p->pFts3Tab; |
+ int i; |
+ |
+ /* Free any prepared statements held */ |
+ for(i=0; i<SizeofArray(pFts3->aStmt); i++){ |
+ sqlite3_finalize(pFts3->aStmt[i]); |
+ } |
+ sqlite3_free(pFts3->zSegmentsTbl); |
+ sqlite3_free(p); |
+ return SQLITE_OK; |
+} |
+ |
+#define FTS4AUX_EQ_CONSTRAINT 1 |
+#define FTS4AUX_GE_CONSTRAINT 2 |
+#define FTS4AUX_LE_CONSTRAINT 4 |
+ |
+/* |
+** xBestIndex - Analyze a WHERE and ORDER BY clause. |
+*/ |
+static int fts3auxBestIndexMethod( |
+ sqlite3_vtab *pVTab, |
+ sqlite3_index_info *pInfo |
+){ |
+ int i; |
+ int iEq = -1; |
+ int iGe = -1; |
+ int iLe = -1; |
+ int iLangid = -1; |
+ int iNext = 1; /* Next free argvIndex value */ |
+ |
+ UNUSED_PARAMETER(pVTab); |
+ |
+ /* This vtab delivers always results in "ORDER BY term ASC" order. */ |
+ if( pInfo->nOrderBy==1 |
+ && pInfo->aOrderBy[0].iColumn==0 |
+ && pInfo->aOrderBy[0].desc==0 |
+ ){ |
+ pInfo->orderByConsumed = 1; |
+ } |
+ |
+ /* Search for equality and range constraints on the "term" column. |
+ ** And equality constraints on the hidden "languageid" column. */ |
+ for(i=0; i<pInfo->nConstraint; i++){ |
+ if( pInfo->aConstraint[i].usable ){ |
+ int op = pInfo->aConstraint[i].op; |
+ int iCol = pInfo->aConstraint[i].iColumn; |
+ |
+ if( iCol==0 ){ |
+ if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; |
+ if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; |
+ if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; |
+ if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; |
+ if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; |
+ } |
+ if( iCol==4 ){ |
+ if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; |
+ } |
+ } |
+ } |
+ |
+ if( iEq>=0 ){ |
+ pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; |
+ pInfo->aConstraintUsage[iEq].argvIndex = iNext++; |
+ pInfo->estimatedCost = 5; |
+ }else{ |
+ pInfo->idxNum = 0; |
+ pInfo->estimatedCost = 20000; |
+ if( iGe>=0 ){ |
+ pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; |
+ pInfo->aConstraintUsage[iGe].argvIndex = iNext++; |
+ pInfo->estimatedCost /= 2; |
+ } |
+ if( iLe>=0 ){ |
+ pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; |
+ pInfo->aConstraintUsage[iLe].argvIndex = iNext++; |
+ pInfo->estimatedCost /= 2; |
+ } |
+ } |
+ if( iLangid>=0 ){ |
+ pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; |
+ pInfo->estimatedCost--; |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xOpen - Open a cursor. |
+*/ |
+static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ |
+ Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ |
+ |
+ UNUSED_PARAMETER(pVTab); |
+ |
+ pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); |
+ if( !pCsr ) return SQLITE_NOMEM; |
+ memset(pCsr, 0, sizeof(Fts3auxCursor)); |
+ |
+ *ppCsr = (sqlite3_vtab_cursor *)pCsr; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xClose - Close a cursor. |
+*/ |
+static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; |
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; |
+ |
+ sqlite3Fts3SegmentsClose(pFts3); |
+ sqlite3Fts3SegReaderFinish(&pCsr->csr); |
+ sqlite3_free((void *)pCsr->filter.zTerm); |
+ sqlite3_free(pCsr->zStop); |
+ sqlite3_free(pCsr->aStat); |
+ sqlite3_free(pCsr); |
+ return SQLITE_OK; |
+} |
+ |
+static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ |
+ if( nSize>pCsr->nStat ){ |
+ struct Fts3auxColstats *aNew; |
+ aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, |
+ sizeof(struct Fts3auxColstats) * nSize |
+ ); |
+ if( aNew==0 ) return SQLITE_NOMEM; |
+ memset(&aNew[pCsr->nStat], 0, |
+ sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) |
+ ); |
+ pCsr->aStat = aNew; |
+ pCsr->nStat = nSize; |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xNext - Advance the cursor to the next row, if any. |
+*/ |
+static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; |
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; |
+ int rc; |
+ |
+ /* Increment our pretend rowid value. */ |
+ pCsr->iRowid++; |
+ |
+ for(pCsr->iCol++; pCsr->iCol<pCsr->nStat; pCsr->iCol++){ |
+ if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; |
+ } |
+ |
+ rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); |
+ if( rc==SQLITE_ROW ){ |
+ int i = 0; |
+ int nDoclist = pCsr->csr.nDoclist; |
+ char *aDoclist = pCsr->csr.aDoclist; |
+ int iCol; |
+ |
+ int eState = 0; |
+ |
+ if( pCsr->zStop ){ |
+ int n = (pCsr->nStop<pCsr->csr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; |
+ int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); |
+ if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ |
+ pCsr->isEof = 1; |
+ return SQLITE_OK; |
+ } |
+ } |
+ |
+ if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; |
+ memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); |
+ iCol = 0; |
+ |
+ while( i<nDoclist ){ |
+ sqlite3_int64 v = 0; |
+ |
+ i += sqlite3Fts3GetVarint(&aDoclist[i], &v); |
+ switch( eState ){ |
+ /* State 0. In this state the integer just read was a docid. */ |
+ case 0: |
+ pCsr->aStat[0].nDoc++; |
+ eState = 1; |
+ iCol = 0; |
+ break; |
+ |
+ /* State 1. In this state we are expecting either a 1, indicating |
+ ** that the following integer will be a column number, or the |
+ ** start of a position list for column 0. |
+ ** |
+ ** The only difference between state 1 and state 2 is that if the |
+ ** integer encountered in state 1 is not 0 or 1, then we need to |
+ ** increment the column 0 "nDoc" count for this term. |
+ */ |
+ case 1: |
+ assert( iCol==0 ); |
+ if( v>1 ){ |
+ pCsr->aStat[1].nDoc++; |
+ } |
+ eState = 2; |
+ /* fall through */ |
+ |
+ case 2: |
+ if( v==0 ){ /* 0x00. Next integer will be a docid. */ |
+ eState = 0; |
+ }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ |
+ eState = 3; |
+ }else{ /* 2 or greater. A position. */ |
+ pCsr->aStat[iCol+1].nOcc++; |
+ pCsr->aStat[0].nOcc++; |
+ } |
+ break; |
+ |
+ /* State 3. The integer just read is a column number. */ |
+ default: assert( eState==3 ); |
+ iCol = (int)v; |
+ if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; |
+ pCsr->aStat[iCol+1].nDoc++; |
+ eState = 2; |
+ break; |
+ } |
+ } |
+ |
+ pCsr->iCol = 0; |
+ rc = SQLITE_OK; |
+ }else{ |
+ pCsr->isEof = 1; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** xFilter - Initialize a cursor to point at the start of its data. |
+*/ |
+static int fts3auxFilterMethod( |
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ |
+ int idxNum, /* Strategy index */ |
+ const char *idxStr, /* Unused */ |
+ int nVal, /* Number of elements in apVal */ |
+ sqlite3_value **apVal /* Arguments for the indexing scheme */ |
+){ |
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; |
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; |
+ int rc; |
+ int isScan = 0; |
+ int iLangVal = 0; /* Language id to query */ |
+ |
+ int iEq = -1; /* Index of term=? value in apVal */ |
+ int iGe = -1; /* Index of term>=? value in apVal */ |
+ int iLe = -1; /* Index of term<=? value in apVal */ |
+ int iLangid = -1; /* Index of languageid=? value in apVal */ |
+ int iNext = 0; |
+ |
+ UNUSED_PARAMETER(nVal); |
+ UNUSED_PARAMETER(idxStr); |
+ |
+ assert( idxStr==0 ); |
+ assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 |
+ || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT |
+ || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) |
+ ); |
+ |
+ if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ |
+ iEq = iNext++; |
+ }else{ |
+ isScan = 1; |
+ if( idxNum & FTS4AUX_GE_CONSTRAINT ){ |
+ iGe = iNext++; |
+ } |
+ if( idxNum & FTS4AUX_LE_CONSTRAINT ){ |
+ iLe = iNext++; |
+ } |
+ } |
+ if( iNext<nVal ){ |
+ iLangid = iNext++; |
+ } |
+ |
+ /* In case this cursor is being reused, close and zero it. */ |
+ testcase(pCsr->filter.zTerm); |
+ sqlite3Fts3SegReaderFinish(&pCsr->csr); |
+ sqlite3_free((void *)pCsr->filter.zTerm); |
+ sqlite3_free(pCsr->aStat); |
+ memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); |
+ |
+ pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; |
+ if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; |
+ |
+ if( iEq>=0 || iGe>=0 ){ |
+ const unsigned char *zStr = sqlite3_value_text(apVal[0]); |
+ assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); |
+ if( zStr ){ |
+ pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); |
+ pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); |
+ if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; |
+ } |
+ } |
+ |
+ if( iLe>=0 ){ |
+ pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); |
+ pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); |
+ if( pCsr->zStop==0 ) return SQLITE_NOMEM; |
+ } |
+ |
+ if( iLangid>=0 ){ |
+ iLangVal = sqlite3_value_int(apVal[iLangid]); |
+ |
+ /* If the user specified a negative value for the languageid, use zero |
+ ** instead. This works, as the "languageid=?" constraint will also |
+ ** be tested by the VDBE layer. The test will always be false (since |
+ ** this module will not return a row with a negative languageid), and |
+ ** so the overall query will return zero rows. */ |
+ if( iLangVal<0 ) iLangVal = 0; |
+ } |
+ pCsr->iLangid = iLangVal; |
+ |
+ rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, |
+ pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr |
+ ); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); |
+ } |
+ |
+ if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); |
+ return rc; |
+} |
+ |
+/* |
+** xEof - Return true if the cursor is at EOF, or false otherwise. |
+*/ |
+static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; |
+ return pCsr->isEof; |
+} |
+ |
+/* |
+** xColumn - Return a column value. |
+*/ |
+static int fts3auxColumnMethod( |
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ |
+ int iCol /* Index of column to read value from */ |
+){ |
+ Fts3auxCursor *p = (Fts3auxCursor *)pCursor; |
+ |
+ assert( p->isEof==0 ); |
+ switch( iCol ){ |
+ case 0: /* term */ |
+ sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); |
+ break; |
+ |
+ case 1: /* col */ |
+ if( p->iCol ){ |
+ sqlite3_result_int(pCtx, p->iCol-1); |
+ }else{ |
+ sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); |
+ } |
+ break; |
+ |
+ case 2: /* documents */ |
+ sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); |
+ break; |
+ |
+ case 3: /* occurrences */ |
+ sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); |
+ break; |
+ |
+ default: /* languageid */ |
+ assert( iCol==4 ); |
+ sqlite3_result_int(pCtx, p->iLangid); |
+ break; |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xRowid - Return the current rowid for the cursor. |
+*/ |
+static int fts3auxRowidMethod( |
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
+ sqlite_int64 *pRowid /* OUT: Rowid value */ |
+){ |
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; |
+ *pRowid = pCsr->iRowid; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Register the fts3aux module with database connection db. Return SQLITE_OK |
+** if successful or an error code if sqlite3_create_module() fails. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ |
+ static const sqlite3_module fts3aux_module = { |
+ 0, /* iVersion */ |
+ fts3auxConnectMethod, /* xCreate */ |
+ fts3auxConnectMethod, /* xConnect */ |
+ fts3auxBestIndexMethod, /* xBestIndex */ |
+ fts3auxDisconnectMethod, /* xDisconnect */ |
+ fts3auxDisconnectMethod, /* xDestroy */ |
+ fts3auxOpenMethod, /* xOpen */ |
+ fts3auxCloseMethod, /* xClose */ |
+ fts3auxFilterMethod, /* xFilter */ |
+ fts3auxNextMethod, /* xNext */ |
+ fts3auxEofMethod, /* xEof */ |
+ fts3auxColumnMethod, /* xColumn */ |
+ fts3auxRowidMethod, /* xRowid */ |
+ 0, /* xUpdate */ |
+ 0, /* xBegin */ |
+ 0, /* xSync */ |
+ 0, /* xCommit */ |
+ 0, /* xRollback */ |
+ 0, /* xFindFunction */ |
+ 0, /* xRename */ |
+ 0, /* xSavepoint */ |
+ 0, /* xRelease */ |
+ 0 /* xRollbackTo */ |
+ }; |
+ int rc; /* Return code */ |
+ |
+ rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); |
+ return rc; |
+} |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_aux.c ********************************************/ |
+/************** Begin file fts3_expr.c ***************************************/ |
+/* |
+** 2008 Nov 28 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This module contains code that implements a parser for fts3 query strings |
+** (the right-hand argument to the MATCH operator). Because the supported |
+** syntax is relatively simple, the whole tokenizer/parser system is |
+** hand-coded. |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* |
+** By default, this module parses the legacy syntax that has been |
+** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS |
+** is defined, then it uses the new syntax. The differences between |
+** the new and the old syntaxes are: |
+** |
+** a) The new syntax supports parenthesis. The old does not. |
+** |
+** b) The new syntax supports the AND and NOT operators. The old does not. |
+** |
+** c) The old syntax supports the "-" token qualifier. This is not |
+** supported by the new syntax (it is replaced by the NOT operator). |
+** |
+** d) When using the old syntax, the OR operator has a greater precedence |
+** than an implicit AND. When using the new, both implicity and explicit |
+** AND operators have a higher precedence than OR. |
+** |
+** If compiled with SQLITE_TEST defined, then this module exports the |
+** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable |
+** to zero causes the module to use the old syntax. If it is set to |
+** non-zero the new syntax is activated. This is so both syntaxes can |
+** be tested using a single build of testfixture. |
+** |
+** The following describes the syntax supported by the fts3 MATCH |
+** operator in a similar format to that used by the lemon parser |
+** generator. This module does not use actually lemon, it uses a |
+** custom parser. |
+** |
+** query ::= andexpr (OR andexpr)*. |
+** |
+** andexpr ::= notexpr (AND? notexpr)*. |
+** |
+** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. |
+** notexpr ::= LP query RP. |
+** |
+** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. |
+** |
+** distance_opt ::= . |
+** distance_opt ::= / INTEGER. |
+** |
+** phrase ::= TOKEN. |
+** phrase ::= COLUMN:TOKEN. |
+** phrase ::= "TOKEN TOKEN TOKEN...". |
+*/ |
+ |
+#ifdef SQLITE_TEST |
+SQLITE_API int sqlite3_fts3_enable_parentheses = 0; |
+#else |
+# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS |
+# define sqlite3_fts3_enable_parentheses 1 |
+# else |
+# define sqlite3_fts3_enable_parentheses 0 |
+# endif |
+#endif |
+ |
+/* |
+** Default span for NEAR operators. |
+*/ |
+#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 |
+ |
+/* #include <string.h> */ |
+/* #include <assert.h> */ |
+ |
+/* |
+** isNot: |
+** This variable is used by function getNextNode(). When getNextNode() is |
+** called, it sets ParseContext.isNot to true if the 'next node' is a |
+** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the |
+** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to |
+** zero. |
+*/ |
+typedef struct ParseContext ParseContext; |
+struct ParseContext { |
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ |
+ int iLangid; /* Language id used with tokenizer */ |
+ const char **azCol; /* Array of column names for fts3 table */ |
+ int bFts4; /* True to allow FTS4-only syntax */ |
+ int nCol; /* Number of entries in azCol[] */ |
+ int iDefaultCol; /* Default column to query */ |
+ int isNot; /* True if getNextNode() sees a unary - */ |
+ sqlite3_context *pCtx; /* Write error message here */ |
+ int nNest; /* Number of nested brackets */ |
+}; |
+ |
+/* |
+** This function is equivalent to the standard isspace() function. |
+** |
+** The standard isspace() can be awkward to use safely, because although it |
+** is defined to accept an argument of type int, its behavior when passed |
+** an integer that falls outside of the range of the unsigned char type |
+** is undefined (and sometimes, "undefined" means segfault). This wrapper |
+** is defined to accept an argument of type char, and always returns 0 for |
+** any values that fall outside of the range of the unsigned char type (i.e. |
+** negative values). |
+*/ |
+static int fts3isspace(char c){ |
+ return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; |
+} |
+ |
+/* |
+** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, |
+** zero the memory before returning a pointer to it. If unsuccessful, |
+** return NULL. |
+*/ |
+static void *fts3MallocZero(int nByte){ |
+ void *pRet = sqlite3_malloc(nByte); |
+ if( pRet ) memset(pRet, 0, nByte); |
+ return pRet; |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( |
+ sqlite3_tokenizer *pTokenizer, |
+ int iLangid, |
+ const char *z, |
+ int n, |
+ sqlite3_tokenizer_cursor **ppCsr |
+){ |
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; |
+ sqlite3_tokenizer_cursor *pCsr = 0; |
+ int rc; |
+ |
+ rc = pModule->xOpen(pTokenizer, z, n, &pCsr); |
+ assert( rc==SQLITE_OK || pCsr==0 ); |
+ if( rc==SQLITE_OK ){ |
+ pCsr->pTokenizer = pTokenizer; |
+ if( pModule->iVersion>=1 ){ |
+ rc = pModule->xLanguageid(pCsr, iLangid); |
+ if( rc!=SQLITE_OK ){ |
+ pModule->xClose(pCsr); |
+ pCsr = 0; |
+ } |
+ } |
+ } |
+ *ppCsr = pCsr; |
+ return rc; |
+} |
+ |
+/* |
+** Function getNextNode(), which is called by fts3ExprParse(), may itself |
+** call fts3ExprParse(). So this forward declaration is required. |
+*/ |
+static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); |
+ |
+/* |
+** Extract the next token from buffer z (length n) using the tokenizer |
+** and other information (column names etc.) in pParse. Create an Fts3Expr |
+** structure of type FTSQUERY_PHRASE containing a phrase consisting of this |
+** single token and set *ppExpr to point to it. If the end of the buffer is |
+** reached before a token is found, set *ppExpr to zero. It is the |
+** responsibility of the caller to eventually deallocate the allocated |
+** Fts3Expr structure (if any) by passing it to sqlite3_free(). |
+** |
+** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation |
+** fails. |
+*/ |
+static int getNextToken( |
+ ParseContext *pParse, /* fts3 query parse context */ |
+ int iCol, /* Value for Fts3Phrase.iColumn */ |
+ const char *z, int n, /* Input string */ |
+ Fts3Expr **ppExpr, /* OUT: expression */ |
+ int *pnConsumed /* OUT: Number of bytes consumed */ |
+){ |
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; |
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; |
+ int rc; |
+ sqlite3_tokenizer_cursor *pCursor; |
+ Fts3Expr *pRet = 0; |
+ int i = 0; |
+ |
+ /* Set variable i to the maximum number of bytes of input to tokenize. */ |
+ for(i=0; i<n; i++){ |
+ if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break; |
+ if( z[i]=='"' ) break; |
+ } |
+ |
+ *pnConsumed = i; |
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor); |
+ if( rc==SQLITE_OK ){ |
+ const char *zToken; |
+ int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; |
+ int nByte; /* total space to allocate */ |
+ |
+ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); |
+ if( rc==SQLITE_OK ){ |
+ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; |
+ pRet = (Fts3Expr *)fts3MallocZero(nByte); |
+ if( !pRet ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ pRet->eType = FTSQUERY_PHRASE; |
+ pRet->pPhrase = (Fts3Phrase *)&pRet[1]; |
+ pRet->pPhrase->nToken = 1; |
+ pRet->pPhrase->iColumn = iCol; |
+ pRet->pPhrase->aToken[0].n = nToken; |
+ pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; |
+ memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); |
+ |
+ if( iEnd<n && z[iEnd]=='*' ){ |
+ pRet->pPhrase->aToken[0].isPrefix = 1; |
+ iEnd++; |
+ } |
+ |
+ while( 1 ){ |
+ if( !sqlite3_fts3_enable_parentheses |
+ && iStart>0 && z[iStart-1]=='-' |
+ ){ |
+ pParse->isNot = 1; |
+ iStart--; |
+ }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ |
+ pRet->pPhrase->aToken[0].bFirst = 1; |
+ iStart--; |
+ }else{ |
+ break; |
+ } |
+ } |
+ |
+ } |
+ *pnConsumed = iEnd; |
+ }else if( i && rc==SQLITE_DONE ){ |
+ rc = SQLITE_OK; |
+ } |
+ |
+ pModule->xClose(pCursor); |
+ } |
+ |
+ *ppExpr = pRet; |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Enlarge a memory allocation. If an out-of-memory allocation occurs, |
+** then free the old allocation. |
+*/ |
+static void *fts3ReallocOrFree(void *pOrig, int nNew){ |
+ void *pRet = sqlite3_realloc(pOrig, nNew); |
+ if( !pRet ){ |
+ sqlite3_free(pOrig); |
+ } |
+ return pRet; |
+} |
+ |
+/* |
+** Buffer zInput, length nInput, contains the contents of a quoted string |
+** that appeared as part of an fts3 query expression. Neither quote character |
+** is included in the buffer. This function attempts to tokenize the entire |
+** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE |
+** containing the results. |
+** |
+** If successful, SQLITE_OK is returned and *ppExpr set to point at the |
+** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory |
+** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set |
+** to 0. |
+*/ |
+static int getNextString( |
+ ParseContext *pParse, /* fts3 query parse context */ |
+ const char *zInput, int nInput, /* Input string */ |
+ Fts3Expr **ppExpr /* OUT: expression */ |
+){ |
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; |
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; |
+ int rc; |
+ Fts3Expr *p = 0; |
+ sqlite3_tokenizer_cursor *pCursor = 0; |
+ char *zTemp = 0; |
+ int nTemp = 0; |
+ |
+ const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); |
+ int nToken = 0; |
+ |
+ /* The final Fts3Expr data structure, including the Fts3Phrase, |
+ ** Fts3PhraseToken structures token buffers are all stored as a single |
+ ** allocation so that the expression can be freed with a single call to |
+ ** sqlite3_free(). Setting this up requires a two pass approach. |
+ ** |
+ ** The first pass, in the block below, uses a tokenizer cursor to iterate |
+ ** through the tokens in the expression. This pass uses fts3ReallocOrFree() |
+ ** to assemble data in two dynamic buffers: |
+ ** |
+ ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase |
+ ** structure, followed by the array of Fts3PhraseToken |
+ ** structures. This pass only populates the Fts3PhraseToken array. |
+ ** |
+ ** Buffer zTemp: Contains copies of all tokens. |
+ ** |
+ ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, |
+ ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase |
+ ** structures. |
+ */ |
+ rc = sqlite3Fts3OpenTokenizer( |
+ pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); |
+ if( rc==SQLITE_OK ){ |
+ int ii; |
+ for(ii=0; rc==SQLITE_OK; ii++){ |
+ const char *zByte; |
+ int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; |
+ rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); |
+ if( rc==SQLITE_OK ){ |
+ Fts3PhraseToken *pToken; |
+ |
+ p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); |
+ if( !p ) goto no_mem; |
+ |
+ zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); |
+ if( !zTemp ) goto no_mem; |
+ |
+ assert( nToken==ii ); |
+ pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; |
+ memset(pToken, 0, sizeof(Fts3PhraseToken)); |
+ |
+ memcpy(&zTemp[nTemp], zByte, nByte); |
+ nTemp += nByte; |
+ |
+ pToken->n = nByte; |
+ pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*'); |
+ pToken->bFirst = (iBegin>0 && zInput[iBegin-1]=='^'); |
+ nToken = ii+1; |
+ } |
+ } |
+ |
+ pModule->xClose(pCursor); |
+ pCursor = 0; |
+ } |
+ |
+ if( rc==SQLITE_DONE ){ |
+ int jj; |
+ char *zBuf = 0; |
+ |
+ p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); |
+ if( !p ) goto no_mem; |
+ memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); |
+ p->eType = FTSQUERY_PHRASE; |
+ p->pPhrase = (Fts3Phrase *)&p[1]; |
+ p->pPhrase->iColumn = pParse->iDefaultCol; |
+ p->pPhrase->nToken = nToken; |
+ |
+ zBuf = (char *)&p->pPhrase->aToken[nToken]; |
+ if( zTemp ){ |
+ memcpy(zBuf, zTemp, nTemp); |
+ sqlite3_free(zTemp); |
+ }else{ |
+ assert( nTemp==0 ); |
+ } |
+ |
+ for(jj=0; jj<p->pPhrase->nToken; jj++){ |
+ p->pPhrase->aToken[jj].z = zBuf; |
+ zBuf += p->pPhrase->aToken[jj].n; |
+ } |
+ rc = SQLITE_OK; |
+ } |
+ |
+ *ppExpr = p; |
+ return rc; |
+no_mem: |
+ |
+ if( pCursor ){ |
+ pModule->xClose(pCursor); |
+ } |
+ sqlite3_free(zTemp); |
+ sqlite3_free(p); |
+ *ppExpr = 0; |
+ return SQLITE_NOMEM; |
+} |
+ |
+/* |
+** The output variable *ppExpr is populated with an allocated Fts3Expr |
+** structure, or set to 0 if the end of the input buffer is reached. |
+** |
+** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM |
+** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. |
+** If SQLITE_ERROR is returned, pContext is populated with an error message. |
+*/ |
+static int getNextNode( |
+ ParseContext *pParse, /* fts3 query parse context */ |
+ const char *z, int n, /* Input string */ |
+ Fts3Expr **ppExpr, /* OUT: expression */ |
+ int *pnConsumed /* OUT: Number of bytes consumed */ |
+){ |
+ static const struct Fts3Keyword { |
+ char *z; /* Keyword text */ |
+ unsigned char n; /* Length of the keyword */ |
+ unsigned char parenOnly; /* Only valid in paren mode */ |
+ unsigned char eType; /* Keyword code */ |
+ } aKeyword[] = { |
+ { "OR" , 2, 0, FTSQUERY_OR }, |
+ { "AND", 3, 1, FTSQUERY_AND }, |
+ { "NOT", 3, 1, FTSQUERY_NOT }, |
+ { "NEAR", 4, 0, FTSQUERY_NEAR } |
+ }; |
+ int ii; |
+ int iCol; |
+ int iColLen; |
+ int rc; |
+ Fts3Expr *pRet = 0; |
+ |
+ const char *zInput = z; |
+ int nInput = n; |
+ |
+ pParse->isNot = 0; |
+ |
+ /* Skip over any whitespace before checking for a keyword, an open or |
+ ** close bracket, or a quoted string. |
+ */ |
+ while( nInput>0 && fts3isspace(*zInput) ){ |
+ nInput--; |
+ zInput++; |
+ } |
+ if( nInput==0 ){ |
+ return SQLITE_DONE; |
+ } |
+ |
+ /* See if we are dealing with a keyword. */ |
+ for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ |
+ const struct Fts3Keyword *pKey = &aKeyword[ii]; |
+ |
+ if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ |
+ continue; |
+ } |
+ |
+ if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ |
+ int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; |
+ int nKey = pKey->n; |
+ char cNext; |
+ |
+ /* If this is a "NEAR" keyword, check for an explicit nearness. */ |
+ if( pKey->eType==FTSQUERY_NEAR ){ |
+ assert( nKey==4 ); |
+ if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ |
+ nNear = 0; |
+ for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ |
+ nNear = nNear * 10 + (zInput[nKey] - '0'); |
+ } |
+ } |
+ } |
+ |
+ /* At this point this is probably a keyword. But for that to be true, |
+ ** the next byte must contain either whitespace, an open or close |
+ ** parenthesis, a quote character, or EOF. |
+ */ |
+ cNext = zInput[nKey]; |
+ if( fts3isspace(cNext) |
+ || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 |
+ ){ |
+ pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); |
+ if( !pRet ){ |
+ return SQLITE_NOMEM; |
+ } |
+ pRet->eType = pKey->eType; |
+ pRet->nNear = nNear; |
+ *ppExpr = pRet; |
+ *pnConsumed = (int)((zInput - z) + nKey); |
+ return SQLITE_OK; |
+ } |
+ |
+ /* Turns out that wasn't a keyword after all. This happens if the |
+ ** user has supplied a token such as "ORacle". Continue. |
+ */ |
+ } |
+ } |
+ |
+ /* See if we are dealing with a quoted phrase. If this is the case, then |
+ ** search for the closing quote and pass the whole string to getNextString() |
+ ** for processing. This is easy to do, as fts3 has no syntax for escaping |
+ ** a quote character embedded in a string. |
+ */ |
+ if( *zInput=='"' ){ |
+ for(ii=1; ii<nInput && zInput[ii]!='"'; ii++); |
+ *pnConsumed = (int)((zInput - z) + ii + 1); |
+ if( ii==nInput ){ |
+ return SQLITE_ERROR; |
+ } |
+ return getNextString(pParse, &zInput[1], ii-1, ppExpr); |
+ } |
+ |
+ if( sqlite3_fts3_enable_parentheses ){ |
+ if( *zInput=='(' ){ |
+ int nConsumed = 0; |
+ pParse->nNest++; |
+ rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); |
+ if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } |
+ *pnConsumed = (int)(zInput - z) + 1 + nConsumed; |
+ return rc; |
+ }else if( *zInput==')' ){ |
+ pParse->nNest--; |
+ *pnConsumed = (int)((zInput - z) + 1); |
+ *ppExpr = 0; |
+ return SQLITE_DONE; |
+ } |
+ } |
+ |
+ /* If control flows to this point, this must be a regular token, or |
+ ** the end of the input. Read a regular token using the sqlite3_tokenizer |
+ ** interface. Before doing so, figure out if there is an explicit |
+ ** column specifier for the token. |
+ ** |
+ ** TODO: Strangely, it is not possible to associate a column specifier |
+ ** with a quoted phrase, only with a single token. Not sure if this was |
+ ** an implementation artifact or an intentional decision when fts3 was |
+ ** first implemented. Whichever it was, this module duplicates the |
+ ** limitation. |
+ */ |
+ iCol = pParse->iDefaultCol; |
+ iColLen = 0; |
+ for(ii=0; ii<pParse->nCol; ii++){ |
+ const char *zStr = pParse->azCol[ii]; |
+ int nStr = (int)strlen(zStr); |
+ if( nInput>nStr && zInput[nStr]==':' |
+ && sqlite3_strnicmp(zStr, zInput, nStr)==0 |
+ ){ |
+ iCol = ii; |
+ iColLen = (int)((zInput - z) + nStr + 1); |
+ break; |
+ } |
+ } |
+ rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); |
+ *pnConsumed += iColLen; |
+ return rc; |
+} |
+ |
+/* |
+** The argument is an Fts3Expr structure for a binary operator (any type |
+** except an FTSQUERY_PHRASE). Return an integer value representing the |
+** precedence of the operator. Lower values have a higher precedence (i.e. |
+** group more tightly). For example, in the C language, the == operator |
+** groups more tightly than ||, and would therefore have a higher precedence. |
+** |
+** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS |
+** is defined), the order of the operators in precedence from highest to |
+** lowest is: |
+** |
+** NEAR |
+** NOT |
+** AND (including implicit ANDs) |
+** OR |
+** |
+** Note that when using the old query syntax, the OR operator has a higher |
+** precedence than the AND operator. |
+*/ |
+static int opPrecedence(Fts3Expr *p){ |
+ assert( p->eType!=FTSQUERY_PHRASE ); |
+ if( sqlite3_fts3_enable_parentheses ){ |
+ return p->eType; |
+ }else if( p->eType==FTSQUERY_NEAR ){ |
+ return 1; |
+ }else if( p->eType==FTSQUERY_OR ){ |
+ return 2; |
+ } |
+ assert( p->eType==FTSQUERY_AND ); |
+ return 3; |
+} |
+ |
+/* |
+** Argument ppHead contains a pointer to the current head of a query |
+** expression tree being parsed. pPrev is the expression node most recently |
+** inserted into the tree. This function adds pNew, which is always a binary |
+** operator node, into the expression tree based on the relative precedence |
+** of pNew and the existing nodes of the tree. This may result in the head |
+** of the tree changing, in which case *ppHead is set to the new root node. |
+*/ |
+static void insertBinaryOperator( |
+ Fts3Expr **ppHead, /* Pointer to the root node of a tree */ |
+ Fts3Expr *pPrev, /* Node most recently inserted into the tree */ |
+ Fts3Expr *pNew /* New binary node to insert into expression tree */ |
+){ |
+ Fts3Expr *pSplit = pPrev; |
+ while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ |
+ pSplit = pSplit->pParent; |
+ } |
+ |
+ if( pSplit->pParent ){ |
+ assert( pSplit->pParent->pRight==pSplit ); |
+ pSplit->pParent->pRight = pNew; |
+ pNew->pParent = pSplit->pParent; |
+ }else{ |
+ *ppHead = pNew; |
+ } |
+ pNew->pLeft = pSplit; |
+ pSplit->pParent = pNew; |
+} |
+ |
+/* |
+** Parse the fts3 query expression found in buffer z, length n. This function |
+** returns either when the end of the buffer is reached or an unmatched |
+** closing bracket - ')' - is encountered. |
+** |
+** If successful, SQLITE_OK is returned, *ppExpr is set to point to the |
+** parsed form of the expression and *pnConsumed is set to the number of |
+** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM |
+** (out of memory error) or SQLITE_ERROR (parse error) is returned. |
+*/ |
+static int fts3ExprParse( |
+ ParseContext *pParse, /* fts3 query parse context */ |
+ const char *z, int n, /* Text of MATCH query */ |
+ Fts3Expr **ppExpr, /* OUT: Parsed query structure */ |
+ int *pnConsumed /* OUT: Number of bytes consumed */ |
+){ |
+ Fts3Expr *pRet = 0; |
+ Fts3Expr *pPrev = 0; |
+ Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ |
+ int nIn = n; |
+ const char *zIn = z; |
+ int rc = SQLITE_OK; |
+ int isRequirePhrase = 1; |
+ |
+ while( rc==SQLITE_OK ){ |
+ Fts3Expr *p = 0; |
+ int nByte = 0; |
+ |
+ rc = getNextNode(pParse, zIn, nIn, &p, &nByte); |
+ assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); |
+ if( rc==SQLITE_OK ){ |
+ if( p ){ |
+ int isPhrase; |
+ |
+ if( !sqlite3_fts3_enable_parentheses |
+ && p->eType==FTSQUERY_PHRASE && pParse->isNot |
+ ){ |
+ /* Create an implicit NOT operator. */ |
+ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); |
+ if( !pNot ){ |
+ sqlite3Fts3ExprFree(p); |
+ rc = SQLITE_NOMEM; |
+ goto exprparse_out; |
+ } |
+ pNot->eType = FTSQUERY_NOT; |
+ pNot->pRight = p; |
+ p->pParent = pNot; |
+ if( pNotBranch ){ |
+ pNot->pLeft = pNotBranch; |
+ pNotBranch->pParent = pNot; |
+ } |
+ pNotBranch = pNot; |
+ p = pPrev; |
+ }else{ |
+ int eType = p->eType; |
+ isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); |
+ |
+ /* The isRequirePhrase variable is set to true if a phrase or |
+ ** an expression contained in parenthesis is required. If a |
+ ** binary operator (AND, OR, NOT or NEAR) is encounted when |
+ ** isRequirePhrase is set, this is a syntax error. |
+ */ |
+ if( !isPhrase && isRequirePhrase ){ |
+ sqlite3Fts3ExprFree(p); |
+ rc = SQLITE_ERROR; |
+ goto exprparse_out; |
+ } |
+ |
+ if( isPhrase && !isRequirePhrase ){ |
+ /* Insert an implicit AND operator. */ |
+ Fts3Expr *pAnd; |
+ assert( pRet && pPrev ); |
+ pAnd = fts3MallocZero(sizeof(Fts3Expr)); |
+ if( !pAnd ){ |
+ sqlite3Fts3ExprFree(p); |
+ rc = SQLITE_NOMEM; |
+ goto exprparse_out; |
+ } |
+ pAnd->eType = FTSQUERY_AND; |
+ insertBinaryOperator(&pRet, pPrev, pAnd); |
+ pPrev = pAnd; |
+ } |
+ |
+ /* This test catches attempts to make either operand of a NEAR |
+ ** operator something other than a phrase. For example, either of |
+ ** the following: |
+ ** |
+ ** (bracketed expression) NEAR phrase |
+ ** phrase NEAR (bracketed expression) |
+ ** |
+ ** Return an error in either case. |
+ */ |
+ if( pPrev && ( |
+ (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) |
+ || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) |
+ )){ |
+ sqlite3Fts3ExprFree(p); |
+ rc = SQLITE_ERROR; |
+ goto exprparse_out; |
+ } |
+ |
+ if( isPhrase ){ |
+ if( pRet ){ |
+ assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); |
+ pPrev->pRight = p; |
+ p->pParent = pPrev; |
+ }else{ |
+ pRet = p; |
+ } |
+ }else{ |
+ insertBinaryOperator(&pRet, pPrev, p); |
+ } |
+ isRequirePhrase = !isPhrase; |
+ } |
+ pPrev = p; |
+ } |
+ assert( nByte>0 ); |
+ } |
+ assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); |
+ nIn -= nByte; |
+ zIn += nByte; |
+ } |
+ |
+ if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ if( rc==SQLITE_DONE ){ |
+ rc = SQLITE_OK; |
+ if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ |
+ if( !pRet ){ |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ Fts3Expr *pIter = pNotBranch; |
+ while( pIter->pLeft ){ |
+ pIter = pIter->pLeft; |
+ } |
+ pIter->pLeft = pRet; |
+ pRet->pParent = pIter; |
+ pRet = pNotBranch; |
+ } |
+ } |
+ } |
+ *pnConsumed = n - nIn; |
+ |
+exprparse_out: |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3Fts3ExprFree(pRet); |
+ sqlite3Fts3ExprFree(pNotBranch); |
+ pRet = 0; |
+ } |
+ *ppExpr = pRet; |
+ return rc; |
+} |
+ |
+/* |
+** Return SQLITE_ERROR if the maximum depth of the expression tree passed |
+** as the only argument is more than nMaxDepth. |
+*/ |
+static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ |
+ int rc = SQLITE_OK; |
+ if( p ){ |
+ if( nMaxDepth<0 ){ |
+ rc = SQLITE_TOOBIG; |
+ }else{ |
+ rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); |
+ } |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This function attempts to transform the expression tree at (*pp) to |
+** an equivalent but more balanced form. The tree is modified in place. |
+** If successful, SQLITE_OK is returned and (*pp) set to point to the |
+** new root expression node. |
+** |
+** nMaxDepth is the maximum allowable depth of the balanced sub-tree. |
+** |
+** Otherwise, if an error occurs, an SQLite error code is returned and |
+** expression (*pp) freed. |
+*/ |
+static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ Fts3Expr *pRoot = *pp; /* Initial root node */ |
+ Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ |
+ int eType = pRoot->eType; /* Type of node in this tree */ |
+ |
+ if( nMaxDepth==0 ){ |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ |
+ Fts3Expr **apLeaf; |
+ apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); |
+ if( 0==apLeaf ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ int i; |
+ Fts3Expr *p; |
+ |
+ /* Set $p to point to the left-most leaf in the tree of eType nodes. */ |
+ for(p=pRoot; p->eType==eType; p=p->pLeft){ |
+ assert( p->pParent==0 || p->pParent->pLeft==p ); |
+ assert( p->pLeft && p->pRight ); |
+ } |
+ |
+ /* This loop runs once for each leaf in the tree of eType nodes. */ |
+ while( 1 ){ |
+ int iLvl; |
+ Fts3Expr *pParent = p->pParent; /* Current parent of p */ |
+ |
+ assert( pParent==0 || pParent->pLeft==p ); |
+ p->pParent = 0; |
+ if( pParent ){ |
+ pParent->pLeft = 0; |
+ }else{ |
+ pRoot = 0; |
+ } |
+ rc = fts3ExprBalance(&p, nMaxDepth-1); |
+ if( rc!=SQLITE_OK ) break; |
+ |
+ for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){ |
+ if( apLeaf[iLvl]==0 ){ |
+ apLeaf[iLvl] = p; |
+ p = 0; |
+ }else{ |
+ assert( pFree ); |
+ pFree->pLeft = apLeaf[iLvl]; |
+ pFree->pRight = p; |
+ pFree->pLeft->pParent = pFree; |
+ pFree->pRight->pParent = pFree; |
+ |
+ p = pFree; |
+ pFree = pFree->pParent; |
+ p->pParent = 0; |
+ apLeaf[iLvl] = 0; |
+ } |
+ } |
+ if( p ){ |
+ sqlite3Fts3ExprFree(p); |
+ rc = SQLITE_TOOBIG; |
+ break; |
+ } |
+ |
+ /* If that was the last leaf node, break out of the loop */ |
+ if( pParent==0 ) break; |
+ |
+ /* Set $p to point to the next leaf in the tree of eType nodes */ |
+ for(p=pParent->pRight; p->eType==eType; p=p->pLeft); |
+ |
+ /* Remove pParent from the original tree. */ |
+ assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); |
+ pParent->pRight->pParent = pParent->pParent; |
+ if( pParent->pParent ){ |
+ pParent->pParent->pLeft = pParent->pRight; |
+ }else{ |
+ assert( pParent==pRoot ); |
+ pRoot = pParent->pRight; |
+ } |
+ |
+ /* Link pParent into the free node list. It will be used as an |
+ ** internal node of the new tree. */ |
+ pParent->pParent = pFree; |
+ pFree = pParent; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ p = 0; |
+ for(i=0; i<nMaxDepth; i++){ |
+ if( apLeaf[i] ){ |
+ if( p==0 ){ |
+ p = apLeaf[i]; |
+ p->pParent = 0; |
+ }else{ |
+ assert( pFree!=0 ); |
+ pFree->pRight = p; |
+ pFree->pLeft = apLeaf[i]; |
+ pFree->pLeft->pParent = pFree; |
+ pFree->pRight->pParent = pFree; |
+ |
+ p = pFree; |
+ pFree = pFree->pParent; |
+ p->pParent = 0; |
+ } |
+ } |
+ } |
+ pRoot = p; |
+ }else{ |
+ /* An error occurred. Delete the contents of the apLeaf[] array |
+ ** and pFree list. Everything else is cleaned up by the call to |
+ ** sqlite3Fts3ExprFree(pRoot) below. */ |
+ Fts3Expr *pDel; |
+ for(i=0; i<nMaxDepth; i++){ |
+ sqlite3Fts3ExprFree(apLeaf[i]); |
+ } |
+ while( (pDel=pFree)!=0 ){ |
+ pFree = pDel->pParent; |
+ sqlite3_free(pDel); |
+ } |
+ } |
+ |
+ assert( pFree==0 ); |
+ sqlite3_free( apLeaf ); |
+ } |
+ }else if( eType==FTSQUERY_NOT ){ |
+ Fts3Expr *pLeft = pRoot->pLeft; |
+ Fts3Expr *pRight = pRoot->pRight; |
+ |
+ pRoot->pLeft = 0; |
+ pRoot->pRight = 0; |
+ pLeft->pParent = 0; |
+ pRight->pParent = 0; |
+ |
+ rc = fts3ExprBalance(&pLeft, nMaxDepth-1); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3ExprBalance(&pRight, nMaxDepth-1); |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3Fts3ExprFree(pRight); |
+ sqlite3Fts3ExprFree(pLeft); |
+ }else{ |
+ assert( pLeft && pRight ); |
+ pRoot->pLeft = pLeft; |
+ pLeft->pParent = pRoot; |
+ pRoot->pRight = pRight; |
+ pRight->pParent = pRoot; |
+ } |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3Fts3ExprFree(pRoot); |
+ pRoot = 0; |
+ } |
+ *pp = pRoot; |
+ return rc; |
+} |
+ |
+/* |
+** This function is similar to sqlite3Fts3ExprParse(), with the following |
+** differences: |
+** |
+** 1. It does not do expression rebalancing. |
+** 2. It does not check that the expression does not exceed the |
+** maximum allowable depth. |
+** 3. Even if it fails, *ppExpr may still be set to point to an |
+** expression tree. It should be deleted using sqlite3Fts3ExprFree() |
+** in this case. |
+*/ |
+static int fts3ExprParseUnbalanced( |
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ |
+ int iLangid, /* Language id for tokenizer */ |
+ char **azCol, /* Array of column names for fts3 table */ |
+ int bFts4, /* True to allow FTS4-only syntax */ |
+ int nCol, /* Number of entries in azCol[] */ |
+ int iDefaultCol, /* Default column to query */ |
+ const char *z, int n, /* Text of MATCH query */ |
+ Fts3Expr **ppExpr /* OUT: Parsed query structure */ |
+){ |
+ int nParsed; |
+ int rc; |
+ ParseContext sParse; |
+ |
+ memset(&sParse, 0, sizeof(ParseContext)); |
+ sParse.pTokenizer = pTokenizer; |
+ sParse.iLangid = iLangid; |
+ sParse.azCol = (const char **)azCol; |
+ sParse.nCol = nCol; |
+ sParse.iDefaultCol = iDefaultCol; |
+ sParse.bFts4 = bFts4; |
+ if( z==0 ){ |
+ *ppExpr = 0; |
+ return SQLITE_OK; |
+ } |
+ if( n<0 ){ |
+ n = (int)strlen(z); |
+ } |
+ rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); |
+ assert( rc==SQLITE_OK || *ppExpr==0 ); |
+ |
+ /* Check for mismatched parenthesis */ |
+ if( rc==SQLITE_OK && sParse.nNest ){ |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Parameters z and n contain a pointer to and length of a buffer containing |
+** an fts3 query expression, respectively. This function attempts to parse the |
+** query expression and create a tree of Fts3Expr structures representing the |
+** parsed expression. If successful, *ppExpr is set to point to the head |
+** of the parsed expression tree and SQLITE_OK is returned. If an error |
+** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse |
+** error) is returned and *ppExpr is set to 0. |
+** |
+** If parameter n is a negative number, then z is assumed to point to a |
+** nul-terminated string and the length is determined using strlen(). |
+** |
+** The first parameter, pTokenizer, is passed the fts3 tokenizer module to |
+** use to normalize query tokens while parsing the expression. The azCol[] |
+** array, which is assumed to contain nCol entries, should contain the names |
+** of each column in the target fts3 table, in order from left to right. |
+** Column names must be nul-terminated strings. |
+** |
+** The iDefaultCol parameter should be passed the index of the table column |
+** that appears on the left-hand-side of the MATCH operator (the default |
+** column to match against for tokens for which a column name is not explicitly |
+** specified as part of the query string), or -1 if tokens may by default |
+** match any table column. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3ExprParse( |
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ |
+ int iLangid, /* Language id for tokenizer */ |
+ char **azCol, /* Array of column names for fts3 table */ |
+ int bFts4, /* True to allow FTS4-only syntax */ |
+ int nCol, /* Number of entries in azCol[] */ |
+ int iDefaultCol, /* Default column to query */ |
+ const char *z, int n, /* Text of MATCH query */ |
+ Fts3Expr **ppExpr, /* OUT: Parsed query structure */ |
+ char **pzErr /* OUT: Error message (sqlite3_malloc) */ |
+){ |
+ int rc = fts3ExprParseUnbalanced( |
+ pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr |
+ ); |
+ |
+ /* Rebalance the expression. And check that its depth does not exceed |
+ ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ |
+ if( rc==SQLITE_OK && *ppExpr ){ |
+ rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3Fts3ExprFree(*ppExpr); |
+ *ppExpr = 0; |
+ if( rc==SQLITE_TOOBIG ){ |
+ sqlite3Fts3ErrMsg(pzErr, |
+ "FTS expression tree is too large (maximum depth %d)", |
+ SQLITE_FTS3_MAX_EXPR_DEPTH |
+ ); |
+ rc = SQLITE_ERROR; |
+ }else if( rc==SQLITE_ERROR ){ |
+ sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Free a single node of an expression tree. |
+*/ |
+static void fts3FreeExprNode(Fts3Expr *p){ |
+ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); |
+ sqlite3Fts3EvalPhraseCleanup(p->pPhrase); |
+ sqlite3_free(p->aMI); |
+ sqlite3_free(p); |
+} |
+ |
+/* |
+** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). |
+** |
+** This function would be simpler if it recursively called itself. But |
+** that would mean passing a sufficiently large expression to ExprParse() |
+** could cause a stack overflow. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ |
+ Fts3Expr *p; |
+ assert( pDel==0 || pDel->pParent==0 ); |
+ for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ |
+ assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); |
+ } |
+ while( p ){ |
+ Fts3Expr *pParent = p->pParent; |
+ fts3FreeExprNode(p); |
+ if( pParent && p==pParent->pLeft && pParent->pRight ){ |
+ p = pParent->pRight; |
+ while( p && (p->pLeft || p->pRight) ){ |
+ assert( p==p->pParent->pRight || p==p->pParent->pLeft ); |
+ p = (p->pLeft ? p->pLeft : p->pRight); |
+ } |
+ }else{ |
+ p = pParent; |
+ } |
+ } |
+} |
+ |
+/**************************************************************************** |
+***************************************************************************** |
+** Everything after this point is just test code. |
+*/ |
+ |
+#ifdef SQLITE_TEST |
+ |
+/* #include <stdio.h> */ |
+ |
+/* |
+** Function to query the hash-table of tokenizers (see README.tokenizers). |
+*/ |
+static int queryTestTokenizer( |
+ sqlite3 *db, |
+ const char *zName, |
+ const sqlite3_tokenizer_module **pp |
+){ |
+ int rc; |
+ sqlite3_stmt *pStmt; |
+ const char zSql[] = "SELECT fts3_tokenizer(?)"; |
+ |
+ *pp = 0; |
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ |
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); |
+ } |
+ } |
+ |
+ return sqlite3_finalize(pStmt); |
+} |
+ |
+/* |
+** Return a pointer to a buffer containing a text representation of the |
+** expression passed as the first argument. The buffer is obtained from |
+** sqlite3_malloc(). It is the responsibility of the caller to use |
+** sqlite3_free() to release the memory. If an OOM condition is encountered, |
+** NULL is returned. |
+** |
+** If the second argument is not NULL, then its contents are prepended to |
+** the returned expression text and then freed using sqlite3_free(). |
+*/ |
+static char *exprToString(Fts3Expr *pExpr, char *zBuf){ |
+ if( pExpr==0 ){ |
+ return sqlite3_mprintf(""); |
+ } |
+ switch( pExpr->eType ){ |
+ case FTSQUERY_PHRASE: { |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ int i; |
+ zBuf = sqlite3_mprintf( |
+ "%zPHRASE %d 0", zBuf, pPhrase->iColumn); |
+ for(i=0; zBuf && i<pPhrase->nToken; i++){ |
+ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, |
+ pPhrase->aToken[i].n, pPhrase->aToken[i].z, |
+ (pPhrase->aToken[i].isPrefix?"+":"") |
+ ); |
+ } |
+ return zBuf; |
+ } |
+ |
+ case FTSQUERY_NEAR: |
+ zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); |
+ break; |
+ case FTSQUERY_NOT: |
+ zBuf = sqlite3_mprintf("%zNOT ", zBuf); |
+ break; |
+ case FTSQUERY_AND: |
+ zBuf = sqlite3_mprintf("%zAND ", zBuf); |
+ break; |
+ case FTSQUERY_OR: |
+ zBuf = sqlite3_mprintf("%zOR ", zBuf); |
+ break; |
+ } |
+ |
+ if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); |
+ if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); |
+ if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); |
+ |
+ if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); |
+ if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); |
+ |
+ return zBuf; |
+} |
+ |
+/* |
+** This is the implementation of a scalar SQL function used to test the |
+** expression parser. It should be called as follows: |
+** |
+** fts3_exprtest(<tokenizer>, <expr>, <column 1>, ...); |
+** |
+** The first argument, <tokenizer>, is the name of the fts3 tokenizer used |
+** to parse the query expression (see README.tokenizers). The second argument |
+** is the query expression to parse. Each subsequent argument is the name |
+** of a column of the fts3 table that the query expression may refer to. |
+** For example: |
+** |
+** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); |
+*/ |
+static void fts3ExprTest( |
+ sqlite3_context *context, |
+ int argc, |
+ sqlite3_value **argv |
+){ |
+ sqlite3_tokenizer_module const *pModule = 0; |
+ sqlite3_tokenizer *pTokenizer = 0; |
+ int rc; |
+ char **azCol = 0; |
+ const char *zExpr; |
+ int nExpr; |
+ int nCol; |
+ int ii; |
+ Fts3Expr *pExpr; |
+ char *zBuf = 0; |
+ sqlite3 *db = sqlite3_context_db_handle(context); |
+ |
+ if( argc<3 ){ |
+ sqlite3_result_error(context, |
+ "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 |
+ ); |
+ return; |
+ } |
+ |
+ rc = queryTestTokenizer(db, |
+ (const char *)sqlite3_value_text(argv[0]), &pModule); |
+ if( rc==SQLITE_NOMEM ){ |
+ sqlite3_result_error_nomem(context); |
+ goto exprtest_out; |
+ }else if( !pModule ){ |
+ sqlite3_result_error(context, "No such tokenizer module", -1); |
+ goto exprtest_out; |
+ } |
+ |
+ rc = pModule->xCreate(0, 0, &pTokenizer); |
+ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); |
+ if( rc==SQLITE_NOMEM ){ |
+ sqlite3_result_error_nomem(context); |
+ goto exprtest_out; |
+ } |
+ pTokenizer->pModule = pModule; |
+ |
+ zExpr = (const char *)sqlite3_value_text(argv[1]); |
+ nExpr = sqlite3_value_bytes(argv[1]); |
+ nCol = argc-2; |
+ azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); |
+ if( !azCol ){ |
+ sqlite3_result_error_nomem(context); |
+ goto exprtest_out; |
+ } |
+ for(ii=0; ii<nCol; ii++){ |
+ azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); |
+ } |
+ |
+ if( sqlite3_user_data(context) ){ |
+ char *zDummy = 0; |
+ rc = sqlite3Fts3ExprParse( |
+ pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy |
+ ); |
+ assert( rc==SQLITE_OK || pExpr==0 ); |
+ sqlite3_free(zDummy); |
+ }else{ |
+ rc = fts3ExprParseUnbalanced( |
+ pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr |
+ ); |
+ } |
+ |
+ if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ |
+ sqlite3Fts3ExprFree(pExpr); |
+ sqlite3_result_error(context, "Error parsing expression", -1); |
+ }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ |
+ sqlite3_result_error_nomem(context); |
+ }else{ |
+ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); |
+ sqlite3_free(zBuf); |
+ } |
+ |
+ sqlite3Fts3ExprFree(pExpr); |
+ |
+exprtest_out: |
+ if( pModule && pTokenizer ){ |
+ rc = pModule->xDestroy(pTokenizer); |
+ } |
+ sqlite3_free(azCol); |
+} |
+ |
+/* |
+** Register the query expression parser test function fts3_exprtest() |
+** with database connection db. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ |
+ int rc = sqlite3_create_function( |
+ db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 |
+ ); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", |
+ -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 |
+ ); |
+ } |
+ return rc; |
+} |
+ |
+#endif |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_expr.c *******************************************/ |
+/************** Begin file fts3_hash.c ***************************************/ |
+/* |
+** 2001 September 22 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+************************************************************************* |
+** This is the implementation of generic hash-tables used in SQLite. |
+** We've modified it slightly to serve as a standalone hash table |
+** implementation for the full-text indexing module. |
+*/ |
+ |
+/* |
+** The code in this file is only compiled if: |
+** |
+** * The FTS3 module is being built as an extension |
+** (in which case SQLITE_CORE is not defined), or |
+** |
+** * The FTS3 module is being built into the core of |
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <assert.h> */ |
+/* #include <stdlib.h> */ |
+/* #include <string.h> */ |
+ |
+/* #include "fts3_hash.h" */ |
+ |
+/* |
+** Malloc and Free functions |
+*/ |
+static void *fts3HashMalloc(int n){ |
+ void *p = sqlite3_malloc(n); |
+ if( p ){ |
+ memset(p, 0, n); |
+ } |
+ return p; |
+} |
+static void fts3HashFree(void *p){ |
+ sqlite3_free(p); |
+} |
+ |
+/* Turn bulk memory into a hash table object by initializing the |
+** fields of the Hash structure. |
+** |
+** "pNew" is a pointer to the hash table that is to be initialized. |
+** keyClass is one of the constants |
+** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass |
+** determines what kind of key the hash table will use. "copyKey" is |
+** true if the hash table should make its own private copy of keys and |
+** false if it should just use the supplied pointer. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ |
+ assert( pNew!=0 ); |
+ assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); |
+ pNew->keyClass = keyClass; |
+ pNew->copyKey = copyKey; |
+ pNew->first = 0; |
+ pNew->count = 0; |
+ pNew->htsize = 0; |
+ pNew->ht = 0; |
+} |
+ |
+/* Remove all entries from a hash table. Reclaim all memory. |
+** Call this routine to delete a hash table or to reset a hash table |
+** to the empty state. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ |
+ Fts3HashElem *elem; /* For looping over all elements of the table */ |
+ |
+ assert( pH!=0 ); |
+ elem = pH->first; |
+ pH->first = 0; |
+ fts3HashFree(pH->ht); |
+ pH->ht = 0; |
+ pH->htsize = 0; |
+ while( elem ){ |
+ Fts3HashElem *next_elem = elem->next; |
+ if( pH->copyKey && elem->pKey ){ |
+ fts3HashFree(elem->pKey); |
+ } |
+ fts3HashFree(elem); |
+ elem = next_elem; |
+ } |
+ pH->count = 0; |
+} |
+ |
+/* |
+** Hash and comparison functions when the mode is FTS3_HASH_STRING |
+*/ |
+static int fts3StrHash(const void *pKey, int nKey){ |
+ const char *z = (const char *)pKey; |
+ unsigned h = 0; |
+ if( nKey<=0 ) nKey = (int) strlen(z); |
+ while( nKey > 0 ){ |
+ h = (h<<3) ^ h ^ *z++; |
+ nKey--; |
+ } |
+ return (int)(h & 0x7fffffff); |
+} |
+static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ |
+ if( n1!=n2 ) return 1; |
+ return strncmp((const char*)pKey1,(const char*)pKey2,n1); |
+} |
+ |
+/* |
+** Hash and comparison functions when the mode is FTS3_HASH_BINARY |
+*/ |
+static int fts3BinHash(const void *pKey, int nKey){ |
+ int h = 0; |
+ const char *z = (const char *)pKey; |
+ while( nKey-- > 0 ){ |
+ h = (h<<3) ^ h ^ *(z++); |
+ } |
+ return h & 0x7fffffff; |
+} |
+static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ |
+ if( n1!=n2 ) return 1; |
+ return memcmp(pKey1,pKey2,n1); |
+} |
+ |
+/* |
+** Return a pointer to the appropriate hash function given the key class. |
+** |
+** The C syntax in this function definition may be unfamilar to some |
+** programmers, so we provide the following additional explanation: |
+** |
+** The name of the function is "ftsHashFunction". The function takes a |
+** single parameter "keyClass". The return value of ftsHashFunction() |
+** is a pointer to another function. Specifically, the return value |
+** of ftsHashFunction() is a pointer to a function that takes two parameters |
+** with types "const void*" and "int" and returns an "int". |
+*/ |
+static int (*ftsHashFunction(int keyClass))(const void*,int){ |
+ if( keyClass==FTS3_HASH_STRING ){ |
+ return &fts3StrHash; |
+ }else{ |
+ assert( keyClass==FTS3_HASH_BINARY ); |
+ return &fts3BinHash; |
+ } |
+} |
+ |
+/* |
+** Return a pointer to the appropriate hash function given the key class. |
+** |
+** For help in interpreted the obscure C code in the function definition, |
+** see the header comment on the previous function. |
+*/ |
+static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ |
+ if( keyClass==FTS3_HASH_STRING ){ |
+ return &fts3StrCompare; |
+ }else{ |
+ assert( keyClass==FTS3_HASH_BINARY ); |
+ return &fts3BinCompare; |
+ } |
+} |
+ |
+/* Link an element into the hash table |
+*/ |
+static void fts3HashInsertElement( |
+ Fts3Hash *pH, /* The complete hash table */ |
+ struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ |
+ Fts3HashElem *pNew /* The element to be inserted */ |
+){ |
+ Fts3HashElem *pHead; /* First element already in pEntry */ |
+ pHead = pEntry->chain; |
+ if( pHead ){ |
+ pNew->next = pHead; |
+ pNew->prev = pHead->prev; |
+ if( pHead->prev ){ pHead->prev->next = pNew; } |
+ else { pH->first = pNew; } |
+ pHead->prev = pNew; |
+ }else{ |
+ pNew->next = pH->first; |
+ if( pH->first ){ pH->first->prev = pNew; } |
+ pNew->prev = 0; |
+ pH->first = pNew; |
+ } |
+ pEntry->count++; |
+ pEntry->chain = pNew; |
+} |
+ |
+ |
+/* Resize the hash table so that it cantains "new_size" buckets. |
+** "new_size" must be a power of 2. The hash table might fail |
+** to resize if sqliteMalloc() fails. |
+** |
+** Return non-zero if a memory allocation error occurs. |
+*/ |
+static int fts3Rehash(Fts3Hash *pH, int new_size){ |
+ struct _fts3ht *new_ht; /* The new hash table */ |
+ Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ |
+ int (*xHash)(const void*,int); /* The hash function */ |
+ |
+ assert( (new_size & (new_size-1))==0 ); |
+ new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); |
+ if( new_ht==0 ) return 1; |
+ fts3HashFree(pH->ht); |
+ pH->ht = new_ht; |
+ pH->htsize = new_size; |
+ xHash = ftsHashFunction(pH->keyClass); |
+ for(elem=pH->first, pH->first=0; elem; elem = next_elem){ |
+ int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); |
+ next_elem = elem->next; |
+ fts3HashInsertElement(pH, &new_ht[h], elem); |
+ } |
+ return 0; |
+} |
+ |
+/* This function (for internal use only) locates an element in an |
+** hash table that matches the given key. The hash for this key has |
+** already been computed and is passed as the 4th parameter. |
+*/ |
+static Fts3HashElem *fts3FindElementByHash( |
+ const Fts3Hash *pH, /* The pH to be searched */ |
+ const void *pKey, /* The key we are searching for */ |
+ int nKey, |
+ int h /* The hash for this key. */ |
+){ |
+ Fts3HashElem *elem; /* Used to loop thru the element list */ |
+ int count; /* Number of elements left to test */ |
+ int (*xCompare)(const void*,int,const void*,int); /* comparison function */ |
+ |
+ if( pH->ht ){ |
+ struct _fts3ht *pEntry = &pH->ht[h]; |
+ elem = pEntry->chain; |
+ count = pEntry->count; |
+ xCompare = ftsCompareFunction(pH->keyClass); |
+ while( count-- && elem ){ |
+ if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ |
+ return elem; |
+ } |
+ elem = elem->next; |
+ } |
+ } |
+ return 0; |
+} |
+ |
+/* Remove a single entry from the hash table given a pointer to that |
+** element and a hash on the element's key. |
+*/ |
+static void fts3RemoveElementByHash( |
+ Fts3Hash *pH, /* The pH containing "elem" */ |
+ Fts3HashElem* elem, /* The element to be removed from the pH */ |
+ int h /* Hash value for the element */ |
+){ |
+ struct _fts3ht *pEntry; |
+ if( elem->prev ){ |
+ elem->prev->next = elem->next; |
+ }else{ |
+ pH->first = elem->next; |
+ } |
+ if( elem->next ){ |
+ elem->next->prev = elem->prev; |
+ } |
+ pEntry = &pH->ht[h]; |
+ if( pEntry->chain==elem ){ |
+ pEntry->chain = elem->next; |
+ } |
+ pEntry->count--; |
+ if( pEntry->count<=0 ){ |
+ pEntry->chain = 0; |
+ } |
+ if( pH->copyKey && elem->pKey ){ |
+ fts3HashFree(elem->pKey); |
+ } |
+ fts3HashFree( elem ); |
+ pH->count--; |
+ if( pH->count<=0 ){ |
+ assert( pH->first==0 ); |
+ assert( pH->count==0 ); |
+ fts3HashClear(pH); |
+ } |
+} |
+ |
+SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( |
+ const Fts3Hash *pH, |
+ const void *pKey, |
+ int nKey |
+){ |
+ int h; /* A hash on key */ |
+ int (*xHash)(const void*,int); /* The hash function */ |
+ |
+ if( pH==0 || pH->ht==0 ) return 0; |
+ xHash = ftsHashFunction(pH->keyClass); |
+ assert( xHash!=0 ); |
+ h = (*xHash)(pKey,nKey); |
+ assert( (pH->htsize & (pH->htsize-1))==0 ); |
+ return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); |
+} |
+ |
+/* |
+** Attempt to locate an element of the hash table pH with a key |
+** that matches pKey,nKey. Return the data for this element if it is |
+** found, or NULL if there is no match. |
+*/ |
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ |
+ Fts3HashElem *pElem; /* The element that matches key (if any) */ |
+ |
+ pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); |
+ return pElem ? pElem->data : 0; |
+} |
+ |
+/* Insert an element into the hash table pH. The key is pKey,nKey |
+** and the data is "data". |
+** |
+** If no element exists with a matching key, then a new |
+** element is created. A copy of the key is made if the copyKey |
+** flag is set. NULL is returned. |
+** |
+** If another element already exists with the same key, then the |
+** new data replaces the old data and the old data is returned. |
+** The key is not copied in this instance. If a malloc fails, then |
+** the new data is returned and the hash table is unchanged. |
+** |
+** If the "data" parameter to this function is NULL, then the |
+** element corresponding to "key" is removed from the hash table. |
+*/ |
+SQLITE_PRIVATE void *sqlite3Fts3HashInsert( |
+ Fts3Hash *pH, /* The hash table to insert into */ |
+ const void *pKey, /* The key */ |
+ int nKey, /* Number of bytes in the key */ |
+ void *data /* The data */ |
+){ |
+ int hraw; /* Raw hash value of the key */ |
+ int h; /* the hash of the key modulo hash table size */ |
+ Fts3HashElem *elem; /* Used to loop thru the element list */ |
+ Fts3HashElem *new_elem; /* New element added to the pH */ |
+ int (*xHash)(const void*,int); /* The hash function */ |
+ |
+ assert( pH!=0 ); |
+ xHash = ftsHashFunction(pH->keyClass); |
+ assert( xHash!=0 ); |
+ hraw = (*xHash)(pKey, nKey); |
+ assert( (pH->htsize & (pH->htsize-1))==0 ); |
+ h = hraw & (pH->htsize-1); |
+ elem = fts3FindElementByHash(pH,pKey,nKey,h); |
+ if( elem ){ |
+ void *old_data = elem->data; |
+ if( data==0 ){ |
+ fts3RemoveElementByHash(pH,elem,h); |
+ }else{ |
+ elem->data = data; |
+ } |
+ return old_data; |
+ } |
+ if( data==0 ) return 0; |
+ if( (pH->htsize==0 && fts3Rehash(pH,8)) |
+ || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) |
+ ){ |
+ pH->count = 0; |
+ return data; |
+ } |
+ assert( pH->htsize>0 ); |
+ new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); |
+ if( new_elem==0 ) return data; |
+ if( pH->copyKey && pKey!=0 ){ |
+ new_elem->pKey = fts3HashMalloc( nKey ); |
+ if( new_elem->pKey==0 ){ |
+ fts3HashFree(new_elem); |
+ return data; |
+ } |
+ memcpy((void*)new_elem->pKey, pKey, nKey); |
+ }else{ |
+ new_elem->pKey = (void*)pKey; |
+ } |
+ new_elem->nKey = nKey; |
+ pH->count++; |
+ assert( pH->htsize>0 ); |
+ assert( (pH->htsize & (pH->htsize-1))==0 ); |
+ h = hraw & (pH->htsize-1); |
+ fts3HashInsertElement(pH, &pH->ht[h], new_elem); |
+ new_elem->data = data; |
+ return 0; |
+} |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_hash.c *******************************************/ |
+/************** Begin file fts3_porter.c *************************************/ |
+/* |
+** 2006 September 30 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+************************************************************************* |
+** Implementation of the full-text-search tokenizer that implements |
+** a Porter stemmer. |
+*/ |
+ |
+/* |
+** The code in this file is only compiled if: |
+** |
+** * The FTS3 module is being built as an extension |
+** (in which case SQLITE_CORE is not defined), or |
+** |
+** * The FTS3 module is being built into the core of |
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <assert.h> */ |
+/* #include <stdlib.h> */ |
+/* #include <stdio.h> */ |
+/* #include <string.h> */ |
+ |
+/* #include "fts3_tokenizer.h" */ |
+ |
+/* |
+** Class derived from sqlite3_tokenizer |
+*/ |
+typedef struct porter_tokenizer { |
+ sqlite3_tokenizer base; /* Base class */ |
+} porter_tokenizer; |
+ |
+/* |
+** Class derived from sqlite3_tokenizer_cursor |
+*/ |
+typedef struct porter_tokenizer_cursor { |
+ sqlite3_tokenizer_cursor base; |
+ const char *zInput; /* input we are tokenizing */ |
+ int nInput; /* size of the input */ |
+ int iOffset; /* current position in zInput */ |
+ int iToken; /* index of next token to be returned */ |
+ char *zToken; /* storage for current token */ |
+ int nAllocated; /* space allocated to zToken buffer */ |
+} porter_tokenizer_cursor; |
+ |
+ |
+/* |
+** Create a new tokenizer instance. |
+*/ |
+static int porterCreate( |
+ int argc, const char * const *argv, |
+ sqlite3_tokenizer **ppTokenizer |
+){ |
+ porter_tokenizer *t; |
+ |
+ UNUSED_PARAMETER(argc); |
+ UNUSED_PARAMETER(argv); |
+ |
+ t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); |
+ if( t==NULL ) return SQLITE_NOMEM; |
+ memset(t, 0, sizeof(*t)); |
+ *ppTokenizer = &t->base; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Destroy a tokenizer |
+*/ |
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){ |
+ sqlite3_free(pTokenizer); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Prepare to begin tokenizing a particular string. The input |
+** string to be tokenized is zInput[0..nInput-1]. A cursor |
+** used to incrementally tokenize this string is returned in |
+** *ppCursor. |
+*/ |
+static int porterOpen( |
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */ |
+ const char *zInput, int nInput, /* String to be tokenized */ |
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ |
+){ |
+ porter_tokenizer_cursor *c; |
+ |
+ UNUSED_PARAMETER(pTokenizer); |
+ |
+ c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); |
+ if( c==NULL ) return SQLITE_NOMEM; |
+ |
+ c->zInput = zInput; |
+ if( zInput==0 ){ |
+ c->nInput = 0; |
+ }else if( nInput<0 ){ |
+ c->nInput = (int)strlen(zInput); |
+ }else{ |
+ c->nInput = nInput; |
+ } |
+ c->iOffset = 0; /* start tokenizing at the beginning */ |
+ c->iToken = 0; |
+ c->zToken = NULL; /* no space allocated, yet. */ |
+ c->nAllocated = 0; |
+ |
+ *ppCursor = &c->base; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Close a tokenization cursor previously opened by a call to |
+** porterOpen() above. |
+*/ |
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){ |
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; |
+ sqlite3_free(c->zToken); |
+ sqlite3_free(c); |
+ return SQLITE_OK; |
+} |
+/* |
+** Vowel or consonant |
+*/ |
+static const char cType[] = { |
+ 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, |
+ 1, 1, 1, 2, 1 |
+}; |
+ |
+/* |
+** isConsonant() and isVowel() determine if their first character in |
+** the string they point to is a consonant or a vowel, according |
+** to Porter ruls. |
+** |
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. |
+** 'Y' is a consonant unless it follows another consonant, |
+** in which case it is a vowel. |
+** |
+** In these routine, the letters are in reverse order. So the 'y' rule |
+** is that 'y' is a consonant unless it is followed by another |
+** consonent. |
+*/ |
+static int isVowel(const char*); |
+static int isConsonant(const char *z){ |
+ int j; |
+ char x = *z; |
+ if( x==0 ) return 0; |
+ assert( x>='a' && x<='z' ); |
+ j = cType[x-'a']; |
+ if( j<2 ) return j; |
+ return z[1]==0 || isVowel(z + 1); |
+} |
+static int isVowel(const char *z){ |
+ int j; |
+ char x = *z; |
+ if( x==0 ) return 0; |
+ assert( x>='a' && x<='z' ); |
+ j = cType[x-'a']; |
+ if( j<2 ) return 1-j; |
+ return isConsonant(z + 1); |
+} |
+ |
+/* |
+** Let any sequence of one or more vowels be represented by V and let |
+** C be sequence of one or more consonants. Then every word can be |
+** represented as: |
+** |
+** [C] (VC){m} [V] |
+** |
+** In prose: A word is an optional consonant followed by zero or |
+** vowel-consonant pairs followed by an optional vowel. "m" is the |
+** number of vowel consonant pairs. This routine computes the value |
+** of m for the first i bytes of a word. |
+** |
+** Return true if the m-value for z is 1 or more. In other words, |
+** return true if z contains at least one vowel that is followed |
+** by a consonant. |
+** |
+** In this routine z[] is in reverse order. So we are really looking |
+** for an instance of a consonant followed by a vowel. |
+*/ |
+static int m_gt_0(const char *z){ |
+ while( isVowel(z) ){ z++; } |
+ if( *z==0 ) return 0; |
+ while( isConsonant(z) ){ z++; } |
+ return *z!=0; |
+} |
+ |
+/* Like mgt0 above except we are looking for a value of m which is |
+** exactly 1 |
+*/ |
+static int m_eq_1(const char *z){ |
+ while( isVowel(z) ){ z++; } |
+ if( *z==0 ) return 0; |
+ while( isConsonant(z) ){ z++; } |
+ if( *z==0 ) return 0; |
+ while( isVowel(z) ){ z++; } |
+ if( *z==0 ) return 1; |
+ while( isConsonant(z) ){ z++; } |
+ return *z==0; |
+} |
+ |
+/* Like mgt0 above except we are looking for a value of m>1 instead |
+** or m>0 |
+*/ |
+static int m_gt_1(const char *z){ |
+ while( isVowel(z) ){ z++; } |
+ if( *z==0 ) return 0; |
+ while( isConsonant(z) ){ z++; } |
+ if( *z==0 ) return 0; |
+ while( isVowel(z) ){ z++; } |
+ if( *z==0 ) return 0; |
+ while( isConsonant(z) ){ z++; } |
+ return *z!=0; |
+} |
+ |
+/* |
+** Return TRUE if there is a vowel anywhere within z[0..n-1] |
+*/ |
+static int hasVowel(const char *z){ |
+ while( isConsonant(z) ){ z++; } |
+ return *z!=0; |
+} |
+ |
+/* |
+** Return TRUE if the word ends in a double consonant. |
+** |
+** The text is reversed here. So we are really looking at |
+** the first two characters of z[]. |
+*/ |
+static int doubleConsonant(const char *z){ |
+ return isConsonant(z) && z[0]==z[1]; |
+} |
+ |
+/* |
+** Return TRUE if the word ends with three letters which |
+** are consonant-vowel-consonent and where the final consonant |
+** is not 'w', 'x', or 'y'. |
+** |
+** The word is reversed here. So we are really checking the |
+** first three letters and the first one cannot be in [wxy]. |
+*/ |
+static int star_oh(const char *z){ |
+ return |
+ isConsonant(z) && |
+ z[0]!='w' && z[0]!='x' && z[0]!='y' && |
+ isVowel(z+1) && |
+ isConsonant(z+2); |
+} |
+ |
+/* |
+** If the word ends with zFrom and xCond() is true for the stem |
+** of the word that preceeds the zFrom ending, then change the |
+** ending to zTo. |
+** |
+** The input word *pz and zFrom are both in reverse order. zTo |
+** is in normal order. |
+** |
+** Return TRUE if zFrom matches. Return FALSE if zFrom does not |
+** match. Not that TRUE is returned even if xCond() fails and |
+** no substitution occurs. |
+*/ |
+static int stem( |
+ char **pz, /* The word being stemmed (Reversed) */ |
+ const char *zFrom, /* If the ending matches this... (Reversed) */ |
+ const char *zTo, /* ... change the ending to this (not reversed) */ |
+ int (*xCond)(const char*) /* Condition that must be true */ |
+){ |
+ char *z = *pz; |
+ while( *zFrom && *zFrom==*z ){ z++; zFrom++; } |
+ if( *zFrom!=0 ) return 0; |
+ if( xCond && !xCond(z) ) return 1; |
+ while( *zTo ){ |
+ *(--z) = *(zTo++); |
+ } |
+ *pz = z; |
+ return 1; |
+} |
+ |
+/* |
+** This is the fallback stemmer used when the porter stemmer is |
+** inappropriate. The input word is copied into the output with |
+** US-ASCII case folding. If the input word is too long (more |
+** than 20 bytes if it contains no digits or more than 6 bytes if |
+** it contains digits) then word is truncated to 20 or 6 bytes |
+** by taking 10 or 3 bytes from the beginning and end. |
+*/ |
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ |
+ int i, mx, j; |
+ int hasDigit = 0; |
+ for(i=0; i<nIn; i++){ |
+ char c = zIn[i]; |
+ if( c>='A' && c<='Z' ){ |
+ zOut[i] = c - 'A' + 'a'; |
+ }else{ |
+ if( c>='0' && c<='9' ) hasDigit = 1; |
+ zOut[i] = c; |
+ } |
+ } |
+ mx = hasDigit ? 3 : 10; |
+ if( nIn>mx*2 ){ |
+ for(j=mx, i=nIn-mx; i<nIn; i++, j++){ |
+ zOut[j] = zOut[i]; |
+ } |
+ i = j; |
+ } |
+ zOut[i] = 0; |
+ *pnOut = i; |
+} |
+ |
+ |
+/* |
+** Stem the input word zIn[0..nIn-1]. Store the output in zOut. |
+** zOut is at least big enough to hold nIn bytes. Write the actual |
+** size of the output word (exclusive of the '\0' terminator) into *pnOut. |
+** |
+** Any upper-case characters in the US-ASCII character set ([A-Z]) |
+** are converted to lower case. Upper-case UTF characters are |
+** unchanged. |
+** |
+** Words that are longer than about 20 bytes are stemmed by retaining |
+** a few bytes from the beginning and the end of the word. If the |
+** word contains digits, 3 bytes are taken from the beginning and |
+** 3 bytes from the end. For long words without digits, 10 bytes |
+** are taken from each end. US-ASCII case folding still applies. |
+** |
+** If the input word contains not digits but does characters not |
+** in [a-zA-Z] then no stemming is attempted and this routine just |
+** copies the input into the input into the output with US-ASCII |
+** case folding. |
+** |
+** Stemming never increases the length of the word. So there is |
+** no chance of overflowing the zOut buffer. |
+*/ |
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ |
+ int i, j; |
+ char zReverse[28]; |
+ char *z, *z2; |
+ if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){ |
+ /* The word is too big or too small for the porter stemmer. |
+ ** Fallback to the copy stemmer */ |
+ copy_stemmer(zIn, nIn, zOut, pnOut); |
+ return; |
+ } |
+ for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){ |
+ char c = zIn[i]; |
+ if( c>='A' && c<='Z' ){ |
+ zReverse[j] = c + 'a' - 'A'; |
+ }else if( c>='a' && c<='z' ){ |
+ zReverse[j] = c; |
+ }else{ |
+ /* The use of a character not in [a-zA-Z] means that we fallback |
+ ** to the copy stemmer */ |
+ copy_stemmer(zIn, nIn, zOut, pnOut); |
+ return; |
+ } |
+ } |
+ memset(&zReverse[sizeof(zReverse)-5], 0, 5); |
+ z = &zReverse[j+1]; |
+ |
+ |
+ /* Step 1a */ |
+ if( z[0]=='s' ){ |
+ if( |
+ !stem(&z, "sess", "ss", 0) && |
+ !stem(&z, "sei", "i", 0) && |
+ !stem(&z, "ss", "ss", 0) |
+ ){ |
+ z++; |
+ } |
+ } |
+ |
+ /* Step 1b */ |
+ z2 = z; |
+ if( stem(&z, "dee", "ee", m_gt_0) ){ |
+ /* Do nothing. The work was all in the test */ |
+ }else if( |
+ (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) |
+ && z!=z2 |
+ ){ |
+ if( stem(&z, "ta", "ate", 0) || |
+ stem(&z, "lb", "ble", 0) || |
+ stem(&z, "zi", "ize", 0) ){ |
+ /* Do nothing. The work was all in the test */ |
+ }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ |
+ z++; |
+ }else if( m_eq_1(z) && star_oh(z) ){ |
+ *(--z) = 'e'; |
+ } |
+ } |
+ |
+ /* Step 1c */ |
+ if( z[0]=='y' && hasVowel(z+1) ){ |
+ z[0] = 'i'; |
+ } |
+ |
+ /* Step 2 */ |
+ switch( z[1] ){ |
+ case 'a': |
+ if( !stem(&z, "lanoita", "ate", m_gt_0) ){ |
+ stem(&z, "lanoit", "tion", m_gt_0); |
+ } |
+ break; |
+ case 'c': |
+ if( !stem(&z, "icne", "ence", m_gt_0) ){ |
+ stem(&z, "icna", "ance", m_gt_0); |
+ } |
+ break; |
+ case 'e': |
+ stem(&z, "rezi", "ize", m_gt_0); |
+ break; |
+ case 'g': |
+ stem(&z, "igol", "log", m_gt_0); |
+ break; |
+ case 'l': |
+ if( !stem(&z, "ilb", "ble", m_gt_0) |
+ && !stem(&z, "illa", "al", m_gt_0) |
+ && !stem(&z, "iltne", "ent", m_gt_0) |
+ && !stem(&z, "ile", "e", m_gt_0) |
+ ){ |
+ stem(&z, "ilsuo", "ous", m_gt_0); |
+ } |
+ break; |
+ case 'o': |
+ if( !stem(&z, "noitazi", "ize", m_gt_0) |
+ && !stem(&z, "noita", "ate", m_gt_0) |
+ ){ |
+ stem(&z, "rota", "ate", m_gt_0); |
+ } |
+ break; |
+ case 's': |
+ if( !stem(&z, "msila", "al", m_gt_0) |
+ && !stem(&z, "ssenevi", "ive", m_gt_0) |
+ && !stem(&z, "ssenluf", "ful", m_gt_0) |
+ ){ |
+ stem(&z, "ssensuo", "ous", m_gt_0); |
+ } |
+ break; |
+ case 't': |
+ if( !stem(&z, "itila", "al", m_gt_0) |
+ && !stem(&z, "itivi", "ive", m_gt_0) |
+ ){ |
+ stem(&z, "itilib", "ble", m_gt_0); |
+ } |
+ break; |
+ } |
+ |
+ /* Step 3 */ |
+ switch( z[0] ){ |
+ case 'e': |
+ if( !stem(&z, "etaci", "ic", m_gt_0) |
+ && !stem(&z, "evita", "", m_gt_0) |
+ ){ |
+ stem(&z, "ezila", "al", m_gt_0); |
+ } |
+ break; |
+ case 'i': |
+ stem(&z, "itici", "ic", m_gt_0); |
+ break; |
+ case 'l': |
+ if( !stem(&z, "laci", "ic", m_gt_0) ){ |
+ stem(&z, "luf", "", m_gt_0); |
+ } |
+ break; |
+ case 's': |
+ stem(&z, "ssen", "", m_gt_0); |
+ break; |
+ } |
+ |
+ /* Step 4 */ |
+ switch( z[1] ){ |
+ case 'a': |
+ if( z[0]=='l' && m_gt_1(z+2) ){ |
+ z += 2; |
+ } |
+ break; |
+ case 'c': |
+ if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ |
+ z += 4; |
+ } |
+ break; |
+ case 'e': |
+ if( z[0]=='r' && m_gt_1(z+2) ){ |
+ z += 2; |
+ } |
+ break; |
+ case 'i': |
+ if( z[0]=='c' && m_gt_1(z+2) ){ |
+ z += 2; |
+ } |
+ break; |
+ case 'l': |
+ if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ |
+ z += 4; |
+ } |
+ break; |
+ case 'n': |
+ if( z[0]=='t' ){ |
+ if( z[2]=='a' ){ |
+ if( m_gt_1(z+3) ){ |
+ z += 3; |
+ } |
+ }else if( z[2]=='e' ){ |
+ if( !stem(&z, "tneme", "", m_gt_1) |
+ && !stem(&z, "tnem", "", m_gt_1) |
+ ){ |
+ stem(&z, "tne", "", m_gt_1); |
+ } |
+ } |
+ } |
+ break; |
+ case 'o': |
+ if( z[0]=='u' ){ |
+ if( m_gt_1(z+2) ){ |
+ z += 2; |
+ } |
+ }else if( z[3]=='s' || z[3]=='t' ){ |
+ stem(&z, "noi", "", m_gt_1); |
+ } |
+ break; |
+ case 's': |
+ if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ |
+ z += 3; |
+ } |
+ break; |
+ case 't': |
+ if( !stem(&z, "eta", "", m_gt_1) ){ |
+ stem(&z, "iti", "", m_gt_1); |
+ } |
+ break; |
+ case 'u': |
+ if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ |
+ z += 3; |
+ } |
+ break; |
+ case 'v': |
+ case 'z': |
+ if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ |
+ z += 3; |
+ } |
+ break; |
+ } |
+ |
+ /* Step 5a */ |
+ if( z[0]=='e' ){ |
+ if( m_gt_1(z+1) ){ |
+ z++; |
+ }else if( m_eq_1(z+1) && !star_oh(z+1) ){ |
+ z++; |
+ } |
+ } |
+ |
+ /* Step 5b */ |
+ if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ |
+ z++; |
+ } |
+ |
+ /* z[] is now the stemmed word in reverse order. Flip it back |
+ ** around into forward order and return. |
+ */ |
+ *pnOut = i = (int)strlen(z); |
+ zOut[i] = 0; |
+ while( *z ){ |
+ zOut[--i] = *(z++); |
+ } |
+} |
+ |
+/* |
+** Characters that can be part of a token. We assume any character |
+** whose value is greater than 0x80 (any UTF character) can be |
+** part of a token. In other words, delimiters all must have |
+** values of 0x7f or lower. |
+*/ |
+static const char porterIdChar[] = { |
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ |
+}; |
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) |
+ |
+/* |
+** Extract the next token from a tokenization cursor. The cursor must |
+** have been opened by a prior call to porterOpen(). |
+*/ |
+static int porterNext( |
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ |
+ const char **pzToken, /* OUT: *pzToken is the token text */ |
+ int *pnBytes, /* OUT: Number of bytes in token */ |
+ int *piStartOffset, /* OUT: Starting offset of token */ |
+ int *piEndOffset, /* OUT: Ending offset of token */ |
+ int *piPosition /* OUT: Position integer of token */ |
+){ |
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; |
+ const char *z = c->zInput; |
+ |
+ while( c->iOffset<c->nInput ){ |
+ int iStartOffset, ch; |
+ |
+ /* Scan past delimiter characters */ |
+ while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){ |
+ c->iOffset++; |
+ } |
+ |
+ /* Count non-delimiter characters. */ |
+ iStartOffset = c->iOffset; |
+ while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){ |
+ c->iOffset++; |
+ } |
+ |
+ if( c->iOffset>iStartOffset ){ |
+ int n = c->iOffset-iStartOffset; |
+ if( n>c->nAllocated ){ |
+ char *pNew; |
+ c->nAllocated = n+20; |
+ pNew = sqlite3_realloc(c->zToken, c->nAllocated); |
+ if( !pNew ) return SQLITE_NOMEM; |
+ c->zToken = pNew; |
+ } |
+ porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); |
+ *pzToken = c->zToken; |
+ *piStartOffset = iStartOffset; |
+ *piEndOffset = c->iOffset; |
+ *piPosition = c->iToken++; |
+ return SQLITE_OK; |
+ } |
+ } |
+ return SQLITE_DONE; |
+} |
+ |
+/* |
+** The set of routines that implement the porter-stemmer tokenizer |
+*/ |
+static const sqlite3_tokenizer_module porterTokenizerModule = { |
+ 0, |
+ porterCreate, |
+ porterDestroy, |
+ porterOpen, |
+ porterClose, |
+ porterNext, |
+ 0 |
+}; |
+ |
+/* |
+** Allocate a new porter tokenizer. Return a pointer to the new |
+** tokenizer in *ppModule |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( |
+ sqlite3_tokenizer_module const**ppModule |
+){ |
+ *ppModule = &porterTokenizerModule; |
+} |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_porter.c *****************************************/ |
+/************** Begin file fts3_tokenizer.c **********************************/ |
+/* |
+** 2007 June 22 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This is part of an SQLite module implementing full-text search. |
+** This particular file implements the generic tokenizer interface. |
+*/ |
+ |
+/* |
+** The code in this file is only compiled if: |
+** |
+** * The FTS3 module is being built as an extension |
+** (in which case SQLITE_CORE is not defined), or |
+** |
+** * The FTS3 module is being built into the core of |
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <assert.h> */ |
+/* #include <string.h> */ |
+ |
+/* |
+** Return true if the two-argument version of fts3_tokenizer() |
+** has been activated via a prior call to sqlite3_db_config(db, |
+** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); |
+*/ |
+static int fts3TokenizerEnabled(sqlite3_context *context){ |
+ sqlite3 *db = sqlite3_context_db_handle(context); |
+ int isEnabled = 0; |
+ sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); |
+ return isEnabled; |
+} |
+ |
+/* |
+** Implementation of the SQL scalar function for accessing the underlying |
+** hash table. This function may be called as follows: |
+** |
+** SELECT <function-name>(<key-name>); |
+** SELECT <function-name>(<key-name>, <pointer>); |
+** |
+** where <function-name> is the name passed as the second argument |
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). |
+** |
+** If the <pointer> argument is specified, it must be a blob value |
+** containing a pointer to be stored as the hash data corresponding |
+** to the string <key-name>. If <pointer> is not specified, then |
+** the string <key-name> must already exist in the has table. Otherwise, |
+** an error is returned. |
+** |
+** Whether or not the <pointer> argument is specified, the value returned |
+** is a blob containing the pointer stored as the hash data corresponding |
+** to string <key-name> (after the hash-table is updated, if applicable). |
+*/ |
+static void fts3TokenizerFunc( |
+ sqlite3_context *context, |
+ int argc, |
+ sqlite3_value **argv |
+){ |
+ Fts3Hash *pHash; |
+ void *pPtr = 0; |
+ const unsigned char *zName; |
+ int nName; |
+ |
+ assert( argc==1 || argc==2 ); |
+ |
+ pHash = (Fts3Hash *)sqlite3_user_data(context); |
+ |
+ zName = sqlite3_value_text(argv[0]); |
+ nName = sqlite3_value_bytes(argv[0])+1; |
+ |
+ if( argc==2 ){ |
+ if( fts3TokenizerEnabled(context) ){ |
+ void *pOld; |
+ int n = sqlite3_value_bytes(argv[1]); |
+ if( zName==0 || n!=sizeof(pPtr) ){ |
+ sqlite3_result_error(context, "argument type mismatch", -1); |
+ return; |
+ } |
+ pPtr = *(void **)sqlite3_value_blob(argv[1]); |
+ pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); |
+ if( pOld==pPtr ){ |
+ sqlite3_result_error(context, "out of memory", -1); |
+ } |
+ }else{ |
+ sqlite3_result_error(context, "fts3tokenize disabled", -1); |
+ return; |
+ } |
+ }else{ |
+ if( zName ){ |
+ pPtr = sqlite3Fts3HashFind(pHash, zName, nName); |
+ } |
+ if( !pPtr ){ |
+ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); |
+ sqlite3_result_error(context, zErr, -1); |
+ sqlite3_free(zErr); |
+ return; |
+ } |
+ } |
+ sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ |
+ static const char isFtsIdChar[] = { |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ |
+ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ |
+ }; |
+ return (c&0x80 || isFtsIdChar[(int)(c)]); |
+} |
+ |
+SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ |
+ const char *z1; |
+ const char *z2 = 0; |
+ |
+ /* Find the start of the next token. */ |
+ z1 = zStr; |
+ while( z2==0 ){ |
+ char c = *z1; |
+ switch( c ){ |
+ case '\0': return 0; /* No more tokens here */ |
+ case '\'': |
+ case '"': |
+ case '`': { |
+ z2 = z1; |
+ while( *++z2 && (*z2!=c || *++z2==c) ); |
+ break; |
+ } |
+ case '[': |
+ z2 = &z1[1]; |
+ while( *z2 && z2[0]!=']' ) z2++; |
+ if( *z2 ) z2++; |
+ break; |
+ |
+ default: |
+ if( sqlite3Fts3IsIdChar(*z1) ){ |
+ z2 = &z1[1]; |
+ while( sqlite3Fts3IsIdChar(*z2) ) z2++; |
+ }else{ |
+ z1++; |
+ } |
+ } |
+ } |
+ |
+ *pn = (int)(z2-z1); |
+ return z1; |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( |
+ Fts3Hash *pHash, /* Tokenizer hash table */ |
+ const char *zArg, /* Tokenizer name */ |
+ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ |
+ char **pzErr /* OUT: Set to malloced error message */ |
+){ |
+ int rc; |
+ char *z = (char *)zArg; |
+ int n = 0; |
+ char *zCopy; |
+ char *zEnd; /* Pointer to nul-term of zCopy */ |
+ sqlite3_tokenizer_module *m; |
+ |
+ zCopy = sqlite3_mprintf("%s", zArg); |
+ if( !zCopy ) return SQLITE_NOMEM; |
+ zEnd = &zCopy[strlen(zCopy)]; |
+ |
+ z = (char *)sqlite3Fts3NextToken(zCopy, &n); |
+ if( z==0 ){ |
+ assert( n==0 ); |
+ z = zCopy; |
+ } |
+ z[n] = '\0'; |
+ sqlite3Fts3Dequote(z); |
+ |
+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); |
+ if( !m ){ |
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ char const **aArg = 0; |
+ int iArg = 0; |
+ z = &z[n+1]; |
+ while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){ |
+ int nNew = sizeof(char *)*(iArg+1); |
+ char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew); |
+ if( !aNew ){ |
+ sqlite3_free(zCopy); |
+ sqlite3_free((void *)aArg); |
+ return SQLITE_NOMEM; |
+ } |
+ aArg = aNew; |
+ aArg[iArg++] = z; |
+ z[n] = '\0'; |
+ sqlite3Fts3Dequote(z); |
+ z = &z[n+1]; |
+ } |
+ rc = m->xCreate(iArg, aArg, ppTok); |
+ assert( rc!=SQLITE_OK || *ppTok ); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); |
+ }else{ |
+ (*ppTok)->pModule = m; |
+ } |
+ sqlite3_free((void *)aArg); |
+ } |
+ |
+ sqlite3_free(zCopy); |
+ return rc; |
+} |
+ |
+ |
+#ifdef SQLITE_TEST |
+ |
+#if defined(INCLUDE_SQLITE_TCL_H) |
+# include "sqlite_tcl.h" |
+#else |
+# include "tcl.h" |
+#endif |
+/* #include <string.h> */ |
+ |
+/* |
+** Implementation of a special SQL scalar function for testing tokenizers |
+** designed to be used in concert with the Tcl testing framework. This |
+** function must be called with two or more arguments: |
+** |
+** SELECT <function-name>(<key-name>, ..., <input-string>); |
+** |
+** where <function-name> is the name passed as the second argument |
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') |
+** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). |
+** |
+** The return value is a string that may be interpreted as a Tcl |
+** list. For each token in the <input-string>, three elements are |
+** added to the returned list. The first is the token position, the |
+** second is the token text (folded, stemmed, etc.) and the third is the |
+** substring of <input-string> associated with the token. For example, |
+** using the built-in "simple" tokenizer: |
+** |
+** SELECT fts_tokenizer_test('simple', 'I don't see how'); |
+** |
+** will return the string: |
+** |
+** "{0 i I 1 dont don't 2 see see 3 how how}" |
+** |
+*/ |
+static void testFunc( |
+ sqlite3_context *context, |
+ int argc, |
+ sqlite3_value **argv |
+){ |
+ Fts3Hash *pHash; |
+ sqlite3_tokenizer_module *p; |
+ sqlite3_tokenizer *pTokenizer = 0; |
+ sqlite3_tokenizer_cursor *pCsr = 0; |
+ |
+ const char *zErr = 0; |
+ |
+ const char *zName; |
+ int nName; |
+ const char *zInput; |
+ int nInput; |
+ |
+ const char *azArg[64]; |
+ |
+ const char *zToken; |
+ int nToken = 0; |
+ int iStart = 0; |
+ int iEnd = 0; |
+ int iPos = 0; |
+ int i; |
+ |
+ Tcl_Obj *pRet; |
+ |
+ if( argc<2 ){ |
+ sqlite3_result_error(context, "insufficient arguments", -1); |
+ return; |
+ } |
+ |
+ nName = sqlite3_value_bytes(argv[0]); |
+ zName = (const char *)sqlite3_value_text(argv[0]); |
+ nInput = sqlite3_value_bytes(argv[argc-1]); |
+ zInput = (const char *)sqlite3_value_text(argv[argc-1]); |
+ |
+ pHash = (Fts3Hash *)sqlite3_user_data(context); |
+ p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); |
+ |
+ if( !p ){ |
+ char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); |
+ sqlite3_result_error(context, zErr2, -1); |
+ sqlite3_free(zErr2); |
+ return; |
+ } |
+ |
+ pRet = Tcl_NewObj(); |
+ Tcl_IncrRefCount(pRet); |
+ |
+ for(i=1; i<argc-1; i++){ |
+ azArg[i-1] = (const char *)sqlite3_value_text(argv[i]); |
+ } |
+ |
+ if( SQLITE_OK!=p->xCreate(argc-2, azArg, &pTokenizer) ){ |
+ zErr = "error in xCreate()"; |
+ goto finish; |
+ } |
+ pTokenizer->pModule = p; |
+ if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ |
+ zErr = "error in xOpen()"; |
+ goto finish; |
+ } |
+ |
+ while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ |
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); |
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); |
+ zToken = &zInput[iStart]; |
+ nToken = iEnd-iStart; |
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); |
+ } |
+ |
+ if( SQLITE_OK!=p->xClose(pCsr) ){ |
+ zErr = "error in xClose()"; |
+ goto finish; |
+ } |
+ if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ |
+ zErr = "error in xDestroy()"; |
+ goto finish; |
+ } |
+ |
+finish: |
+ if( zErr ){ |
+ sqlite3_result_error(context, zErr, -1); |
+ }else{ |
+ sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); |
+ } |
+ Tcl_DecrRefCount(pRet); |
+} |
+ |
+static |
+int registerTokenizer( |
+ sqlite3 *db, |
+ char *zName, |
+ const sqlite3_tokenizer_module *p |
+){ |
+ int rc; |
+ sqlite3_stmt *pStmt; |
+ const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; |
+ |
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); |
+ sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); |
+ sqlite3_step(pStmt); |
+ |
+ return sqlite3_finalize(pStmt); |
+} |
+ |
+ |
+static |
+int queryTokenizer( |
+ sqlite3 *db, |
+ char *zName, |
+ const sqlite3_tokenizer_module **pp |
+){ |
+ int rc; |
+ sqlite3_stmt *pStmt; |
+ const char zSql[] = "SELECT fts3_tokenizer(?)"; |
+ |
+ *pp = 0; |
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ |
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); |
+ } |
+ } |
+ |
+ return sqlite3_finalize(pStmt); |
+} |
+ |
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); |
+ |
+/* |
+** Implementation of the scalar function fts3_tokenizer_internal_test(). |
+** This function is used for testing only, it is not included in the |
+** build unless SQLITE_TEST is defined. |
+** |
+** The purpose of this is to test that the fts3_tokenizer() function |
+** can be used as designed by the C-code in the queryTokenizer and |
+** registerTokenizer() functions above. These two functions are repeated |
+** in the README.tokenizer file as an example, so it is important to |
+** test them. |
+** |
+** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar |
+** function with no arguments. An assert() will fail if a problem is |
+** detected. i.e.: |
+** |
+** SELECT fts3_tokenizer_internal_test(); |
+** |
+*/ |
+static void intTestFunc( |
+ sqlite3_context *context, |
+ int argc, |
+ sqlite3_value **argv |
+){ |
+ int rc; |
+ const sqlite3_tokenizer_module *p1; |
+ const sqlite3_tokenizer_module *p2; |
+ sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); |
+ |
+ UNUSED_PARAMETER(argc); |
+ UNUSED_PARAMETER(argv); |
+ |
+ /* Test the query function */ |
+ sqlite3Fts3SimpleTokenizerModule(&p1); |
+ rc = queryTokenizer(db, "simple", &p2); |
+ assert( rc==SQLITE_OK ); |
+ assert( p1==p2 ); |
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2); |
+ assert( rc==SQLITE_ERROR ); |
+ assert( p2==0 ); |
+ assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); |
+ |
+ /* Test the storage function */ |
+ if( fts3TokenizerEnabled(context) ){ |
+ rc = registerTokenizer(db, "nosuchtokenizer", p1); |
+ assert( rc==SQLITE_OK ); |
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2); |
+ assert( rc==SQLITE_OK ); |
+ assert( p2==p1 ); |
+ } |
+ |
+ sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); |
+} |
+ |
+#endif |
+ |
+/* |
+** Set up SQL objects in database db used to access the contents of |
+** the hash table pointed to by argument pHash. The hash table must |
+** been initialized to use string keys, and to take a private copy |
+** of the key when a value is inserted. i.e. by a call similar to: |
+** |
+** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); |
+** |
+** This function adds a scalar function (see header comment above |
+** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is |
+** defined at compilation time, a temporary virtual table (see header |
+** comment above struct HashTableVtab) to the database schema. Both |
+** provide read/write access to the contents of *pHash. |
+** |
+** The third argument to this function, zName, is used as the name |
+** of both the scalar and, if created, the virtual table. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3InitHashTable( |
+ sqlite3 *db, |
+ Fts3Hash *pHash, |
+ const char *zName |
+){ |
+ int rc = SQLITE_OK; |
+ void *p = (void *)pHash; |
+ const int any = SQLITE_ANY; |
+ |
+#ifdef SQLITE_TEST |
+ char *zTest = 0; |
+ char *zTest2 = 0; |
+ void *pdb = (void *)db; |
+ zTest = sqlite3_mprintf("%s_test", zName); |
+ zTest2 = sqlite3_mprintf("%s_internal_test", zName); |
+ if( !zTest || !zTest2 ){ |
+ rc = SQLITE_NOMEM; |
+ } |
+#endif |
+ |
+ if( SQLITE_OK==rc ){ |
+ rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); |
+ } |
+ if( SQLITE_OK==rc ){ |
+ rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); |
+ } |
+#ifdef SQLITE_TEST |
+ if( SQLITE_OK==rc ){ |
+ rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); |
+ } |
+ if( SQLITE_OK==rc ){ |
+ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); |
+ } |
+#endif |
+ |
+#ifdef SQLITE_TEST |
+ sqlite3_free(zTest); |
+ sqlite3_free(zTest2); |
+#endif |
+ |
+ return rc; |
+} |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_tokenizer.c **************************************/ |
+/************** Begin file fts3_tokenizer1.c *********************************/ |
+/* |
+** 2006 Oct 10 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** Implementation of the "simple" full-text-search tokenizer. |
+*/ |
+ |
+/* |
+** The code in this file is only compiled if: |
+** |
+** * The FTS3 module is being built as an extension |
+** (in which case SQLITE_CORE is not defined), or |
+** |
+** * The FTS3 module is being built into the core of |
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <assert.h> */ |
+/* #include <stdlib.h> */ |
+/* #include <stdio.h> */ |
+/* #include <string.h> */ |
+ |
+/* #include "fts3_tokenizer.h" */ |
+ |
+typedef struct simple_tokenizer { |
+ sqlite3_tokenizer base; |
+ char delim[128]; /* flag ASCII delimiters */ |
+} simple_tokenizer; |
+ |
+typedef struct simple_tokenizer_cursor { |
+ sqlite3_tokenizer_cursor base; |
+ const char *pInput; /* input we are tokenizing */ |
+ int nBytes; /* size of the input */ |
+ int iOffset; /* current position in pInput */ |
+ int iToken; /* index of next token to be returned */ |
+ char *pToken; /* storage for current token */ |
+ int nTokenAllocated; /* space allocated to zToken buffer */ |
+} simple_tokenizer_cursor; |
+ |
+ |
+static int simpleDelim(simple_tokenizer *t, unsigned char c){ |
+ return c<0x80 && t->delim[c]; |
+} |
+static int fts3_isalnum(int x){ |
+ return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); |
+} |
+ |
+/* |
+** Create a new tokenizer instance. |
+*/ |
+static int simpleCreate( |
+ int argc, const char * const *argv, |
+ sqlite3_tokenizer **ppTokenizer |
+){ |
+ simple_tokenizer *t; |
+ |
+ t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); |
+ if( t==NULL ) return SQLITE_NOMEM; |
+ memset(t, 0, sizeof(*t)); |
+ |
+ /* TODO(shess) Delimiters need to remain the same from run to run, |
+ ** else we need to reindex. One solution would be a meta-table to |
+ ** track such information in the database, then we'd only want this |
+ ** information on the initial create. |
+ */ |
+ if( argc>1 ){ |
+ int i, n = (int)strlen(argv[1]); |
+ for(i=0; i<n; i++){ |
+ unsigned char ch = argv[1][i]; |
+ /* We explicitly don't support UTF-8 delimiters for now. */ |
+ if( ch>=0x80 ){ |
+ sqlite3_free(t); |
+ return SQLITE_ERROR; |
+ } |
+ t->delim[ch] = 1; |
+ } |
+ } else { |
+ /* Mark non-alphanumeric ASCII characters as delimiters */ |
+ int i; |
+ for(i=1; i<0x80; i++){ |
+ t->delim[i] = !fts3_isalnum(i) ? -1 : 0; |
+ } |
+ } |
+ |
+ *ppTokenizer = &t->base; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Destroy a tokenizer |
+*/ |
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ |
+ sqlite3_free(pTokenizer); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Prepare to begin tokenizing a particular string. The input |
+** string to be tokenized is pInput[0..nBytes-1]. A cursor |
+** used to incrementally tokenize this string is returned in |
+** *ppCursor. |
+*/ |
+static int simpleOpen( |
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */ |
+ const char *pInput, int nBytes, /* String to be tokenized */ |
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ |
+){ |
+ simple_tokenizer_cursor *c; |
+ |
+ UNUSED_PARAMETER(pTokenizer); |
+ |
+ c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); |
+ if( c==NULL ) return SQLITE_NOMEM; |
+ |
+ c->pInput = pInput; |
+ if( pInput==0 ){ |
+ c->nBytes = 0; |
+ }else if( nBytes<0 ){ |
+ c->nBytes = (int)strlen(pInput); |
+ }else{ |
+ c->nBytes = nBytes; |
+ } |
+ c->iOffset = 0; /* start tokenizing at the beginning */ |
+ c->iToken = 0; |
+ c->pToken = NULL; /* no space allocated, yet. */ |
+ c->nTokenAllocated = 0; |
+ |
+ *ppCursor = &c->base; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Close a tokenization cursor previously opened by a call to |
+** simpleOpen() above. |
+*/ |
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ |
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; |
+ sqlite3_free(c->pToken); |
+ sqlite3_free(c); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Extract the next token from a tokenization cursor. The cursor must |
+** have been opened by a prior call to simpleOpen(). |
+*/ |
+static int simpleNext( |
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ |
+ const char **ppToken, /* OUT: *ppToken is the token text */ |
+ int *pnBytes, /* OUT: Number of bytes in token */ |
+ int *piStartOffset, /* OUT: Starting offset of token */ |
+ int *piEndOffset, /* OUT: Ending offset of token */ |
+ int *piPosition /* OUT: Position integer of token */ |
+){ |
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; |
+ simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; |
+ unsigned char *p = (unsigned char *)c->pInput; |
+ |
+ while( c->iOffset<c->nBytes ){ |
+ int iStartOffset; |
+ |
+ /* Scan past delimiter characters */ |
+ while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){ |
+ c->iOffset++; |
+ } |
+ |
+ /* Count non-delimiter characters. */ |
+ iStartOffset = c->iOffset; |
+ while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){ |
+ c->iOffset++; |
+ } |
+ |
+ if( c->iOffset>iStartOffset ){ |
+ int i, n = c->iOffset-iStartOffset; |
+ if( n>c->nTokenAllocated ){ |
+ char *pNew; |
+ c->nTokenAllocated = n+20; |
+ pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); |
+ if( !pNew ) return SQLITE_NOMEM; |
+ c->pToken = pNew; |
+ } |
+ for(i=0; i<n; i++){ |
+ /* TODO(shess) This needs expansion to handle UTF-8 |
+ ** case-insensitivity. |
+ */ |
+ unsigned char ch = p[iStartOffset+i]; |
+ c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); |
+ } |
+ *ppToken = c->pToken; |
+ *pnBytes = n; |
+ *piStartOffset = iStartOffset; |
+ *piEndOffset = c->iOffset; |
+ *piPosition = c->iToken++; |
+ |
+ return SQLITE_OK; |
+ } |
+ } |
+ return SQLITE_DONE; |
+} |
+ |
+/* |
+** The set of routines that implement the simple tokenizer |
+*/ |
+static const sqlite3_tokenizer_module simpleTokenizerModule = { |
+ 0, |
+ simpleCreate, |
+ simpleDestroy, |
+ simpleOpen, |
+ simpleClose, |
+ simpleNext, |
+ 0, |
+}; |
+ |
+/* |
+** Allocate a new simple tokenizer. Return a pointer to the new |
+** tokenizer in *ppModule |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( |
+ sqlite3_tokenizer_module const**ppModule |
+){ |
+ *ppModule = &simpleTokenizerModule; |
+} |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_tokenizer1.c *************************************/ |
+/************** Begin file fts3_tokenize_vtab.c ******************************/ |
+/* |
+** 2013 Apr 22 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This file contains code for the "fts3tokenize" virtual table module. |
+** An fts3tokenize virtual table is created as follows: |
+** |
+** CREATE VIRTUAL TABLE <tbl> USING fts3tokenize( |
+** <tokenizer-name>, <arg-1>, ... |
+** ); |
+** |
+** The table created has the following schema: |
+** |
+** CREATE TABLE <tbl>(input, token, start, end, position) |
+** |
+** When queried, the query must include a WHERE clause of type: |
+** |
+** input = <string> |
+** |
+** The virtual table module tokenizes this <string>, using the FTS3 |
+** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE |
+** statement and returns one row for each token in the result. With |
+** fields set as follows: |
+** |
+** input: Always set to a copy of <string> |
+** token: A token from the input. |
+** start: Byte offset of the token within the input <string>. |
+** end: Byte offset of the byte immediately following the end of the |
+** token within the input string. |
+** pos: Token offset of token within input. |
+** |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <string.h> */ |
+/* #include <assert.h> */ |
+ |
+typedef struct Fts3tokTable Fts3tokTable; |
+typedef struct Fts3tokCursor Fts3tokCursor; |
+ |
+/* |
+** Virtual table structure. |
+*/ |
+struct Fts3tokTable { |
+ sqlite3_vtab base; /* Base class used by SQLite core */ |
+ const sqlite3_tokenizer_module *pMod; |
+ sqlite3_tokenizer *pTok; |
+}; |
+ |
+/* |
+** Virtual table cursor structure. |
+*/ |
+struct Fts3tokCursor { |
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */ |
+ char *zInput; /* Input string */ |
+ sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ |
+ int iRowid; /* Current 'rowid' value */ |
+ const char *zToken; /* Current 'token' value */ |
+ int nToken; /* Size of zToken in bytes */ |
+ int iStart; /* Current 'start' value */ |
+ int iEnd; /* Current 'end' value */ |
+ int iPos; /* Current 'pos' value */ |
+}; |
+ |
+/* |
+** Query FTS for the tokenizer implementation named zName. |
+*/ |
+static int fts3tokQueryTokenizer( |
+ Fts3Hash *pHash, |
+ const char *zName, |
+ const sqlite3_tokenizer_module **pp, |
+ char **pzErr |
+){ |
+ sqlite3_tokenizer_module *p; |
+ int nName = (int)strlen(zName); |
+ |
+ p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); |
+ if( !p ){ |
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); |
+ return SQLITE_ERROR; |
+ } |
+ |
+ *pp = p; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The second argument, argv[], is an array of pointers to nul-terminated |
+** strings. This function makes a copy of the array and strings into a |
+** single block of memory. It then dequotes any of the strings that appear |
+** to be quoted. |
+** |
+** If successful, output parameter *pazDequote is set to point at the |
+** array of dequoted strings and SQLITE_OK is returned. The caller is |
+** responsible for eventually calling sqlite3_free() to free the array |
+** in this case. Or, if an error occurs, an SQLite error code is returned. |
+** The final value of *pazDequote is undefined in this case. |
+*/ |
+static int fts3tokDequoteArray( |
+ int argc, /* Number of elements in argv[] */ |
+ const char * const *argv, /* Input array */ |
+ char ***pazDequote /* Output array */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ if( argc==0 ){ |
+ *pazDequote = 0; |
+ }else{ |
+ int i; |
+ int nByte = 0; |
+ char **azDequote; |
+ |
+ for(i=0; i<argc; i++){ |
+ nByte += (int)(strlen(argv[i]) + 1); |
+ } |
+ |
+ *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte); |
+ if( azDequote==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ char *pSpace = (char *)&azDequote[argc]; |
+ for(i=0; i<argc; i++){ |
+ int n = (int)strlen(argv[i]); |
+ azDequote[i] = pSpace; |
+ memcpy(pSpace, argv[i], n+1); |
+ sqlite3Fts3Dequote(pSpace); |
+ pSpace += (n+1); |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Schema of the tokenizer table. |
+*/ |
+#define FTS3_TOK_SCHEMA "CREATE TABLE x(input, token, start, end, position)" |
+ |
+/* |
+** This function does all the work for both the xConnect and xCreate methods. |
+** These tables have no persistent representation of their own, so xConnect |
+** and xCreate are identical operations. |
+** |
+** argv[0]: module name |
+** argv[1]: database name |
+** argv[2]: table name |
+** argv[3]: first argument (tokenizer name) |
+*/ |
+static int fts3tokConnectMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pHash, /* Hash table of tokenizers */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ Fts3tokTable *pTab = 0; |
+ const sqlite3_tokenizer_module *pMod = 0; |
+ sqlite3_tokenizer *pTok = 0; |
+ int rc; |
+ char **azDequote = 0; |
+ int nDequote; |
+ |
+ rc = sqlite3_declare_vtab(db, FTS3_TOK_SCHEMA); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ nDequote = argc-3; |
+ rc = fts3tokDequoteArray(nDequote, &argv[3], &azDequote); |
+ |
+ if( rc==SQLITE_OK ){ |
+ const char *zModule; |
+ if( nDequote<1 ){ |
+ zModule = "simple"; |
+ }else{ |
+ zModule = azDequote[0]; |
+ } |
+ rc = fts3tokQueryTokenizer((Fts3Hash*)pHash, zModule, &pMod, pzErr); |
+ } |
+ |
+ assert( (rc==SQLITE_OK)==(pMod!=0) ); |
+ if( rc==SQLITE_OK ){ |
+ const char * const *azArg = (const char * const *)&azDequote[1]; |
+ rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); |
+ if( pTab==0 ){ |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ memset(pTab, 0, sizeof(Fts3tokTable)); |
+ pTab->pMod = pMod; |
+ pTab->pTok = pTok; |
+ *ppVtab = &pTab->base; |
+ }else{ |
+ if( pTok ){ |
+ pMod->xDestroy(pTok); |
+ } |
+ } |
+ |
+ sqlite3_free(azDequote); |
+ return rc; |
+} |
+ |
+/* |
+** This function does the work for both the xDisconnect and xDestroy methods. |
+** These tables have no persistent representation of their own, so xDisconnect |
+** and xDestroy are identical operations. |
+*/ |
+static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ |
+ Fts3tokTable *pTab = (Fts3tokTable *)pVtab; |
+ |
+ pTab->pMod->xDestroy(pTab->pTok); |
+ sqlite3_free(pTab); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xBestIndex - Analyze a WHERE and ORDER BY clause. |
+*/ |
+static int fts3tokBestIndexMethod( |
+ sqlite3_vtab *pVTab, |
+ sqlite3_index_info *pInfo |
+){ |
+ int i; |
+ UNUSED_PARAMETER(pVTab); |
+ |
+ for(i=0; i<pInfo->nConstraint; i++){ |
+ if( pInfo->aConstraint[i].usable |
+ && pInfo->aConstraint[i].iColumn==0 |
+ && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ |
+ ){ |
+ pInfo->idxNum = 1; |
+ pInfo->aConstraintUsage[i].argvIndex = 1; |
+ pInfo->aConstraintUsage[i].omit = 1; |
+ pInfo->estimatedCost = 1; |
+ return SQLITE_OK; |
+ } |
+ } |
+ |
+ pInfo->idxNum = 0; |
+ assert( pInfo->estimatedCost>1000000.0 ); |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xOpen - Open a cursor. |
+*/ |
+static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ |
+ Fts3tokCursor *pCsr; |
+ UNUSED_PARAMETER(pVTab); |
+ |
+ pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); |
+ if( pCsr==0 ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pCsr, 0, sizeof(Fts3tokCursor)); |
+ |
+ *ppCsr = (sqlite3_vtab_cursor *)pCsr; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Reset the tokenizer cursor passed as the only argument. As if it had |
+** just been returned by fts3tokOpenMethod(). |
+*/ |
+static void fts3tokResetCursor(Fts3tokCursor *pCsr){ |
+ if( pCsr->pCsr ){ |
+ Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); |
+ pTab->pMod->xClose(pCsr->pCsr); |
+ pCsr->pCsr = 0; |
+ } |
+ sqlite3_free(pCsr->zInput); |
+ pCsr->zInput = 0; |
+ pCsr->zToken = 0; |
+ pCsr->nToken = 0; |
+ pCsr->iStart = 0; |
+ pCsr->iEnd = 0; |
+ pCsr->iPos = 0; |
+ pCsr->iRowid = 0; |
+} |
+ |
+/* |
+** xClose - Close a cursor. |
+*/ |
+static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; |
+ |
+ fts3tokResetCursor(pCsr); |
+ sqlite3_free(pCsr); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xNext - Advance the cursor to the next row, if any. |
+*/ |
+static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; |
+ Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); |
+ int rc; /* Return code */ |
+ |
+ pCsr->iRowid++; |
+ rc = pTab->pMod->xNext(pCsr->pCsr, |
+ &pCsr->zToken, &pCsr->nToken, |
+ &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos |
+ ); |
+ |
+ if( rc!=SQLITE_OK ){ |
+ fts3tokResetCursor(pCsr); |
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** xFilter - Initialize a cursor to point at the start of its data. |
+*/ |
+static int fts3tokFilterMethod( |
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ |
+ int idxNum, /* Strategy index */ |
+ const char *idxStr, /* Unused */ |
+ int nVal, /* Number of elements in apVal */ |
+ sqlite3_value **apVal /* Arguments for the indexing scheme */ |
+){ |
+ int rc = SQLITE_ERROR; |
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; |
+ Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); |
+ UNUSED_PARAMETER(idxStr); |
+ UNUSED_PARAMETER(nVal); |
+ |
+ fts3tokResetCursor(pCsr); |
+ if( idxNum==1 ){ |
+ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); |
+ int nByte = sqlite3_value_bytes(apVal[0]); |
+ pCsr->zInput = sqlite3_malloc(nByte+1); |
+ if( pCsr->zInput==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memcpy(pCsr->zInput, zByte, nByte); |
+ pCsr->zInput[nByte] = 0; |
+ rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); |
+ if( rc==SQLITE_OK ){ |
+ pCsr->pCsr->pTokenizer = pTab->pTok; |
+ } |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ) return rc; |
+ return fts3tokNextMethod(pCursor); |
+} |
+ |
+/* |
+** xEof - Return true if the cursor is at EOF, or false otherwise. |
+*/ |
+static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; |
+ return (pCsr->zToken==0); |
+} |
+ |
+/* |
+** xColumn - Return a column value. |
+*/ |
+static int fts3tokColumnMethod( |
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ |
+ int iCol /* Index of column to read value from */ |
+){ |
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; |
+ |
+ /* CREATE TABLE x(input, token, start, end, position) */ |
+ switch( iCol ){ |
+ case 0: |
+ sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); |
+ break; |
+ case 1: |
+ sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); |
+ break; |
+ case 2: |
+ sqlite3_result_int(pCtx, pCsr->iStart); |
+ break; |
+ case 3: |
+ sqlite3_result_int(pCtx, pCsr->iEnd); |
+ break; |
+ default: |
+ assert( iCol==4 ); |
+ sqlite3_result_int(pCtx, pCsr->iPos); |
+ break; |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xRowid - Return the current rowid for the cursor. |
+*/ |
+static int fts3tokRowidMethod( |
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
+ sqlite_int64 *pRowid /* OUT: Rowid value */ |
+){ |
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; |
+ *pRowid = (sqlite3_int64)pCsr->iRowid; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Register the fts3tok module with database connection db. Return SQLITE_OK |
+** if successful or an error code if sqlite3_create_module() fails. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ |
+ static const sqlite3_module fts3tok_module = { |
+ 0, /* iVersion */ |
+ fts3tokConnectMethod, /* xCreate */ |
+ fts3tokConnectMethod, /* xConnect */ |
+ fts3tokBestIndexMethod, /* xBestIndex */ |
+ fts3tokDisconnectMethod, /* xDisconnect */ |
+ fts3tokDisconnectMethod, /* xDestroy */ |
+ fts3tokOpenMethod, /* xOpen */ |
+ fts3tokCloseMethod, /* xClose */ |
+ fts3tokFilterMethod, /* xFilter */ |
+ fts3tokNextMethod, /* xNext */ |
+ fts3tokEofMethod, /* xEof */ |
+ fts3tokColumnMethod, /* xColumn */ |
+ fts3tokRowidMethod, /* xRowid */ |
+ 0, /* xUpdate */ |
+ 0, /* xBegin */ |
+ 0, /* xSync */ |
+ 0, /* xCommit */ |
+ 0, /* xRollback */ |
+ 0, /* xFindFunction */ |
+ 0, /* xRename */ |
+ 0, /* xSavepoint */ |
+ 0, /* xRelease */ |
+ 0 /* xRollbackTo */ |
+ }; |
+ int rc; /* Return code */ |
+ |
+ rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); |
+ return rc; |
+} |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_tokenize_vtab.c **********************************/ |
+/************** Begin file fts3_write.c **************************************/ |
+/* |
+** 2009 Oct 23 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This file is part of the SQLite FTS3 extension module. Specifically, |
+** this file contains code to insert, update and delete rows from FTS3 |
+** tables. It also contains code to merge FTS3 b-tree segments. Some |
+** of the sub-routines used to merge segments are also used by the query |
+** code in fts3.c. |
+*/ |
+ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <string.h> */ |
+/* #include <assert.h> */ |
+/* #include <stdlib.h> */ |
+ |
+ |
+#define FTS_MAX_APPENDABLE_HEIGHT 16 |
+ |
+/* |
+** When full-text index nodes are loaded from disk, the buffer that they |
+** are loaded into has the following number of bytes of padding at the end |
+** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer |
+** of 920 bytes is allocated for it. |
+** |
+** This means that if we have a pointer into a buffer containing node data, |
+** it is always safe to read up to two varints from it without risking an |
+** overread, even if the node data is corrupted. |
+*/ |
+#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) |
+ |
+/* |
+** Under certain circumstances, b-tree nodes (doclists) can be loaded into |
+** memory incrementally instead of all at once. This can be a big performance |
+** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() |
+** method before retrieving all query results (as may happen, for example, |
+** if a query has a LIMIT clause). |
+** |
+** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD |
+** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. |
+** The code is written so that the hard lower-limit for each of these values |
+** is 1. Clearly such small values would be inefficient, but can be useful |
+** for testing purposes. |
+** |
+** If this module is built with SQLITE_TEST defined, these constants may |
+** be overridden at runtime for testing purposes. File fts3_test.c contains |
+** a Tcl interface to read and write the values. |
+*/ |
+#ifdef SQLITE_TEST |
+int test_fts3_node_chunksize = (4*1024); |
+int test_fts3_node_chunk_threshold = (4*1024)*4; |
+# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize |
+# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold |
+#else |
+# define FTS3_NODE_CHUNKSIZE (4*1024) |
+# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) |
+#endif |
+ |
+/* |
+** The two values that may be meaningfully bound to the :1 parameter in |
+** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. |
+*/ |
+#define FTS_STAT_DOCTOTAL 0 |
+#define FTS_STAT_INCRMERGEHINT 1 |
+#define FTS_STAT_AUTOINCRMERGE 2 |
+ |
+/* |
+** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic |
+** and incremental merge operation that takes place. This is used for |
+** debugging FTS only, it should not usually be turned on in production |
+** systems. |
+*/ |
+#ifdef FTS3_LOG_MERGES |
+static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ |
+ sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); |
+} |
+#else |
+#define fts3LogMerge(x, y) |
+#endif |
+ |
+ |
+typedef struct PendingList PendingList; |
+typedef struct SegmentNode SegmentNode; |
+typedef struct SegmentWriter SegmentWriter; |
+ |
+/* |
+** An instance of the following data structure is used to build doclists |
+** incrementally. See function fts3PendingListAppend() for details. |
+*/ |
+struct PendingList { |
+ int nData; |
+ char *aData; |
+ int nSpace; |
+ sqlite3_int64 iLastDocid; |
+ sqlite3_int64 iLastCol; |
+ sqlite3_int64 iLastPos; |
+}; |
+ |
+ |
+/* |
+** Each cursor has a (possibly empty) linked list of the following objects. |
+*/ |
+struct Fts3DeferredToken { |
+ Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ |
+ int iCol; /* Column token must occur in */ |
+ Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ |
+ PendingList *pList; /* Doclist is assembled here */ |
+}; |
+ |
+/* |
+** An instance of this structure is used to iterate through the terms on |
+** a contiguous set of segment b-tree leaf nodes. Although the details of |
+** this structure are only manipulated by code in this file, opaque handles |
+** of type Fts3SegReader* are also used by code in fts3.c to iterate through |
+** terms when querying the full-text index. See functions: |
+** |
+** sqlite3Fts3SegReaderNew() |
+** sqlite3Fts3SegReaderFree() |
+** sqlite3Fts3SegReaderIterate() |
+** |
+** Methods used to manipulate Fts3SegReader structures: |
+** |
+** fts3SegReaderNext() |
+** fts3SegReaderFirstDocid() |
+** fts3SegReaderNextDocid() |
+*/ |
+struct Fts3SegReader { |
+ int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ |
+ u8 bLookup; /* True for a lookup only */ |
+ u8 rootOnly; /* True for a root-only reader */ |
+ |
+ sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ |
+ sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ |
+ sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ |
+ sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ |
+ |
+ char *aNode; /* Pointer to node data (or NULL) */ |
+ int nNode; /* Size of buffer at aNode (or 0) */ |
+ int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ |
+ sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ |
+ |
+ Fts3HashElem **ppNextElem; |
+ |
+ /* Variables set by fts3SegReaderNext(). These may be read directly |
+ ** by the caller. They are valid from the time SegmentReaderNew() returns |
+ ** until SegmentReaderNext() returns something other than SQLITE_OK |
+ ** (i.e. SQLITE_DONE). |
+ */ |
+ int nTerm; /* Number of bytes in current term */ |
+ char *zTerm; /* Pointer to current term */ |
+ int nTermAlloc; /* Allocated size of zTerm buffer */ |
+ char *aDoclist; /* Pointer to doclist of current entry */ |
+ int nDoclist; /* Size of doclist in current entry */ |
+ |
+ /* The following variables are used by fts3SegReaderNextDocid() to iterate |
+ ** through the current doclist (aDoclist/nDoclist). |
+ */ |
+ char *pOffsetList; |
+ int nOffsetList; /* For descending pending seg-readers only */ |
+ sqlite3_int64 iDocid; |
+}; |
+ |
+#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) |
+#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) |
+ |
+/* |
+** An instance of this structure is used to create a segment b-tree in the |
+** database. The internal details of this type are only accessed by the |
+** following functions: |
+** |
+** fts3SegWriterAdd() |
+** fts3SegWriterFlush() |
+** fts3SegWriterFree() |
+*/ |
+struct SegmentWriter { |
+ SegmentNode *pTree; /* Pointer to interior tree structure */ |
+ sqlite3_int64 iFirst; /* First slot in %_segments written */ |
+ sqlite3_int64 iFree; /* Next free slot in %_segments */ |
+ char *zTerm; /* Pointer to previous term buffer */ |
+ int nTerm; /* Number of bytes in zTerm */ |
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */ |
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ |
+ int nSize; /* Size of allocation at aData */ |
+ int nData; /* Bytes of data in aData */ |
+ char *aData; /* Pointer to block from malloc() */ |
+ i64 nLeafData; /* Number of bytes of leaf data written */ |
+}; |
+ |
+/* |
+** Type SegmentNode is used by the following three functions to create |
+** the interior part of the segment b+-tree structures (everything except |
+** the leaf nodes). These functions and type are only ever used by code |
+** within the fts3SegWriterXXX() family of functions described above. |
+** |
+** fts3NodeAddTerm() |
+** fts3NodeWrite() |
+** fts3NodeFree() |
+** |
+** When a b+tree is written to the database (either as a result of a merge |
+** or the pending-terms table being flushed), leaves are written into the |
+** database file as soon as they are completely populated. The interior of |
+** the tree is assembled in memory and written out only once all leaves have |
+** been populated and stored. This is Ok, as the b+-tree fanout is usually |
+** very large, meaning that the interior of the tree consumes relatively |
+** little memory. |
+*/ |
+struct SegmentNode { |
+ SegmentNode *pParent; /* Parent node (or NULL for root node) */ |
+ SegmentNode *pRight; /* Pointer to right-sibling */ |
+ SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ |
+ int nEntry; /* Number of terms written to node so far */ |
+ char *zTerm; /* Pointer to previous term buffer */ |
+ int nTerm; /* Number of bytes in zTerm */ |
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */ |
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ |
+ int nData; /* Bytes of valid data so far */ |
+ char *aData; /* Node data */ |
+}; |
+ |
+/* |
+** Valid values for the second argument to fts3SqlStmt(). |
+*/ |
+#define SQL_DELETE_CONTENT 0 |
+#define SQL_IS_EMPTY 1 |
+#define SQL_DELETE_ALL_CONTENT 2 |
+#define SQL_DELETE_ALL_SEGMENTS 3 |
+#define SQL_DELETE_ALL_SEGDIR 4 |
+#define SQL_DELETE_ALL_DOCSIZE 5 |
+#define SQL_DELETE_ALL_STAT 6 |
+#define SQL_SELECT_CONTENT_BY_ROWID 7 |
+#define SQL_NEXT_SEGMENT_INDEX 8 |
+#define SQL_INSERT_SEGMENTS 9 |
+#define SQL_NEXT_SEGMENTS_ID 10 |
+#define SQL_INSERT_SEGDIR 11 |
+#define SQL_SELECT_LEVEL 12 |
+#define SQL_SELECT_LEVEL_RANGE 13 |
+#define SQL_SELECT_LEVEL_COUNT 14 |
+#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 |
+#define SQL_DELETE_SEGDIR_LEVEL 16 |
+#define SQL_DELETE_SEGMENTS_RANGE 17 |
+#define SQL_CONTENT_INSERT 18 |
+#define SQL_DELETE_DOCSIZE 19 |
+#define SQL_REPLACE_DOCSIZE 20 |
+#define SQL_SELECT_DOCSIZE 21 |
+#define SQL_SELECT_STAT 22 |
+#define SQL_REPLACE_STAT 23 |
+ |
+#define SQL_SELECT_ALL_PREFIX_LEVEL 24 |
+#define SQL_DELETE_ALL_TERMS_SEGDIR 25 |
+#define SQL_DELETE_SEGDIR_RANGE 26 |
+#define SQL_SELECT_ALL_LANGID 27 |
+#define SQL_FIND_MERGE_LEVEL 28 |
+#define SQL_MAX_LEAF_NODE_ESTIMATE 29 |
+#define SQL_DELETE_SEGDIR_ENTRY 30 |
+#define SQL_SHIFT_SEGDIR_ENTRY 31 |
+#define SQL_SELECT_SEGDIR 32 |
+#define SQL_CHOMP_SEGDIR 33 |
+#define SQL_SEGMENT_IS_APPENDABLE 34 |
+#define SQL_SELECT_INDEXES 35 |
+#define SQL_SELECT_MXLEVEL 36 |
+ |
+#define SQL_SELECT_LEVEL_RANGE2 37 |
+#define SQL_UPDATE_LEVEL_IDX 38 |
+#define SQL_UPDATE_LEVEL 39 |
+ |
+/* |
+** This function is used to obtain an SQLite prepared statement handle |
+** for the statement identified by the second argument. If successful, |
+** *pp is set to the requested statement handle and SQLITE_OK returned. |
+** Otherwise, an SQLite error code is returned and *pp is set to 0. |
+** |
+** If argument apVal is not NULL, then it must point to an array with |
+** at least as many entries as the requested statement has bound |
+** parameters. The values are bound to the statements parameters before |
+** returning. |
+*/ |
+static int fts3SqlStmt( |
+ Fts3Table *p, /* Virtual table handle */ |
+ int eStmt, /* One of the SQL_XXX constants above */ |
+ sqlite3_stmt **pp, /* OUT: Statement handle */ |
+ sqlite3_value **apVal /* Values to bind to statement */ |
+){ |
+ const char *azSql[] = { |
+/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", |
+/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", |
+/* 2 */ "DELETE FROM %Q.'%q_content'", |
+/* 3 */ "DELETE FROM %Q.'%q_segments'", |
+/* 4 */ "DELETE FROM %Q.'%q_segdir'", |
+/* 5 */ "DELETE FROM %Q.'%q_docsize'", |
+/* 6 */ "DELETE FROM %Q.'%q_stat'", |
+/* 7 */ "SELECT %s WHERE rowid=?", |
+/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", |
+/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", |
+/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", |
+/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", |
+ |
+ /* Return segments in order from oldest to newest.*/ |
+/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " |
+ "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", |
+/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " |
+ "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" |
+ "ORDER BY level DESC, idx ASC", |
+ |
+/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", |
+/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", |
+ |
+/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", |
+/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", |
+/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", |
+/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", |
+/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", |
+/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", |
+/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", |
+/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", |
+/* 24 */ "", |
+/* 25 */ "", |
+ |
+/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", |
+/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", |
+ |
+/* This statement is used to determine which level to read the input from |
+** when performing an incremental merge. It returns the absolute level number |
+** of the oldest level in the db that contains at least ? segments. Or, |
+** if no level in the FTS index contains more than ? segments, the statement |
+** returns zero rows. */ |
+/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " |
+ " GROUP BY level HAVING cnt>=?" |
+ " ORDER BY (level %% 1024) ASC LIMIT 1", |
+ |
+/* Estimate the upper limit on the number of leaf nodes in a new segment |
+** created by merging the oldest :2 segments from absolute level :1. See |
+** function sqlite3Fts3Incrmerge() for details. */ |
+/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " |
+ " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", |
+ |
+/* SQL_DELETE_SEGDIR_ENTRY |
+** Delete the %_segdir entry on absolute level :1 with index :2. */ |
+/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", |
+ |
+/* SQL_SHIFT_SEGDIR_ENTRY |
+** Modify the idx value for the segment with idx=:3 on absolute level :2 |
+** to :1. */ |
+/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", |
+ |
+/* SQL_SELECT_SEGDIR |
+** Read a single entry from the %_segdir table. The entry from absolute |
+** level :1 with index value :2. */ |
+/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " |
+ "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", |
+ |
+/* SQL_CHOMP_SEGDIR |
+** Update the start_block (:1) and root (:2) fields of the %_segdir |
+** entry located on absolute level :3 with index :4. */ |
+/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" |
+ "WHERE level = ? AND idx = ?", |
+ |
+/* SQL_SEGMENT_IS_APPENDABLE |
+** Return a single row if the segment with end_block=? is appendable. Or |
+** no rows otherwise. */ |
+/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", |
+ |
+/* SQL_SELECT_INDEXES |
+** Return the list of valid segment indexes for absolute level ? */ |
+/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", |
+ |
+/* SQL_SELECT_MXLEVEL |
+** Return the largest relative level in the FTS index or indexes. */ |
+/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", |
+ |
+ /* Return segments in order from oldest to newest.*/ |
+/* 37 */ "SELECT level, idx, end_block " |
+ "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " |
+ "ORDER BY level DESC, idx ASC", |
+ |
+ /* Update statements used while promoting segments */ |
+/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " |
+ "WHERE level=? AND idx=?", |
+/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" |
+ |
+ }; |
+ int rc = SQLITE_OK; |
+ sqlite3_stmt *pStmt; |
+ |
+ assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); |
+ assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); |
+ |
+ pStmt = p->aStmt[eStmt]; |
+ if( !pStmt ){ |
+ char *zSql; |
+ if( eStmt==SQL_CONTENT_INSERT ){ |
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); |
+ }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ |
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); |
+ }else{ |
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); |
+ } |
+ if( !zSql ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); |
+ sqlite3_free(zSql); |
+ assert( rc==SQLITE_OK || pStmt==0 ); |
+ p->aStmt[eStmt] = pStmt; |
+ } |
+ } |
+ if( apVal ){ |
+ int i; |
+ int nParam = sqlite3_bind_parameter_count(pStmt); |
+ for(i=0; rc==SQLITE_OK && i<nParam; i++){ |
+ rc = sqlite3_bind_value(pStmt, i+1, apVal[i]); |
+ } |
+ } |
+ *pp = pStmt; |
+ return rc; |
+} |
+ |
+ |
+static int fts3SelectDocsize( |
+ Fts3Table *pTab, /* FTS3 table handle */ |
+ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ |
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */ |
+){ |
+ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ |
+ int rc; /* Return code */ |
+ |
+ rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pStmt, 1, iDocid); |
+ rc = sqlite3_step(pStmt); |
+ if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ |
+ rc = sqlite3_reset(pStmt); |
+ if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; |
+ pStmt = 0; |
+ }else{ |
+ rc = SQLITE_OK; |
+ } |
+ } |
+ |
+ *ppStmt = pStmt; |
+ return rc; |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal( |
+ Fts3Table *pTab, /* Fts3 table handle */ |
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */ |
+){ |
+ sqlite3_stmt *pStmt = 0; |
+ int rc; |
+ rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); |
+ if( sqlite3_step(pStmt)!=SQLITE_ROW |
+ || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB |
+ ){ |
+ rc = sqlite3_reset(pStmt); |
+ if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; |
+ pStmt = 0; |
+ } |
+ } |
+ *ppStmt = pStmt; |
+ return rc; |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize( |
+ Fts3Table *pTab, /* Fts3 table handle */ |
+ sqlite3_int64 iDocid, /* Docid to read size data for */ |
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */ |
+){ |
+ return fts3SelectDocsize(pTab, iDocid, ppStmt); |
+} |
+ |
+/* |
+** Similar to fts3SqlStmt(). Except, after binding the parameters in |
+** array apVal[] to the SQL statement identified by eStmt, the statement |
+** is executed. |
+** |
+** Returns SQLITE_OK if the statement is successfully executed, or an |
+** SQLite error code otherwise. |
+*/ |
+static void fts3SqlExec( |
+ int *pRC, /* Result code */ |
+ Fts3Table *p, /* The FTS3 table */ |
+ int eStmt, /* Index of statement to evaluate */ |
+ sqlite3_value **apVal /* Parameters to bind */ |
+){ |
+ sqlite3_stmt *pStmt; |
+ int rc; |
+ if( *pRC ) return; |
+ rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_step(pStmt); |
+ rc = sqlite3_reset(pStmt); |
+ } |
+ *pRC = rc; |
+} |
+ |
+ |
+/* |
+** This function ensures that the caller has obtained an exclusive |
+** shared-cache table-lock on the %_segdir table. This is required before |
+** writing data to the fts3 table. If this lock is not acquired first, then |
+** the caller may end up attempting to take this lock as part of committing |
+** a transaction, causing SQLite to return SQLITE_LOCKED or |
+** LOCKED_SHAREDCACHEto a COMMIT command. |
+** |
+** It is best to avoid this because if FTS3 returns any error when |
+** committing a transaction, the whole transaction will be rolled back. |
+** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. |
+** It can still happen if the user locks the underlying tables directly |
+** instead of accessing them via FTS. |
+*/ |
+static int fts3Writelock(Fts3Table *p){ |
+ int rc = SQLITE_OK; |
+ |
+ if( p->nPendingData==0 ){ |
+ sqlite3_stmt *pStmt; |
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_null(pStmt, 1); |
+ sqlite3_step(pStmt); |
+ rc = sqlite3_reset(pStmt); |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** FTS maintains a separate indexes for each language-id (a 32-bit integer). |
+** Within each language id, a separate index is maintained to store the |
+** document terms, and each configured prefix size (configured the FTS |
+** "prefix=" option). And each index consists of multiple levels ("relative |
+** levels"). |
+** |
+** All three of these values (the language id, the specific index and the |
+** level within the index) are encoded in 64-bit integer values stored |
+** in the %_segdir table on disk. This function is used to convert three |
+** separate component values into the single 64-bit integer value that |
+** can be used to query the %_segdir table. |
+** |
+** Specifically, each language-id/index combination is allocated 1024 |
+** 64-bit integer level values ("absolute levels"). The main terms index |
+** for language-id 0 is allocate values 0-1023. The first prefix index |
+** (if any) for language-id 0 is allocated values 1024-2047. And so on. |
+** Language 1 indexes are allocated immediately following language 0. |
+** |
+** So, for a system with nPrefix prefix indexes configured, the block of |
+** absolute levels that corresponds to language-id iLangid and index |
+** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). |
+*/ |
+static sqlite3_int64 getAbsoluteLevel( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ int iLangid, /* Language id */ |
+ int iIndex, /* Index in p->aIndex[] */ |
+ int iLevel /* Level of segments */ |
+){ |
+ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ |
+ assert( iLangid>=0 ); |
+ assert( p->nIndex>0 ); |
+ assert( iIndex>=0 && iIndex<p->nIndex ); |
+ |
+ iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; |
+ return iBase + iLevel; |
+} |
+ |
+/* |
+** Set *ppStmt to a statement handle that may be used to iterate through |
+** all rows in the %_segdir table, from oldest to newest. If successful, |
+** return SQLITE_OK. If an error occurs while preparing the statement, |
+** return an SQLite error code. |
+** |
+** There is only ever one instance of this SQL statement compiled for |
+** each FTS3 table. |
+** |
+** The statement returns the following columns from the %_segdir table: |
+** |
+** 0: idx |
+** 1: start_block |
+** 2: leaves_end_block |
+** 3: end_block |
+** 4: root |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( |
+ Fts3Table *p, /* FTS3 table */ |
+ int iLangid, /* Language being queried */ |
+ int iIndex, /* Index for p->aIndex[] */ |
+ int iLevel, /* Level to select (relative level) */ |
+ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ |
+){ |
+ int rc; |
+ sqlite3_stmt *pStmt = 0; |
+ |
+ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); |
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); |
+ assert( iIndex>=0 && iIndex<p->nIndex ); |
+ |
+ if( iLevel<0 ){ |
+ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ |
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); |
+ sqlite3_bind_int64(pStmt, 2, |
+ getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) |
+ ); |
+ } |
+ }else{ |
+ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ |
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); |
+ } |
+ } |
+ *ppStmt = pStmt; |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Append a single varint to a PendingList buffer. SQLITE_OK is returned |
+** if successful, or an SQLite error code otherwise. |
+** |
+** This function also serves to allocate the PendingList structure itself. |
+** For example, to create a new PendingList structure containing two |
+** varints: |
+** |
+** PendingList *p = 0; |
+** fts3PendingListAppendVarint(&p, 1); |
+** fts3PendingListAppendVarint(&p, 2); |
+*/ |
+static int fts3PendingListAppendVarint( |
+ PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ |
+ sqlite3_int64 i /* Value to append to data */ |
+){ |
+ PendingList *p = *pp; |
+ |
+ /* Allocate or grow the PendingList as required. */ |
+ if( !p ){ |
+ p = sqlite3_malloc(sizeof(*p) + 100); |
+ if( !p ){ |
+ return SQLITE_NOMEM; |
+ } |
+ p->nSpace = 100; |
+ p->aData = (char *)&p[1]; |
+ p->nData = 0; |
+ } |
+ else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ |
+ int nNew = p->nSpace * 2; |
+ p = sqlite3_realloc(p, sizeof(*p) + nNew); |
+ if( !p ){ |
+ sqlite3_free(*pp); |
+ *pp = 0; |
+ return SQLITE_NOMEM; |
+ } |
+ p->nSpace = nNew; |
+ p->aData = (char *)&p[1]; |
+ } |
+ |
+ /* Append the new serialized varint to the end of the list. */ |
+ p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); |
+ p->aData[p->nData] = '\0'; |
+ *pp = p; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Add a docid/column/position entry to a PendingList structure. Non-zero |
+** is returned if the structure is sqlite3_realloced as part of adding |
+** the entry. Otherwise, zero. |
+** |
+** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. |
+** Zero is always returned in this case. Otherwise, if no OOM error occurs, |
+** it is set to SQLITE_OK. |
+*/ |
+static int fts3PendingListAppend( |
+ PendingList **pp, /* IN/OUT: PendingList structure */ |
+ sqlite3_int64 iDocid, /* Docid for entry to add */ |
+ sqlite3_int64 iCol, /* Column for entry to add */ |
+ sqlite3_int64 iPos, /* Position of term for entry to add */ |
+ int *pRc /* OUT: Return code */ |
+){ |
+ PendingList *p = *pp; |
+ int rc = SQLITE_OK; |
+ |
+ assert( !p || p->iLastDocid<=iDocid ); |
+ |
+ if( !p || p->iLastDocid!=iDocid ){ |
+ sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); |
+ if( p ){ |
+ assert( p->nData<p->nSpace ); |
+ assert( p->aData[p->nData]==0 ); |
+ p->nData++; |
+ } |
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ |
+ goto pendinglistappend_out; |
+ } |
+ p->iLastCol = -1; |
+ p->iLastPos = 0; |
+ p->iLastDocid = iDocid; |
+ } |
+ if( iCol>0 && p->iLastCol!=iCol ){ |
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) |
+ || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) |
+ ){ |
+ goto pendinglistappend_out; |
+ } |
+ p->iLastCol = iCol; |
+ p->iLastPos = 0; |
+ } |
+ if( iCol>=0 ){ |
+ assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); |
+ rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); |
+ if( rc==SQLITE_OK ){ |
+ p->iLastPos = iPos; |
+ } |
+ } |
+ |
+ pendinglistappend_out: |
+ *pRc = rc; |
+ if( p!=*pp ){ |
+ *pp = p; |
+ return 1; |
+ } |
+ return 0; |
+} |
+ |
+/* |
+** Free a PendingList object allocated by fts3PendingListAppend(). |
+*/ |
+static void fts3PendingListDelete(PendingList *pList){ |
+ sqlite3_free(pList); |
+} |
+ |
+/* |
+** Add an entry to one of the pending-terms hash tables. |
+*/ |
+static int fts3PendingTermsAddOne( |
+ Fts3Table *p, |
+ int iCol, |
+ int iPos, |
+ Fts3Hash *pHash, /* Pending terms hash table to add entry to */ |
+ const char *zToken, |
+ int nToken |
+){ |
+ PendingList *pList; |
+ int rc = SQLITE_OK; |
+ |
+ pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); |
+ if( pList ){ |
+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); |
+ } |
+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ |
+ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ |
+ /* Malloc failed while inserting the new entry. This can only |
+ ** happen if there was no previous entry for this token. |
+ */ |
+ assert( 0==fts3HashFind(pHash, zToken, nToken) ); |
+ sqlite3_free(pList); |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ if( rc==SQLITE_OK ){ |
+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Tokenize the nul-terminated string zText and add all tokens to the |
+** pending-terms hash-table. The docid used is that currently stored in |
+** p->iPrevDocid, and the column is specified by argument iCol. |
+** |
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. |
+*/ |
+static int fts3PendingTermsAdd( |
+ Fts3Table *p, /* Table into which text will be inserted */ |
+ int iLangid, /* Language id to use */ |
+ const char *zText, /* Text of document to be inserted */ |
+ int iCol, /* Column into which text is being inserted */ |
+ u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ |
+){ |
+ int rc; |
+ int iStart = 0; |
+ int iEnd = 0; |
+ int iPos = 0; |
+ int nWord = 0; |
+ |
+ char const *zToken; |
+ int nToken = 0; |
+ |
+ sqlite3_tokenizer *pTokenizer = p->pTokenizer; |
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; |
+ sqlite3_tokenizer_cursor *pCsr; |
+ int (*xNext)(sqlite3_tokenizer_cursor *pCursor, |
+ const char**,int*,int*,int*,int*); |
+ |
+ assert( pTokenizer && pModule ); |
+ |
+ /* If the user has inserted a NULL value, this function may be called with |
+ ** zText==0. In this case, add zero token entries to the hash table and |
+ ** return early. */ |
+ if( zText==0 ){ |
+ *pnWord = 0; |
+ return SQLITE_OK; |
+ } |
+ |
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ xNext = pModule->xNext; |
+ while( SQLITE_OK==rc |
+ && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) |
+ ){ |
+ int i; |
+ if( iPos>=nWord ) nWord = iPos+1; |
+ |
+ /* Positions cannot be negative; we use -1 as a terminator internally. |
+ ** Tokens must have a non-zero length. |
+ */ |
+ if( iPos<0 || !zToken || nToken<=0 ){ |
+ rc = SQLITE_ERROR; |
+ break; |
+ } |
+ |
+ /* Add the term to the terms index */ |
+ rc = fts3PendingTermsAddOne( |
+ p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken |
+ ); |
+ |
+ /* Add the term to each of the prefix indexes that it is not too |
+ ** short for. */ |
+ for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){ |
+ struct Fts3Index *pIndex = &p->aIndex[i]; |
+ if( nToken<pIndex->nPrefix ) continue; |
+ rc = fts3PendingTermsAddOne( |
+ p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix |
+ ); |
+ } |
+ } |
+ |
+ pModule->xClose(pCsr); |
+ *pnWord += nWord; |
+ return (rc==SQLITE_DONE ? SQLITE_OK : rc); |
+} |
+ |
+/* |
+** Calling this function indicates that subsequent calls to |
+** fts3PendingTermsAdd() are to add term/position-list pairs for the |
+** contents of the document with docid iDocid. |
+*/ |
+static int fts3PendingTermsDocid( |
+ Fts3Table *p, /* Full-text table handle */ |
+ int bDelete, /* True if this op is a delete */ |
+ int iLangid, /* Language id of row being written */ |
+ sqlite_int64 iDocid /* Docid of row being written */ |
+){ |
+ assert( iLangid>=0 ); |
+ assert( bDelete==1 || bDelete==0 ); |
+ |
+ /* TODO(shess) Explore whether partially flushing the buffer on |
+ ** forced-flush would provide better performance. I suspect that if |
+ ** we ordered the doclists by size and flushed the largest until the |
+ ** buffer was half empty, that would let the less frequent terms |
+ ** generate longer doclists. |
+ */ |
+ if( iDocid<p->iPrevDocid |
+ || (iDocid==p->iPrevDocid && p->bPrevDelete==0) |
+ || p->iPrevLangid!=iLangid |
+ || p->nPendingData>p->nMaxPendingData |
+ ){ |
+ int rc = sqlite3Fts3PendingTermsFlush(p); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ p->iPrevDocid = iDocid; |
+ p->iPrevLangid = iLangid; |
+ p->bPrevDelete = bDelete; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Discard the contents of the pending-terms hash tables. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ |
+ int i; |
+ for(i=0; i<p->nIndex; i++){ |
+ Fts3HashElem *pElem; |
+ Fts3Hash *pHash = &p->aIndex[i].hPending; |
+ for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ |
+ PendingList *pList = (PendingList *)fts3HashData(pElem); |
+ fts3PendingListDelete(pList); |
+ } |
+ fts3HashClear(pHash); |
+ } |
+ p->nPendingData = 0; |
+} |
+ |
+/* |
+** This function is called by the xUpdate() method as part of an INSERT |
+** operation. It adds entries for each term in the new record to the |
+** pendingTerms hash table. |
+** |
+** Argument apVal is the same as the similarly named argument passed to |
+** fts3InsertData(). Parameter iDocid is the docid of the new row. |
+*/ |
+static int fts3InsertTerms( |
+ Fts3Table *p, |
+ int iLangid, |
+ sqlite3_value **apVal, |
+ u32 *aSz |
+){ |
+ int i; /* Iterator variable */ |
+ for(i=2; i<p->nColumn+2; i++){ |
+ int iCol = i-2; |
+ if( p->abNotindexed[iCol]==0 ){ |
+ const char *zText = (const char *)sqlite3_value_text(apVal[i]); |
+ int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); |
+ } |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function is called by the xUpdate() method for an INSERT operation. |
+** The apVal parameter is passed a copy of the apVal argument passed by |
+** SQLite to the xUpdate() method. i.e: |
+** |
+** apVal[0] Not used for INSERT. |
+** apVal[1] rowid |
+** apVal[2] Left-most user-defined column |
+** ... |
+** apVal[p->nColumn+1] Right-most user-defined column |
+** apVal[p->nColumn+2] Hidden column with same name as table |
+** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) |
+** apVal[p->nColumn+4] Hidden languageid column |
+*/ |
+static int fts3InsertData( |
+ Fts3Table *p, /* Full-text table */ |
+ sqlite3_value **apVal, /* Array of values to insert */ |
+ sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ |
+){ |
+ int rc; /* Return code */ |
+ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ |
+ |
+ if( p->zContentTbl ){ |
+ sqlite3_value *pRowid = apVal[p->nColumn+3]; |
+ if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ |
+ pRowid = apVal[1]; |
+ } |
+ if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ |
+ return SQLITE_CONSTRAINT; |
+ } |
+ *piDocid = sqlite3_value_int64(pRowid); |
+ return SQLITE_OK; |
+ } |
+ |
+ /* Locate the statement handle used to insert data into the %_content |
+ ** table. The SQL for this statement is: |
+ ** |
+ ** INSERT INTO %_content VALUES(?, ?, ?, ...) |
+ ** |
+ ** The statement features N '?' variables, where N is the number of user |
+ ** defined columns in the FTS3 table, plus one for the docid field. |
+ */ |
+ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); |
+ if( rc==SQLITE_OK && p->zLanguageid ){ |
+ rc = sqlite3_bind_int( |
+ pContentInsert, p->nColumn+2, |
+ sqlite3_value_int(apVal[p->nColumn+4]) |
+ ); |
+ } |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ /* There is a quirk here. The users INSERT statement may have specified |
+ ** a value for the "rowid" field, for the "docid" field, or for both. |
+ ** Which is a problem, since "rowid" and "docid" are aliases for the |
+ ** same value. For example: |
+ ** |
+ ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); |
+ ** |
+ ** In FTS3, this is an error. It is an error to specify non-NULL values |
+ ** for both docid and some other rowid alias. |
+ */ |
+ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ |
+ if( SQLITE_NULL==sqlite3_value_type(apVal[0]) |
+ && SQLITE_NULL!=sqlite3_value_type(apVal[1]) |
+ ){ |
+ /* A rowid/docid conflict. */ |
+ return SQLITE_ERROR; |
+ } |
+ rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ |
+ /* Execute the statement to insert the record. Set *piDocid to the |
+ ** new docid value. |
+ */ |
+ sqlite3_step(pContentInsert); |
+ rc = sqlite3_reset(pContentInsert); |
+ |
+ *piDocid = sqlite3_last_insert_rowid(p->db); |
+ return rc; |
+} |
+ |
+ |
+ |
+/* |
+** Remove all data from the FTS3 table. Clear the hash table containing |
+** pending terms. |
+*/ |
+static int fts3DeleteAll(Fts3Table *p, int bContent){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ |
+ /* Discard the contents of the pending-terms hash table. */ |
+ sqlite3Fts3PendingTermsClear(p); |
+ |
+ /* Delete everything from the shadow tables. Except, leave %_content as |
+ ** is if bContent is false. */ |
+ assert( p->zContentTbl==0 || bContent==0 ); |
+ if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); |
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); |
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); |
+ if( p->bHasDocsize ){ |
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); |
+ } |
+ if( p->bHasStat ){ |
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** |
+*/ |
+static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ |
+ int iLangid = 0; |
+ if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); |
+ return iLangid; |
+} |
+ |
+/* |
+** The first element in the apVal[] array is assumed to contain the docid |
+** (an integer) of a row about to be deleted. Remove all terms from the |
+** full-text index. |
+*/ |
+static void fts3DeleteTerms( |
+ int *pRC, /* Result code */ |
+ Fts3Table *p, /* The FTS table to delete from */ |
+ sqlite3_value *pRowid, /* The docid to be deleted */ |
+ u32 *aSz, /* Sizes of deleted document written here */ |
+ int *pbFound /* OUT: Set to true if row really does exist */ |
+){ |
+ int rc; |
+ sqlite3_stmt *pSelect; |
+ |
+ assert( *pbFound==0 ); |
+ if( *pRC ) return; |
+ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); |
+ if( rc==SQLITE_OK ){ |
+ if( SQLITE_ROW==sqlite3_step(pSelect) ){ |
+ int i; |
+ int iLangid = langidFromSelect(p, pSelect); |
+ i64 iDocid = sqlite3_column_int64(pSelect, 0); |
+ rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); |
+ for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ |
+ int iCol = i-1; |
+ if( p->abNotindexed[iCol]==0 ){ |
+ const char *zText = (const char *)sqlite3_column_text(pSelect, i); |
+ rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); |
+ aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); |
+ } |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_reset(pSelect); |
+ *pRC = rc; |
+ return; |
+ } |
+ *pbFound = 1; |
+ } |
+ rc = sqlite3_reset(pSelect); |
+ }else{ |
+ sqlite3_reset(pSelect); |
+ } |
+ *pRC = rc; |
+} |
+ |
+/* |
+** Forward declaration to account for the circular dependency between |
+** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). |
+*/ |
+static int fts3SegmentMerge(Fts3Table *, int, int, int); |
+ |
+/* |
+** This function allocates a new level iLevel index in the segdir table. |
+** Usually, indexes are allocated within a level sequentially starting |
+** with 0, so the allocated index is one greater than the value returned |
+** by: |
+** |
+** SELECT max(idx) FROM %_segdir WHERE level = :iLevel |
+** |
+** However, if there are already FTS3_MERGE_COUNT indexes at the requested |
+** level, they are merged into a single level (iLevel+1) segment and the |
+** allocated index is 0. |
+** |
+** If successful, *piIdx is set to the allocated index slot and SQLITE_OK |
+** returned. Otherwise, an SQLite error code is returned. |
+*/ |
+static int fts3AllocateSegdirIdx( |
+ Fts3Table *p, |
+ int iLangid, /* Language id */ |
+ int iIndex, /* Index for p->aIndex */ |
+ int iLevel, |
+ int *piIdx |
+){ |
+ int rc; /* Return Code */ |
+ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ |
+ int iNext = 0; /* Result of query pNextIdx */ |
+ |
+ assert( iLangid>=0 ); |
+ assert( p->nIndex>=1 ); |
+ |
+ /* Set variable iNext to the next available segdir index at level iLevel. */ |
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64( |
+ pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) |
+ ); |
+ if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ |
+ iNext = sqlite3_column_int(pNextIdx, 0); |
+ } |
+ rc = sqlite3_reset(pNextIdx); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already |
+ ** full, merge all segments in level iLevel into a single iLevel+1 |
+ ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, |
+ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. |
+ */ |
+ if( iNext>=FTS3_MERGE_COUNT ){ |
+ fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); |
+ rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); |
+ *piIdx = 0; |
+ }else{ |
+ *piIdx = iNext; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** The %_segments table is declared as follows: |
+** |
+** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) |
+** |
+** This function reads data from a single row of the %_segments table. The |
+** specific row is identified by the iBlockid parameter. If paBlob is not |
+** NULL, then a buffer is allocated using sqlite3_malloc() and populated |
+** with the contents of the blob stored in the "block" column of the |
+** identified table row is. Whether or not paBlob is NULL, *pnBlob is set |
+** to the size of the blob in bytes before returning. |
+** |
+** If an error occurs, or the table does not contain the specified row, |
+** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If |
+** paBlob is non-NULL, then it is the responsibility of the caller to |
+** eventually free the returned buffer. |
+** |
+** This function may leave an open sqlite3_blob* handle in the |
+** Fts3Table.pSegments variable. This handle is reused by subsequent calls |
+** to this function. The handle may be closed by calling the |
+** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy |
+** performance improvement, but the blob handle should always be closed |
+** before control is returned to the user (to prevent a lock being held |
+** on the database file for longer than necessary). Thus, any virtual table |
+** method (xFilter etc.) that may directly or indirectly call this function |
+** must call sqlite3Fts3SegmentsClose() before returning. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ |
+ char **paBlob, /* OUT: Blob data in malloc'd buffer */ |
+ int *pnBlob, /* OUT: Size of blob data */ |
+ int *pnLoad /* OUT: Bytes actually loaded */ |
+){ |
+ int rc; /* Return code */ |
+ |
+ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ |
+ assert( pnBlob ); |
+ |
+ if( p->pSegments ){ |
+ rc = sqlite3_blob_reopen(p->pSegments, iBlockid); |
+ }else{ |
+ if( 0==p->zSegmentsTbl ){ |
+ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); |
+ if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; |
+ } |
+ rc = sqlite3_blob_open( |
+ p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments |
+ ); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ int nByte = sqlite3_blob_bytes(p->pSegments); |
+ *pnBlob = nByte; |
+ if( paBlob ){ |
+ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); |
+ if( !aByte ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ |
+ nByte = FTS3_NODE_CHUNKSIZE; |
+ *pnLoad = nByte; |
+ } |
+ rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); |
+ memset(&aByte[nByte], 0, FTS3_NODE_PADDING); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_free(aByte); |
+ aByte = 0; |
+ } |
+ } |
+ *paBlob = aByte; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Close the blob handle at p->pSegments, if it is open. See comments above |
+** the sqlite3Fts3ReadBlock() function for details. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ |
+ sqlite3_blob_close(p->pSegments); |
+ p->pSegments = 0; |
+} |
+ |
+static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ |
+ int nRead; /* Number of bytes to read */ |
+ int rc; /* Return code */ |
+ |
+ nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); |
+ rc = sqlite3_blob_read( |
+ pReader->pBlob, |
+ &pReader->aNode[pReader->nPopulate], |
+ nRead, |
+ pReader->nPopulate |
+ ); |
+ |
+ if( rc==SQLITE_OK ){ |
+ pReader->nPopulate += nRead; |
+ memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); |
+ if( pReader->nPopulate==pReader->nNode ){ |
+ sqlite3_blob_close(pReader->pBlob); |
+ pReader->pBlob = 0; |
+ pReader->nPopulate = 0; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ |
+ int rc = SQLITE_OK; |
+ assert( !pReader->pBlob |
+ || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) |
+ ); |
+ while( pReader->pBlob && rc==SQLITE_OK |
+ && (pFrom - pReader->aNode + nByte)>pReader->nPopulate |
+ ){ |
+ rc = fts3SegReaderIncrRead(pReader); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Set an Fts3SegReader cursor to point at EOF. |
+*/ |
+static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ |
+ if( !fts3SegReaderIsRootOnly(pSeg) ){ |
+ sqlite3_free(pSeg->aNode); |
+ sqlite3_blob_close(pSeg->pBlob); |
+ pSeg->pBlob = 0; |
+ } |
+ pSeg->aNode = 0; |
+} |
+ |
+/* |
+** Move the iterator passed as the first argument to the next term in the |
+** segment. If successful, SQLITE_OK is returned. If there is no next term, |
+** SQLITE_DONE. Otherwise, an SQLite error code. |
+*/ |
+static int fts3SegReaderNext( |
+ Fts3Table *p, |
+ Fts3SegReader *pReader, |
+ int bIncr |
+){ |
+ int rc; /* Return code of various sub-routines */ |
+ char *pNext; /* Cursor variable */ |
+ int nPrefix; /* Number of bytes in term prefix */ |
+ int nSuffix; /* Number of bytes in term suffix */ |
+ |
+ if( !pReader->aDoclist ){ |
+ pNext = pReader->aNode; |
+ }else{ |
+ pNext = &pReader->aDoclist[pReader->nDoclist]; |
+ } |
+ |
+ if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ |
+ |
+ if( fts3SegReaderIsPending(pReader) ){ |
+ Fts3HashElem *pElem = *(pReader->ppNextElem); |
+ sqlite3_free(pReader->aNode); |
+ pReader->aNode = 0; |
+ if( pElem ){ |
+ char *aCopy; |
+ PendingList *pList = (PendingList *)fts3HashData(pElem); |
+ int nCopy = pList->nData+1; |
+ pReader->zTerm = (char *)fts3HashKey(pElem); |
+ pReader->nTerm = fts3HashKeysize(pElem); |
+ aCopy = (char*)sqlite3_malloc(nCopy); |
+ if( !aCopy ) return SQLITE_NOMEM; |
+ memcpy(aCopy, pList->aData, nCopy); |
+ pReader->nNode = pReader->nDoclist = nCopy; |
+ pReader->aNode = pReader->aDoclist = aCopy; |
+ pReader->ppNextElem++; |
+ assert( pReader->aNode ); |
+ } |
+ return SQLITE_OK; |
+ } |
+ |
+ fts3SegReaderSetEof(pReader); |
+ |
+ /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf |
+ ** blocks have already been traversed. */ |
+ assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); |
+ if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ |
+ return SQLITE_OK; |
+ } |
+ |
+ rc = sqlite3Fts3ReadBlock( |
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, |
+ (bIncr ? &pReader->nPopulate : 0) |
+ ); |
+ if( rc!=SQLITE_OK ) return rc; |
+ assert( pReader->pBlob==0 ); |
+ if( bIncr && pReader->nPopulate<pReader->nNode ){ |
+ pReader->pBlob = p->pSegments; |
+ p->pSegments = 0; |
+ } |
+ pNext = pReader->aNode; |
+ } |
+ |
+ assert( !fts3SegReaderIsPending(pReader) ); |
+ |
+ rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ /* Because of the FTS3_NODE_PADDING bytes of padding, the following is |
+ ** safe (no risk of overread) even if the node data is corrupted. */ |
+ pNext += fts3GetVarint32(pNext, &nPrefix); |
+ pNext += fts3GetVarint32(pNext, &nSuffix); |
+ if( nPrefix<0 || nSuffix<=0 |
+ || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] |
+ ){ |
+ return FTS_CORRUPT_VTAB; |
+ } |
+ |
+ if( nPrefix+nSuffix>pReader->nTermAlloc ){ |
+ int nNew = (nPrefix+nSuffix)*2; |
+ char *zNew = sqlite3_realloc(pReader->zTerm, nNew); |
+ if( !zNew ){ |
+ return SQLITE_NOMEM; |
+ } |
+ pReader->zTerm = zNew; |
+ pReader->nTermAlloc = nNew; |
+ } |
+ |
+ rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); |
+ pReader->nTerm = nPrefix+nSuffix; |
+ pNext += nSuffix; |
+ pNext += fts3GetVarint32(pNext, &pReader->nDoclist); |
+ pReader->aDoclist = pNext; |
+ pReader->pOffsetList = 0; |
+ |
+ /* Check that the doclist does not appear to extend past the end of the |
+ ** b-tree node. And that the final byte of the doclist is 0x00. If either |
+ ** of these statements is untrue, then the data structure is corrupt. |
+ */ |
+ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] |
+ || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) |
+ ){ |
+ return FTS_CORRUPT_VTAB; |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Set the SegReader to point to the first docid in the doclist associated |
+** with the current term. |
+*/ |
+static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ |
+ int rc = SQLITE_OK; |
+ assert( pReader->aDoclist ); |
+ assert( !pReader->pOffsetList ); |
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ |
+ u8 bEof = 0; |
+ pReader->iDocid = 0; |
+ pReader->nOffsetList = 0; |
+ sqlite3Fts3DoclistPrev(0, |
+ pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, |
+ &pReader->iDocid, &pReader->nOffsetList, &bEof |
+ ); |
+ }else{ |
+ rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); |
+ if( rc==SQLITE_OK ){ |
+ int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); |
+ pReader->pOffsetList = &pReader->aDoclist[n]; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Advance the SegReader to point to the next docid in the doclist |
+** associated with the current term. |
+** |
+** If arguments ppOffsetList and pnOffsetList are not NULL, then |
+** *ppOffsetList is set to point to the first column-offset list |
+** in the doclist entry (i.e. immediately past the docid varint). |
+** *pnOffsetList is set to the length of the set of column-offset |
+** lists, not including the nul-terminator byte. For example: |
+*/ |
+static int fts3SegReaderNextDocid( |
+ Fts3Table *pTab, |
+ Fts3SegReader *pReader, /* Reader to advance to next docid */ |
+ char **ppOffsetList, /* OUT: Pointer to current position-list */ |
+ int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ |
+){ |
+ int rc = SQLITE_OK; |
+ char *p = pReader->pOffsetList; |
+ char c = 0; |
+ |
+ assert( p ); |
+ |
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ |
+ /* A pending-terms seg-reader for an FTS4 table that uses order=desc. |
+ ** Pending-terms doclists are always built up in ascending order, so |
+ ** we have to iterate through them backwards here. */ |
+ u8 bEof = 0; |
+ if( ppOffsetList ){ |
+ *ppOffsetList = pReader->pOffsetList; |
+ *pnOffsetList = pReader->nOffsetList - 1; |
+ } |
+ sqlite3Fts3DoclistPrev(0, |
+ pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, |
+ &pReader->nOffsetList, &bEof |
+ ); |
+ if( bEof ){ |
+ pReader->pOffsetList = 0; |
+ }else{ |
+ pReader->pOffsetList = p; |
+ } |
+ }else{ |
+ char *pEnd = &pReader->aDoclist[pReader->nDoclist]; |
+ |
+ /* Pointer p currently points at the first byte of an offset list. The |
+ ** following block advances it to point one byte past the end of |
+ ** the same offset list. */ |
+ while( 1 ){ |
+ |
+ /* The following line of code (and the "p++" below the while() loop) is |
+ ** normally all that is required to move pointer p to the desired |
+ ** position. The exception is if this node is being loaded from disk |
+ ** incrementally and pointer "p" now points to the first byte past |
+ ** the populated part of pReader->aNode[]. |
+ */ |
+ while( *p | c ) c = *p++ & 0x80; |
+ assert( *p==0 ); |
+ |
+ if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; |
+ rc = fts3SegReaderIncrRead(pReader); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ p++; |
+ |
+ /* If required, populate the output variables with a pointer to and the |
+ ** size of the previous offset-list. |
+ */ |
+ if( ppOffsetList ){ |
+ *ppOffsetList = pReader->pOffsetList; |
+ *pnOffsetList = (int)(p - pReader->pOffsetList - 1); |
+ } |
+ |
+ /* List may have been edited in place by fts3EvalNearTrim() */ |
+ while( p<pEnd && *p==0 ) p++; |
+ |
+ /* If there are no more entries in the doclist, set pOffsetList to |
+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and |
+ ** Fts3SegReader.pOffsetList to point to the next offset list before |
+ ** returning. |
+ */ |
+ if( p>=pEnd ){ |
+ pReader->pOffsetList = 0; |
+ }else{ |
+ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_int64 iDelta; |
+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); |
+ if( pTab->bDescIdx ){ |
+ pReader->iDocid -= iDelta; |
+ }else{ |
+ pReader->iDocid += iDelta; |
+ } |
+ } |
+ } |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+ |
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( |
+ Fts3Cursor *pCsr, |
+ Fts3MultiSegReader *pMsr, |
+ int *pnOvfl |
+){ |
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; |
+ int nOvfl = 0; |
+ int ii; |
+ int rc = SQLITE_OK; |
+ int pgsz = p->nPgsz; |
+ |
+ assert( p->bFts4 ); |
+ assert( pgsz>0 ); |
+ |
+ for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){ |
+ Fts3SegReader *pReader = pMsr->apSegment[ii]; |
+ if( !fts3SegReaderIsPending(pReader) |
+ && !fts3SegReaderIsRootOnly(pReader) |
+ ){ |
+ sqlite3_int64 jj; |
+ for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ |
+ int nBlob; |
+ rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); |
+ if( rc!=SQLITE_OK ) break; |
+ if( (nBlob+35)>pgsz ){ |
+ nOvfl += (nBlob + 34)/pgsz; |
+ } |
+ } |
+ } |
+ } |
+ *pnOvfl = nOvfl; |
+ return rc; |
+} |
+ |
+/* |
+** Free all allocations associated with the iterator passed as the |
+** second argument. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ |
+ if( pReader ){ |
+ if( !fts3SegReaderIsPending(pReader) ){ |
+ sqlite3_free(pReader->zTerm); |
+ } |
+ if( !fts3SegReaderIsRootOnly(pReader) ){ |
+ sqlite3_free(pReader->aNode); |
+ } |
+ sqlite3_blob_close(pReader->pBlob); |
+ } |
+ sqlite3_free(pReader); |
+} |
+ |
+/* |
+** Allocate a new SegReader object. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( |
+ int iAge, /* Segment "age". */ |
+ int bLookup, /* True for a lookup only */ |
+ sqlite3_int64 iStartLeaf, /* First leaf to traverse */ |
+ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ |
+ sqlite3_int64 iEndBlock, /* Final block of segment */ |
+ const char *zRoot, /* Buffer containing root node */ |
+ int nRoot, /* Size of buffer containing root node */ |
+ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ |
+){ |
+ Fts3SegReader *pReader; /* Newly allocated SegReader object */ |
+ int nExtra = 0; /* Bytes to allocate segment root node */ |
+ |
+ assert( iStartLeaf<=iEndLeaf ); |
+ if( iStartLeaf==0 ){ |
+ nExtra = nRoot + FTS3_NODE_PADDING; |
+ } |
+ |
+ pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); |
+ if( !pReader ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pReader, 0, sizeof(Fts3SegReader)); |
+ pReader->iIdx = iAge; |
+ pReader->bLookup = bLookup!=0; |
+ pReader->iStartBlock = iStartLeaf; |
+ pReader->iLeafEndBlock = iEndLeaf; |
+ pReader->iEndBlock = iEndBlock; |
+ |
+ if( nExtra ){ |
+ /* The entire segment is stored in the root node. */ |
+ pReader->aNode = (char *)&pReader[1]; |
+ pReader->rootOnly = 1; |
+ pReader->nNode = nRoot; |
+ memcpy(pReader->aNode, zRoot, nRoot); |
+ memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); |
+ }else{ |
+ pReader->iCurrentBlock = iStartLeaf-1; |
+ } |
+ *ppReader = pReader; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This is a comparison function used as a qsort() callback when sorting |
+** an array of pending terms by term. This occurs as part of flushing |
+** the contents of the pending-terms hash table to the database. |
+*/ |
+static int SQLITE_CDECL fts3CompareElemByTerm( |
+ const void *lhs, |
+ const void *rhs |
+){ |
+ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); |
+ char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); |
+ int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); |
+ int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); |
+ |
+ int n = (n1<n2 ? n1 : n2); |
+ int c = memcmp(z1, z2, n); |
+ if( c==0 ){ |
+ c = n1 - n2; |
+ } |
+ return c; |
+} |
+ |
+/* |
+** This function is used to allocate an Fts3SegReader that iterates through |
+** a subset of the terms stored in the Fts3Table.pendingTerms array. |
+** |
+** If the isPrefixIter parameter is zero, then the returned SegReader iterates |
+** through each term in the pending-terms table. Or, if isPrefixIter is |
+** non-zero, it iterates through each term and its prefixes. For example, if |
+** the pending terms hash table contains the terms "sqlite", "mysql" and |
+** "firebird", then the iterator visits the following 'terms' (in the order |
+** shown): |
+** |
+** f fi fir fire fireb firebi firebir firebird |
+** m my mys mysq mysql |
+** s sq sql sqli sqlit sqlite |
+** |
+** Whereas if isPrefixIter is zero, the terms visited are: |
+** |
+** firebird mysql sqlite |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( |
+ Fts3Table *p, /* Virtual table handle */ |
+ int iIndex, /* Index for p->aIndex */ |
+ const char *zTerm, /* Term to search for */ |
+ int nTerm, /* Size of buffer zTerm */ |
+ int bPrefix, /* True for a prefix iterator */ |
+ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ |
+){ |
+ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ |
+ Fts3HashElem *pE; /* Iterator variable */ |
+ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ |
+ int nElem = 0; /* Size of array at aElem */ |
+ int rc = SQLITE_OK; /* Return Code */ |
+ Fts3Hash *pHash; |
+ |
+ pHash = &p->aIndex[iIndex].hPending; |
+ if( bPrefix ){ |
+ int nAlloc = 0; /* Size of allocated array at aElem */ |
+ |
+ for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ |
+ char *zKey = (char *)fts3HashKey(pE); |
+ int nKey = fts3HashKeysize(pE); |
+ if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ |
+ if( nElem==nAlloc ){ |
+ Fts3HashElem **aElem2; |
+ nAlloc += 16; |
+ aElem2 = (Fts3HashElem **)sqlite3_realloc( |
+ aElem, nAlloc*sizeof(Fts3HashElem *) |
+ ); |
+ if( !aElem2 ){ |
+ rc = SQLITE_NOMEM; |
+ nElem = 0; |
+ break; |
+ } |
+ aElem = aElem2; |
+ } |
+ |
+ aElem[nElem++] = pE; |
+ } |
+ } |
+ |
+ /* If more than one term matches the prefix, sort the Fts3HashElem |
+ ** objects in term order using qsort(). This uses the same comparison |
+ ** callback as is used when flushing terms to disk. |
+ */ |
+ if( nElem>1 ){ |
+ qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); |
+ } |
+ |
+ }else{ |
+ /* The query is a simple term lookup that matches at most one term in |
+ ** the index. All that is required is a straight hash-lookup. |
+ ** |
+ ** Because the stack address of pE may be accessed via the aElem pointer |
+ ** below, the "Fts3HashElem *pE" must be declared so that it is valid |
+ ** within this entire function, not just this "else{...}" block. |
+ */ |
+ pE = fts3HashFindElem(pHash, zTerm, nTerm); |
+ if( pE ){ |
+ aElem = &pE; |
+ nElem = 1; |
+ } |
+ } |
+ |
+ if( nElem>0 ){ |
+ int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); |
+ pReader = (Fts3SegReader *)sqlite3_malloc(nByte); |
+ if( !pReader ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(pReader, 0, nByte); |
+ pReader->iIdx = 0x7FFFFFFF; |
+ pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; |
+ memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); |
+ } |
+ } |
+ |
+ if( bPrefix ){ |
+ sqlite3_free(aElem); |
+ } |
+ *ppReader = pReader; |
+ return rc; |
+} |
+ |
+/* |
+** Compare the entries pointed to by two Fts3SegReader structures. |
+** Comparison is as follows: |
+** |
+** 1) EOF is greater than not EOF. |
+** |
+** 2) The current terms (if any) are compared using memcmp(). If one |
+** term is a prefix of another, the longer term is considered the |
+** larger. |
+** |
+** 3) By segment age. An older segment is considered larger. |
+*/ |
+static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ |
+ int rc; |
+ if( pLhs->aNode && pRhs->aNode ){ |
+ int rc2 = pLhs->nTerm - pRhs->nTerm; |
+ if( rc2<0 ){ |
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); |
+ }else{ |
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); |
+ } |
+ if( rc==0 ){ |
+ rc = rc2; |
+ } |
+ }else{ |
+ rc = (pLhs->aNode==0) - (pRhs->aNode==0); |
+ } |
+ if( rc==0 ){ |
+ rc = pRhs->iIdx - pLhs->iIdx; |
+ } |
+ assert( rc!=0 ); |
+ return rc; |
+} |
+ |
+/* |
+** A different comparison function for SegReader structures. In this |
+** version, it is assumed that each SegReader points to an entry in |
+** a doclist for identical terms. Comparison is made as follows: |
+** |
+** 1) EOF (end of doclist in this case) is greater than not EOF. |
+** |
+** 2) By current docid. |
+** |
+** 3) By segment age. An older segment is considered larger. |
+*/ |
+static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ |
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); |
+ if( rc==0 ){ |
+ if( pLhs->iDocid==pRhs->iDocid ){ |
+ rc = pRhs->iIdx - pLhs->iIdx; |
+ }else{ |
+ rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; |
+ } |
+ } |
+ assert( pLhs->aNode && pRhs->aNode ); |
+ return rc; |
+} |
+static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ |
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); |
+ if( rc==0 ){ |
+ if( pLhs->iDocid==pRhs->iDocid ){ |
+ rc = pRhs->iIdx - pLhs->iIdx; |
+ }else{ |
+ rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; |
+ } |
+ } |
+ assert( pLhs->aNode && pRhs->aNode ); |
+ return rc; |
+} |
+ |
+/* |
+** Compare the term that the Fts3SegReader object passed as the first argument |
+** points to with the term specified by arguments zTerm and nTerm. |
+** |
+** If the pSeg iterator is already at EOF, return 0. Otherwise, return |
+** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are |
+** equal, or +ve if the pSeg term is greater than zTerm/nTerm. |
+*/ |
+static int fts3SegReaderTermCmp( |
+ Fts3SegReader *pSeg, /* Segment reader object */ |
+ const char *zTerm, /* Term to compare to */ |
+ int nTerm /* Size of term zTerm in bytes */ |
+){ |
+ int res = 0; |
+ if( pSeg->aNode ){ |
+ if( pSeg->nTerm>nTerm ){ |
+ res = memcmp(pSeg->zTerm, zTerm, nTerm); |
+ }else{ |
+ res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); |
+ } |
+ if( res==0 ){ |
+ res = pSeg->nTerm-nTerm; |
+ } |
+ } |
+ return res; |
+} |
+ |
+/* |
+** Argument apSegment is an array of nSegment elements. It is known that |
+** the final (nSegment-nSuspect) members are already in sorted order |
+** (according to the comparison function provided). This function shuffles |
+** the array around until all entries are in sorted order. |
+*/ |
+static void fts3SegReaderSort( |
+ Fts3SegReader **apSegment, /* Array to sort entries of */ |
+ int nSegment, /* Size of apSegment array */ |
+ int nSuspect, /* Unsorted entry count */ |
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ |
+){ |
+ int i; /* Iterator variable */ |
+ |
+ assert( nSuspect<=nSegment ); |
+ |
+ if( nSuspect==nSegment ) nSuspect--; |
+ for(i=nSuspect-1; i>=0; i--){ |
+ int j; |
+ for(j=i; j<(nSegment-1); j++){ |
+ Fts3SegReader *pTmp; |
+ if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; |
+ pTmp = apSegment[j+1]; |
+ apSegment[j+1] = apSegment[j]; |
+ apSegment[j] = pTmp; |
+ } |
+ } |
+ |
+#ifndef NDEBUG |
+ /* Check that the list really is sorted now. */ |
+ for(i=0; i<(nSuspect-1); i++){ |
+ assert( xCmp(apSegment[i], apSegment[i+1])<0 ); |
+ } |
+#endif |
+} |
+ |
+/* |
+** Insert a record into the %_segments table. |
+*/ |
+static int fts3WriteSegment( |
+ Fts3Table *p, /* Virtual table handle */ |
+ sqlite3_int64 iBlock, /* Block id for new block */ |
+ char *z, /* Pointer to buffer containing block data */ |
+ int n /* Size of buffer z in bytes */ |
+){ |
+ sqlite3_stmt *pStmt; |
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pStmt, 1, iBlock); |
+ sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); |
+ sqlite3_step(pStmt); |
+ rc = sqlite3_reset(pStmt); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Find the largest relative level number in the table. If successful, set |
+** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, |
+** set *pnMax to zero and return an SQLite error code. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ |
+ int rc; |
+ int mxLevel = 0; |
+ sqlite3_stmt *pStmt = 0; |
+ |
+ rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ mxLevel = sqlite3_column_int(pStmt, 0); |
+ } |
+ rc = sqlite3_reset(pStmt); |
+ } |
+ *pnMax = mxLevel; |
+ return rc; |
+} |
+ |
+/* |
+** Insert a record into the %_segdir table. |
+*/ |
+static int fts3WriteSegdir( |
+ Fts3Table *p, /* Virtual table handle */ |
+ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ |
+ int iIdx, /* Value for "idx" field */ |
+ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ |
+ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ |
+ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ |
+ sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ |
+ char *zRoot, /* Blob value for "root" field */ |
+ int nRoot /* Number of bytes in buffer zRoot */ |
+){ |
+ sqlite3_stmt *pStmt; |
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pStmt, 1, iLevel); |
+ sqlite3_bind_int(pStmt, 2, iIdx); |
+ sqlite3_bind_int64(pStmt, 3, iStartBlock); |
+ sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); |
+ if( nLeafData==0 ){ |
+ sqlite3_bind_int64(pStmt, 5, iEndBlock); |
+ }else{ |
+ char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); |
+ if( !zEnd ) return SQLITE_NOMEM; |
+ sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); |
+ } |
+ sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); |
+ sqlite3_step(pStmt); |
+ rc = sqlite3_reset(pStmt); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Return the size of the common prefix (if any) shared by zPrev and |
+** zNext, in bytes. For example, |
+** |
+** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 |
+** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 |
+** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 |
+*/ |
+static int fts3PrefixCompress( |
+ const char *zPrev, /* Buffer containing previous term */ |
+ int nPrev, /* Size of buffer zPrev in bytes */ |
+ const char *zNext, /* Buffer containing next term */ |
+ int nNext /* Size of buffer zNext in bytes */ |
+){ |
+ int n; |
+ UNUSED_PARAMETER(nNext); |
+ for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++); |
+ return n; |
+} |
+ |
+/* |
+** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger |
+** (according to memcmp) than the previous term. |
+*/ |
+static int fts3NodeAddTerm( |
+ Fts3Table *p, /* Virtual table handle */ |
+ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ |
+ int isCopyTerm, /* True if zTerm/nTerm is transient */ |
+ const char *zTerm, /* Pointer to buffer containing term */ |
+ int nTerm /* Size of term in bytes */ |
+){ |
+ SegmentNode *pTree = *ppTree; |
+ int rc; |
+ SegmentNode *pNew; |
+ |
+ /* First try to append the term to the current node. Return early if |
+ ** this is possible. |
+ */ |
+ if( pTree ){ |
+ int nData = pTree->nData; /* Current size of node in bytes */ |
+ int nReq = nData; /* Required space after adding zTerm */ |
+ int nPrefix; /* Number of bytes of prefix compression */ |
+ int nSuffix; /* Suffix length */ |
+ |
+ nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); |
+ nSuffix = nTerm-nPrefix; |
+ |
+ nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; |
+ if( nReq<=p->nNodeSize || !pTree->zTerm ){ |
+ |
+ if( nReq>p->nNodeSize ){ |
+ /* An unusual case: this is the first term to be added to the node |
+ ** and the static node buffer (p->nNodeSize bytes) is not large |
+ ** enough. Use a separately malloced buffer instead This wastes |
+ ** p->nNodeSize bytes, but since this scenario only comes about when |
+ ** the database contain two terms that share a prefix of almost 2KB, |
+ ** this is not expected to be a serious problem. |
+ */ |
+ assert( pTree->aData==(char *)&pTree[1] ); |
+ pTree->aData = (char *)sqlite3_malloc(nReq); |
+ if( !pTree->aData ){ |
+ return SQLITE_NOMEM; |
+ } |
+ } |
+ |
+ if( pTree->zTerm ){ |
+ /* There is no prefix-length field for first term in a node */ |
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); |
+ } |
+ |
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); |
+ memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); |
+ pTree->nData = nData + nSuffix; |
+ pTree->nEntry++; |
+ |
+ if( isCopyTerm ){ |
+ if( pTree->nMalloc<nTerm ){ |
+ char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2); |
+ if( !zNew ){ |
+ return SQLITE_NOMEM; |
+ } |
+ pTree->nMalloc = nTerm*2; |
+ pTree->zMalloc = zNew; |
+ } |
+ pTree->zTerm = pTree->zMalloc; |
+ memcpy(pTree->zTerm, zTerm, nTerm); |
+ pTree->nTerm = nTerm; |
+ }else{ |
+ pTree->zTerm = (char *)zTerm; |
+ pTree->nTerm = nTerm; |
+ } |
+ return SQLITE_OK; |
+ } |
+ } |
+ |
+ /* If control flows to here, it was not possible to append zTerm to the |
+ ** current node. Create a new node (a right-sibling of the current node). |
+ ** If this is the first node in the tree, the term is added to it. |
+ ** |
+ ** Otherwise, the term is not added to the new node, it is left empty for |
+ ** now. Instead, the term is inserted into the parent of pTree. If pTree |
+ ** has no parent, one is created here. |
+ */ |
+ pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); |
+ if( !pNew ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pNew, 0, sizeof(SegmentNode)); |
+ pNew->nData = 1 + FTS3_VARINT_MAX; |
+ pNew->aData = (char *)&pNew[1]; |
+ |
+ if( pTree ){ |
+ SegmentNode *pParent = pTree->pParent; |
+ rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); |
+ if( pTree->pParent==0 ){ |
+ pTree->pParent = pParent; |
+ } |
+ pTree->pRight = pNew; |
+ pNew->pLeftmost = pTree->pLeftmost; |
+ pNew->pParent = pParent; |
+ pNew->zMalloc = pTree->zMalloc; |
+ pNew->nMalloc = pTree->nMalloc; |
+ pTree->zMalloc = 0; |
+ }else{ |
+ pNew->pLeftmost = pNew; |
+ rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); |
+ } |
+ |
+ *ppTree = pNew; |
+ return rc; |
+} |
+ |
+/* |
+** Helper function for fts3NodeWrite(). |
+*/ |
+static int fts3TreeFinishNode( |
+ SegmentNode *pTree, |
+ int iHeight, |
+ sqlite3_int64 iLeftChild |
+){ |
+ int nStart; |
+ assert( iHeight>=1 && iHeight<128 ); |
+ nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); |
+ pTree->aData[nStart] = (char)iHeight; |
+ sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); |
+ return nStart; |
+} |
+ |
+/* |
+** Write the buffer for the segment node pTree and all of its peers to the |
+** database. Then call this function recursively to write the parent of |
+** pTree and its peers to the database. |
+** |
+** Except, if pTree is a root node, do not write it to the database. Instead, |
+** set output variables *paRoot and *pnRoot to contain the root node. |
+** |
+** If successful, SQLITE_OK is returned and output variable *piLast is |
+** set to the largest blockid written to the database (or zero if no |
+** blocks were written to the db). Otherwise, an SQLite error code is |
+** returned. |
+*/ |
+static int fts3NodeWrite( |
+ Fts3Table *p, /* Virtual table handle */ |
+ SegmentNode *pTree, /* SegmentNode handle */ |
+ int iHeight, /* Height of this node in tree */ |
+ sqlite3_int64 iLeaf, /* Block id of first leaf node */ |
+ sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ |
+ sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ |
+ char **paRoot, /* OUT: Data for root node */ |
+ int *pnRoot /* OUT: Size of root node in bytes */ |
+){ |
+ int rc = SQLITE_OK; |
+ |
+ if( !pTree->pParent ){ |
+ /* Root node of the tree. */ |
+ int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); |
+ *piLast = iFree-1; |
+ *pnRoot = pTree->nData - nStart; |
+ *paRoot = &pTree->aData[nStart]; |
+ }else{ |
+ SegmentNode *pIter; |
+ sqlite3_int64 iNextFree = iFree; |
+ sqlite3_int64 iNextLeaf = iLeaf; |
+ for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ |
+ int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); |
+ int nWrite = pIter->nData - nStart; |
+ |
+ rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); |
+ iNextFree++; |
+ iNextLeaf += (pIter->nEntry+1); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ assert( iNextLeaf==iFree ); |
+ rc = fts3NodeWrite( |
+ p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot |
+ ); |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Free all memory allocations associated with the tree pTree. |
+*/ |
+static void fts3NodeFree(SegmentNode *pTree){ |
+ if( pTree ){ |
+ SegmentNode *p = pTree->pLeftmost; |
+ fts3NodeFree(p->pParent); |
+ while( p ){ |
+ SegmentNode *pRight = p->pRight; |
+ if( p->aData!=(char *)&p[1] ){ |
+ sqlite3_free(p->aData); |
+ } |
+ assert( pRight==0 || p->zMalloc==0 ); |
+ sqlite3_free(p->zMalloc); |
+ sqlite3_free(p); |
+ p = pRight; |
+ } |
+ } |
+} |
+ |
+/* |
+** Add a term to the segment being constructed by the SegmentWriter object |
+** *ppWriter. When adding the first term to a segment, *ppWriter should |
+** be passed NULL. This function will allocate a new SegmentWriter object |
+** and return it via the input/output variable *ppWriter in this case. |
+** |
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. |
+*/ |
+static int fts3SegWriterAdd( |
+ Fts3Table *p, /* Virtual table handle */ |
+ SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ |
+ int isCopyTerm, /* True if buffer zTerm must be copied */ |
+ const char *zTerm, /* Pointer to buffer containing term */ |
+ int nTerm, /* Size of term in bytes */ |
+ const char *aDoclist, /* Pointer to buffer containing doclist */ |
+ int nDoclist /* Size of doclist in bytes */ |
+){ |
+ int nPrefix; /* Size of term prefix in bytes */ |
+ int nSuffix; /* Size of term suffix in bytes */ |
+ int nReq; /* Number of bytes required on leaf page */ |
+ int nData; |
+ SegmentWriter *pWriter = *ppWriter; |
+ |
+ if( !pWriter ){ |
+ int rc; |
+ sqlite3_stmt *pStmt; |
+ |
+ /* Allocate the SegmentWriter structure */ |
+ pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); |
+ if( !pWriter ) return SQLITE_NOMEM; |
+ memset(pWriter, 0, sizeof(SegmentWriter)); |
+ *ppWriter = pWriter; |
+ |
+ /* Allocate a buffer in which to accumulate data */ |
+ pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); |
+ if( !pWriter->aData ) return SQLITE_NOMEM; |
+ pWriter->nSize = p->nNodeSize; |
+ |
+ /* Find the next free blockid in the %_segments table */ |
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); |
+ if( rc!=SQLITE_OK ) return rc; |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ pWriter->iFree = sqlite3_column_int64(pStmt, 0); |
+ pWriter->iFirst = pWriter->iFree; |
+ } |
+ rc = sqlite3_reset(pStmt); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ nData = pWriter->nData; |
+ |
+ nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); |
+ nSuffix = nTerm-nPrefix; |
+ |
+ /* Figure out how many bytes are required by this new entry */ |
+ nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ |
+ sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ |
+ nSuffix + /* Term suffix */ |
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ |
+ nDoclist; /* Doclist data */ |
+ |
+ if( nData>0 && nData+nReq>p->nNodeSize ){ |
+ int rc; |
+ |
+ /* The current leaf node is full. Write it out to the database. */ |
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); |
+ if( rc!=SQLITE_OK ) return rc; |
+ p->nLeafAdd++; |
+ |
+ /* Add the current term to the interior node tree. The term added to |
+ ** the interior tree must: |
+ ** |
+ ** a) be greater than the largest term on the leaf node just written |
+ ** to the database (still available in pWriter->zTerm), and |
+ ** |
+ ** b) be less than or equal to the term about to be added to the new |
+ ** leaf node (zTerm/nTerm). |
+ ** |
+ ** In other words, it must be the prefix of zTerm 1 byte longer than |
+ ** the common prefix (if any) of zTerm and pWriter->zTerm. |
+ */ |
+ assert( nPrefix<nTerm ); |
+ rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ nData = 0; |
+ pWriter->nTerm = 0; |
+ |
+ nPrefix = 0; |
+ nSuffix = nTerm; |
+ nReq = 1 + /* varint containing prefix size */ |
+ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ |
+ nTerm + /* Term suffix */ |
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ |
+ nDoclist; /* Doclist data */ |
+ } |
+ |
+ /* Increase the total number of bytes written to account for the new entry. */ |
+ pWriter->nLeafData += nReq; |
+ |
+ /* If the buffer currently allocated is too small for this entry, realloc |
+ ** the buffer to make it large enough. |
+ */ |
+ if( nReq>pWriter->nSize ){ |
+ char *aNew = sqlite3_realloc(pWriter->aData, nReq); |
+ if( !aNew ) return SQLITE_NOMEM; |
+ pWriter->aData = aNew; |
+ pWriter->nSize = nReq; |
+ } |
+ assert( nData+nReq<=pWriter->nSize ); |
+ |
+ /* Append the prefix-compressed term and doclist to the buffer. */ |
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); |
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); |
+ memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); |
+ nData += nSuffix; |
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); |
+ memcpy(&pWriter->aData[nData], aDoclist, nDoclist); |
+ pWriter->nData = nData + nDoclist; |
+ |
+ /* Save the current term so that it can be used to prefix-compress the next. |
+ ** If the isCopyTerm parameter is true, then the buffer pointed to by |
+ ** zTerm is transient, so take a copy of the term data. Otherwise, just |
+ ** store a copy of the pointer. |
+ */ |
+ if( isCopyTerm ){ |
+ if( nTerm>pWriter->nMalloc ){ |
+ char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); |
+ if( !zNew ){ |
+ return SQLITE_NOMEM; |
+ } |
+ pWriter->nMalloc = nTerm*2; |
+ pWriter->zMalloc = zNew; |
+ pWriter->zTerm = zNew; |
+ } |
+ assert( pWriter->zTerm==pWriter->zMalloc ); |
+ memcpy(pWriter->zTerm, zTerm, nTerm); |
+ }else{ |
+ pWriter->zTerm = (char *)zTerm; |
+ } |
+ pWriter->nTerm = nTerm; |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Flush all data associated with the SegmentWriter object pWriter to the |
+** database. This function must be called after all terms have been added |
+** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is |
+** returned. Otherwise, an SQLite error code. |
+*/ |
+static int fts3SegWriterFlush( |
+ Fts3Table *p, /* Virtual table handle */ |
+ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ |
+ sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ |
+ int iIdx /* Value for 'idx' column of %_segdir */ |
+){ |
+ int rc; /* Return code */ |
+ if( pWriter->pTree ){ |
+ sqlite3_int64 iLast = 0; /* Largest block id written to database */ |
+ sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ |
+ char *zRoot = NULL; /* Pointer to buffer containing root node */ |
+ int nRoot = 0; /* Size of buffer zRoot */ |
+ |
+ iLastLeaf = pWriter->iFree; |
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3NodeWrite(p, pWriter->pTree, 1, |
+ pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3WriteSegdir(p, iLevel, iIdx, |
+ pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); |
+ } |
+ }else{ |
+ /* The entire tree fits on the root node. Write it to the segdir table. */ |
+ rc = fts3WriteSegdir(p, iLevel, iIdx, |
+ 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); |
+ } |
+ p->nLeafAdd++; |
+ return rc; |
+} |
+ |
+/* |
+** Release all memory held by the SegmentWriter object passed as the |
+** first argument. |
+*/ |
+static void fts3SegWriterFree(SegmentWriter *pWriter){ |
+ if( pWriter ){ |
+ sqlite3_free(pWriter->aData); |
+ sqlite3_free(pWriter->zMalloc); |
+ fts3NodeFree(pWriter->pTree); |
+ sqlite3_free(pWriter); |
+ } |
+} |
+ |
+/* |
+** The first value in the apVal[] array is assumed to contain an integer. |
+** This function tests if there exist any documents with docid values that |
+** are different from that integer. i.e. if deleting the document with docid |
+** pRowid would mean the FTS3 table were empty. |
+** |
+** If successful, *pisEmpty is set to true if the table is empty except for |
+** document pRowid, or false otherwise, and SQLITE_OK is returned. If an |
+** error occurs, an SQLite error code is returned. |
+*/ |
+static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ |
+ sqlite3_stmt *pStmt; |
+ int rc; |
+ if( p->zContentTbl ){ |
+ /* If using the content=xxx option, assume the table is never empty */ |
+ *pisEmpty = 0; |
+ rc = SQLITE_OK; |
+ }else{ |
+ rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); |
+ if( rc==SQLITE_OK ){ |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ *pisEmpty = sqlite3_column_int(pStmt, 0); |
+ } |
+ rc = sqlite3_reset(pStmt); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Set *pnMax to the largest segment level in the database for the index |
+** iIndex. |
+** |
+** Segment levels are stored in the 'level' column of the %_segdir table. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code if not. |
+*/ |
+static int fts3SegmentMaxLevel( |
+ Fts3Table *p, |
+ int iLangid, |
+ int iIndex, |
+ sqlite3_int64 *pnMax |
+){ |
+ sqlite3_stmt *pStmt; |
+ int rc; |
+ assert( iIndex>=0 && iIndex<p->nIndex ); |
+ |
+ /* Set pStmt to the compiled version of: |
+ ** |
+ ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? |
+ ** |
+ ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). |
+ */ |
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); |
+ if( rc!=SQLITE_OK ) return rc; |
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); |
+ sqlite3_bind_int64(pStmt, 2, |
+ getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) |
+ ); |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ *pnMax = sqlite3_column_int64(pStmt, 0); |
+ } |
+ return sqlite3_reset(pStmt); |
+} |
+ |
+/* |
+** iAbsLevel is an absolute level that may be assumed to exist within |
+** the database. This function checks if it is the largest level number |
+** within its index. Assuming no error occurs, *pbMax is set to 1 if |
+** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK |
+** is returned. If an error occurs, an error code is returned and the |
+** final value of *pbMax is undefined. |
+*/ |
+static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ |
+ |
+ /* Set pStmt to the compiled version of: |
+ ** |
+ ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? |
+ ** |
+ ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). |
+ */ |
+ sqlite3_stmt *pStmt; |
+ int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); |
+ if( rc!=SQLITE_OK ) return rc; |
+ sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); |
+ sqlite3_bind_int64(pStmt, 2, |
+ ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL |
+ ); |
+ |
+ *pbMax = 0; |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; |
+ } |
+ return sqlite3_reset(pStmt); |
+} |
+ |
+/* |
+** Delete all entries in the %_segments table associated with the segment |
+** opened with seg-reader pSeg. This function does not affect the contents |
+** of the %_segdir table. |
+*/ |
+static int fts3DeleteSegment( |
+ Fts3Table *p, /* FTS table handle */ |
+ Fts3SegReader *pSeg /* Segment to delete */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ if( pSeg->iStartBlock ){ |
+ sqlite3_stmt *pDelete; /* SQL statement to delete rows */ |
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); |
+ sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); |
+ sqlite3_step(pDelete); |
+ rc = sqlite3_reset(pDelete); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This function is used after merging multiple segments into a single large |
+** segment to delete the old, now redundant, segment b-trees. Specifically, |
+** it: |
+** |
+** 1) Deletes all %_segments entries for the segments associated with |
+** each of the SegReader objects in the array passed as the third |
+** argument, and |
+** |
+** 2) deletes all %_segdir entries with level iLevel, or all %_segdir |
+** entries regardless of level if (iLevel<0). |
+** |
+** SQLITE_OK is returned if successful, otherwise an SQLite error code. |
+*/ |
+static int fts3DeleteSegdir( |
+ Fts3Table *p, /* Virtual table handle */ |
+ int iLangid, /* Language id */ |
+ int iIndex, /* Index for p->aIndex */ |
+ int iLevel, /* Level of %_segdir entries to delete */ |
+ Fts3SegReader **apSegment, /* Array of SegReader objects */ |
+ int nReader /* Size of array apSegment */ |
+){ |
+ int rc = SQLITE_OK; /* Return Code */ |
+ int i; /* Iterator variable */ |
+ sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ |
+ |
+ for(i=0; rc==SQLITE_OK && i<nReader; i++){ |
+ rc = fts3DeleteSegment(p, apSegment[i]); |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); |
+ if( iLevel==FTS3_SEGCURSOR_ALL ){ |
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); |
+ sqlite3_bind_int64(pDelete, 2, |
+ getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) |
+ ); |
+ } |
+ }else{ |
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64( |
+ pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) |
+ ); |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_step(pDelete); |
+ rc = sqlite3_reset(pDelete); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** When this function is called, buffer *ppList (size *pnList bytes) contains |
+** a position list that may (or may not) feature multiple columns. This |
+** function adjusts the pointer *ppList and the length *pnList so that they |
+** identify the subset of the position list that corresponds to column iCol. |
+** |
+** If there are no entries in the input position list for column iCol, then |
+** *pnList is set to zero before returning. |
+** |
+** If parameter bZero is non-zero, then any part of the input list following |
+** the end of the output list is zeroed before returning. |
+*/ |
+static void fts3ColumnFilter( |
+ int iCol, /* Column to filter on */ |
+ int bZero, /* Zero out anything following *ppList */ |
+ char **ppList, /* IN/OUT: Pointer to position list */ |
+ int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ |
+){ |
+ char *pList = *ppList; |
+ int nList = *pnList; |
+ char *pEnd = &pList[nList]; |
+ int iCurrent = 0; |
+ char *p = pList; |
+ |
+ assert( iCol>=0 ); |
+ while( 1 ){ |
+ char c = 0; |
+ while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80; |
+ |
+ if( iCol==iCurrent ){ |
+ nList = (int)(p - pList); |
+ break; |
+ } |
+ |
+ nList -= (int)(p - pList); |
+ pList = p; |
+ if( nList==0 ){ |
+ break; |
+ } |
+ p = &pList[1]; |
+ p += fts3GetVarint32(p, &iCurrent); |
+ } |
+ |
+ if( bZero && &pList[nList]!=pEnd ){ |
+ memset(&pList[nList], 0, pEnd - &pList[nList]); |
+ } |
+ *ppList = pList; |
+ *pnList = nList; |
+} |
+ |
+/* |
+** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any |
+** existing data). Grow the buffer if required. |
+** |
+** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered |
+** trying to resize the buffer, return SQLITE_NOMEM. |
+*/ |
+static int fts3MsrBufferData( |
+ Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ |
+ char *pList, |
+ int nList |
+){ |
+ if( nList>pMsr->nBuffer ){ |
+ char *pNew; |
+ pMsr->nBuffer = nList*2; |
+ pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); |
+ if( !pNew ) return SQLITE_NOMEM; |
+ pMsr->aBuffer = pNew; |
+ } |
+ |
+ memcpy(pMsr->aBuffer, pList, nList); |
+ return SQLITE_OK; |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( |
+ Fts3Table *p, /* Virtual table handle */ |
+ Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ |
+ sqlite3_int64 *piDocid, /* OUT: Docid value */ |
+ char **paPoslist, /* OUT: Pointer to position list */ |
+ int *pnPoslist /* OUT: Size of position list in bytes */ |
+){ |
+ int nMerge = pMsr->nAdvance; |
+ Fts3SegReader **apSegment = pMsr->apSegment; |
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( |
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp |
+ ); |
+ |
+ if( nMerge==0 ){ |
+ *paPoslist = 0; |
+ return SQLITE_OK; |
+ } |
+ |
+ while( 1 ){ |
+ Fts3SegReader *pSeg; |
+ pSeg = pMsr->apSegment[0]; |
+ |
+ if( pSeg->pOffsetList==0 ){ |
+ *paPoslist = 0; |
+ break; |
+ }else{ |
+ int rc; |
+ char *pList; |
+ int nList; |
+ int j; |
+ sqlite3_int64 iDocid = apSegment[0]->iDocid; |
+ |
+ rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); |
+ j = 1; |
+ while( rc==SQLITE_OK |
+ && j<nMerge |
+ && apSegment[j]->pOffsetList |
+ && apSegment[j]->iDocid==iDocid |
+ ){ |
+ rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); |
+ j++; |
+ } |
+ if( rc!=SQLITE_OK ) return rc; |
+ fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); |
+ |
+ if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ |
+ rc = fts3MsrBufferData(pMsr, pList, nList+1); |
+ if( rc!=SQLITE_OK ) return rc; |
+ assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); |
+ pList = pMsr->aBuffer; |
+ } |
+ |
+ if( pMsr->iColFilter>=0 ){ |
+ fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); |
+ } |
+ |
+ if( nList>0 ){ |
+ *paPoslist = pList; |
+ *piDocid = iDocid; |
+ *pnPoslist = nList; |
+ break; |
+ } |
+ } |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+static int fts3SegReaderStart( |
+ Fts3Table *p, /* Virtual table handle */ |
+ Fts3MultiSegReader *pCsr, /* Cursor object */ |
+ const char *zTerm, /* Term searched for (or NULL) */ |
+ int nTerm /* Length of zTerm in bytes */ |
+){ |
+ int i; |
+ int nSeg = pCsr->nSegment; |
+ |
+ /* If the Fts3SegFilter defines a specific term (or term prefix) to search |
+ ** for, then advance each segment iterator until it points to a term of |
+ ** equal or greater value than the specified term. This prevents many |
+ ** unnecessary merge/sort operations for the case where single segment |
+ ** b-tree leaf nodes contain more than one term. |
+ */ |
+ for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){ |
+ int res = 0; |
+ Fts3SegReader *pSeg = pCsr->apSegment[i]; |
+ do { |
+ int rc = fts3SegReaderNext(p, pSeg, 0); |
+ if( rc!=SQLITE_OK ) return rc; |
+ }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); |
+ |
+ if( pSeg->bLookup && res!=0 ){ |
+ fts3SegReaderSetEof(pSeg); |
+ } |
+ } |
+ fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); |
+ |
+ return SQLITE_OK; |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( |
+ Fts3Table *p, /* Virtual table handle */ |
+ Fts3MultiSegReader *pCsr, /* Cursor object */ |
+ Fts3SegFilter *pFilter /* Restrictions on range of iteration */ |
+){ |
+ pCsr->pFilter = pFilter; |
+ return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( |
+ Fts3Table *p, /* Virtual table handle */ |
+ Fts3MultiSegReader *pCsr, /* Cursor object */ |
+ int iCol, /* Column to match on. */ |
+ const char *zTerm, /* Term to iterate through a doclist for */ |
+ int nTerm /* Number of bytes in zTerm */ |
+){ |
+ int i; |
+ int rc; |
+ int nSegment = pCsr->nSegment; |
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( |
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp |
+ ); |
+ |
+ assert( pCsr->pFilter==0 ); |
+ assert( zTerm && nTerm>0 ); |
+ |
+ /* Advance each segment iterator until it points to the term zTerm/nTerm. */ |
+ rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ /* Determine how many of the segments actually point to zTerm/nTerm. */ |
+ for(i=0; i<nSegment; i++){ |
+ Fts3SegReader *pSeg = pCsr->apSegment[i]; |
+ if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ |
+ break; |
+ } |
+ } |
+ pCsr->nAdvance = i; |
+ |
+ /* Advance each of the segments to point to the first docid. */ |
+ for(i=0; i<pCsr->nAdvance; i++){ |
+ rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); |
+ |
+ assert( iCol<0 || iCol<p->nColumn ); |
+ pCsr->iColFilter = iCol; |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function is called on a MultiSegReader that has been started using |
+** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also |
+** have been made. Calling this function puts the MultiSegReader in such |
+** a state that if the next two calls are: |
+** |
+** sqlite3Fts3SegReaderStart() |
+** sqlite3Fts3SegReaderStep() |
+** |
+** then the entire doclist for the term is available in |
+** MultiSegReader.aDoclist/nDoclist. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ |
+ int i; /* Used to iterate through segment-readers */ |
+ |
+ assert( pCsr->zTerm==0 ); |
+ assert( pCsr->nTerm==0 ); |
+ assert( pCsr->aDoclist==0 ); |
+ assert( pCsr->nDoclist==0 ); |
+ |
+ pCsr->nAdvance = 0; |
+ pCsr->bRestart = 1; |
+ for(i=0; i<pCsr->nSegment; i++){ |
+ pCsr->apSegment[i]->pOffsetList = 0; |
+ pCsr->apSegment[i]->nOffsetList = 0; |
+ pCsr->apSegment[i]->iDocid = 0; |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+ |
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( |
+ Fts3Table *p, /* Virtual table handle */ |
+ Fts3MultiSegReader *pCsr /* Cursor object */ |
+){ |
+ int rc = SQLITE_OK; |
+ |
+ int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); |
+ int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); |
+ int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); |
+ int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); |
+ int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); |
+ int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); |
+ |
+ Fts3SegReader **apSegment = pCsr->apSegment; |
+ int nSegment = pCsr->nSegment; |
+ Fts3SegFilter *pFilter = pCsr->pFilter; |
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( |
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp |
+ ); |
+ |
+ if( pCsr->nSegment==0 ) return SQLITE_OK; |
+ |
+ do { |
+ int nMerge; |
+ int i; |
+ |
+ /* Advance the first pCsr->nAdvance entries in the apSegment[] array |
+ ** forward. Then sort the list in order of current term again. |
+ */ |
+ for(i=0; i<pCsr->nAdvance; i++){ |
+ Fts3SegReader *pSeg = apSegment[i]; |
+ if( pSeg->bLookup ){ |
+ fts3SegReaderSetEof(pSeg); |
+ }else{ |
+ rc = fts3SegReaderNext(p, pSeg, 0); |
+ } |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); |
+ pCsr->nAdvance = 0; |
+ |
+ /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ |
+ assert( rc==SQLITE_OK ); |
+ if( apSegment[0]->aNode==0 ) break; |
+ |
+ pCsr->nTerm = apSegment[0]->nTerm; |
+ pCsr->zTerm = apSegment[0]->zTerm; |
+ |
+ /* If this is a prefix-search, and if the term that apSegment[0] points |
+ ** to does not share a suffix with pFilter->zTerm/nTerm, then all |
+ ** required callbacks have been made. In this case exit early. |
+ ** |
+ ** Similarly, if this is a search for an exact match, and the first term |
+ ** of segment apSegment[0] is not a match, exit early. |
+ */ |
+ if( pFilter->zTerm && !isScan ){ |
+ if( pCsr->nTerm<pFilter->nTerm |
+ || (!isPrefix && pCsr->nTerm>pFilter->nTerm) |
+ || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) |
+ ){ |
+ break; |
+ } |
+ } |
+ |
+ nMerge = 1; |
+ while( nMerge<nSegment |
+ && apSegment[nMerge]->aNode |
+ && apSegment[nMerge]->nTerm==pCsr->nTerm |
+ && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) |
+ ){ |
+ nMerge++; |
+ } |
+ |
+ assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); |
+ if( nMerge==1 |
+ && !isIgnoreEmpty |
+ && !isFirst |
+ && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) |
+ ){ |
+ pCsr->nDoclist = apSegment[0]->nDoclist; |
+ if( fts3SegReaderIsPending(apSegment[0]) ){ |
+ rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); |
+ pCsr->aDoclist = pCsr->aBuffer; |
+ }else{ |
+ pCsr->aDoclist = apSegment[0]->aDoclist; |
+ } |
+ if( rc==SQLITE_OK ) rc = SQLITE_ROW; |
+ }else{ |
+ int nDoclist = 0; /* Size of doclist */ |
+ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ |
+ |
+ /* The current term of the first nMerge entries in the array |
+ ** of Fts3SegReader objects is the same. The doclists must be merged |
+ ** and a single term returned with the merged doclist. |
+ */ |
+ for(i=0; i<nMerge; i++){ |
+ fts3SegReaderFirstDocid(p, apSegment[i]); |
+ } |
+ fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp); |
+ while( apSegment[0]->pOffsetList ){ |
+ int j; /* Number of segments that share a docid */ |
+ char *pList = 0; |
+ int nList = 0; |
+ int nByte; |
+ sqlite3_int64 iDocid = apSegment[0]->iDocid; |
+ fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); |
+ j = 1; |
+ while( j<nMerge |
+ && apSegment[j]->pOffsetList |
+ && apSegment[j]->iDocid==iDocid |
+ ){ |
+ fts3SegReaderNextDocid(p, apSegment[j], 0, 0); |
+ j++; |
+ } |
+ |
+ if( isColFilter ){ |
+ fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); |
+ } |
+ |
+ if( !isIgnoreEmpty || nList>0 ){ |
+ |
+ /* Calculate the 'docid' delta value to write into the merged |
+ ** doclist. */ |
+ sqlite3_int64 iDelta; |
+ if( p->bDescIdx && nDoclist>0 ){ |
+ iDelta = iPrev - iDocid; |
+ }else{ |
+ iDelta = iDocid - iPrev; |
+ } |
+ assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); |
+ assert( nDoclist>0 || iDelta==iDocid ); |
+ |
+ nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); |
+ if( nDoclist+nByte>pCsr->nBuffer ){ |
+ char *aNew; |
+ pCsr->nBuffer = (nDoclist+nByte)*2; |
+ aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); |
+ if( !aNew ){ |
+ return SQLITE_NOMEM; |
+ } |
+ pCsr->aBuffer = aNew; |
+ } |
+ |
+ if( isFirst ){ |
+ char *a = &pCsr->aBuffer[nDoclist]; |
+ int nWrite; |
+ |
+ nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); |
+ if( nWrite ){ |
+ iPrev = iDocid; |
+ nDoclist += nWrite; |
+ } |
+ }else{ |
+ nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); |
+ iPrev = iDocid; |
+ if( isRequirePos ){ |
+ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); |
+ nDoclist += nList; |
+ pCsr->aBuffer[nDoclist++] = '\0'; |
+ } |
+ } |
+ } |
+ |
+ fts3SegReaderSort(apSegment, nMerge, j, xCmp); |
+ } |
+ if( nDoclist>0 ){ |
+ pCsr->aDoclist = pCsr->aBuffer; |
+ pCsr->nDoclist = nDoclist; |
+ rc = SQLITE_ROW; |
+ } |
+ } |
+ pCsr->nAdvance = nMerge; |
+ }while( rc==SQLITE_OK ); |
+ |
+ return rc; |
+} |
+ |
+ |
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( |
+ Fts3MultiSegReader *pCsr /* Cursor object */ |
+){ |
+ if( pCsr ){ |
+ int i; |
+ for(i=0; i<pCsr->nSegment; i++){ |
+ sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); |
+ } |
+ sqlite3_free(pCsr->apSegment); |
+ sqlite3_free(pCsr->aBuffer); |
+ |
+ pCsr->nSegment = 0; |
+ pCsr->apSegment = 0; |
+ pCsr->aBuffer = 0; |
+ } |
+} |
+ |
+/* |
+** Decode the "end_block" field, selected by column iCol of the SELECT |
+** statement passed as the first argument. |
+** |
+** The "end_block" field may contain either an integer, or a text field |
+** containing the text representation of two non-negative integers separated |
+** by one or more space (0x20) characters. In the first case, set *piEndBlock |
+** to the integer value and *pnByte to zero before returning. In the second, |
+** set *piEndBlock to the first value and *pnByte to the second. |
+*/ |
+static void fts3ReadEndBlockField( |
+ sqlite3_stmt *pStmt, |
+ int iCol, |
+ i64 *piEndBlock, |
+ i64 *pnByte |
+){ |
+ const unsigned char *zText = sqlite3_column_text(pStmt, iCol); |
+ if( zText ){ |
+ int i; |
+ int iMul = 1; |
+ i64 iVal = 0; |
+ for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ |
+ iVal = iVal*10 + (zText[i] - '0'); |
+ } |
+ *piEndBlock = iVal; |
+ while( zText[i]==' ' ) i++; |
+ iVal = 0; |
+ if( zText[i]=='-' ){ |
+ i++; |
+ iMul = -1; |
+ } |
+ for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ |
+ iVal = iVal*10 + (zText[i] - '0'); |
+ } |
+ *pnByte = (iVal * (i64)iMul); |
+ } |
+} |
+ |
+ |
+/* |
+** A segment of size nByte bytes has just been written to absolute level |
+** iAbsLevel. Promote any segments that should be promoted as a result. |
+*/ |
+static int fts3PromoteSegments( |
+ Fts3Table *p, /* FTS table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute level just updated */ |
+ sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ |
+){ |
+ int rc = SQLITE_OK; |
+ sqlite3_stmt *pRange; |
+ |
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); |
+ |
+ if( rc==SQLITE_OK ){ |
+ int bOk = 0; |
+ i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; |
+ i64 nLimit = (nByte*3)/2; |
+ |
+ /* Loop through all entries in the %_segdir table corresponding to |
+ ** segments in this index on levels greater than iAbsLevel. If there is |
+ ** at least one such segment, and it is possible to determine that all |
+ ** such segments are smaller than nLimit bytes in size, they will be |
+ ** promoted to level iAbsLevel. */ |
+ sqlite3_bind_int64(pRange, 1, iAbsLevel+1); |
+ sqlite3_bind_int64(pRange, 2, iLast); |
+ while( SQLITE_ROW==sqlite3_step(pRange) ){ |
+ i64 nSize = 0, dummy; |
+ fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); |
+ if( nSize<=0 || nSize>nLimit ){ |
+ /* If nSize==0, then the %_segdir.end_block field does not not |
+ ** contain a size value. This happens if it was written by an |
+ ** old version of FTS. In this case it is not possible to determine |
+ ** the size of the segment, and so segment promotion does not |
+ ** take place. */ |
+ bOk = 0; |
+ break; |
+ } |
+ bOk = 1; |
+ } |
+ rc = sqlite3_reset(pRange); |
+ |
+ if( bOk ){ |
+ int iIdx = 0; |
+ sqlite3_stmt *pUpdate1 = 0; |
+ sqlite3_stmt *pUpdate2 = 0; |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ |
+ /* Loop through all %_segdir entries for segments in this index with |
+ ** levels equal to or greater than iAbsLevel. As each entry is visited, |
+ ** updated it to set (level = -1) and (idx = N), where N is 0 for the |
+ ** oldest segment in the range, 1 for the next oldest, and so on. |
+ ** |
+ ** In other words, move all segments being promoted to level -1, |
+ ** setting the "idx" fields as appropriate to keep them in the same |
+ ** order. The contents of level -1 (which is never used, except |
+ ** transiently here), will be moved back to level iAbsLevel below. */ |
+ sqlite3_bind_int64(pRange, 1, iAbsLevel); |
+ while( SQLITE_ROW==sqlite3_step(pRange) ){ |
+ sqlite3_bind_int(pUpdate1, 1, iIdx++); |
+ sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); |
+ sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); |
+ sqlite3_step(pUpdate1); |
+ rc = sqlite3_reset(pUpdate1); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_reset(pRange); |
+ break; |
+ } |
+ } |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_reset(pRange); |
+ } |
+ |
+ /* Move level -1 to level iAbsLevel */ |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); |
+ sqlite3_step(pUpdate2); |
+ rc = sqlite3_reset(pUpdate2); |
+ } |
+ } |
+ } |
+ |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Merge all level iLevel segments in the database into a single |
+** iLevel+1 segment. Or, if iLevel<0, merge all segments into a |
+** single segment with a level equal to the numerically largest level |
+** currently present in the database. |
+** |
+** If this function is called with iLevel<0, but there is only one |
+** segment in the database, SQLITE_DONE is returned immediately. |
+** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, |
+** an SQLite error code is returned. |
+*/ |
+static int fts3SegmentMerge( |
+ Fts3Table *p, |
+ int iLangid, /* Language id to merge */ |
+ int iIndex, /* Index in p->aIndex[] to merge */ |
+ int iLevel /* Level to merge */ |
+){ |
+ int rc; /* Return code */ |
+ int iIdx = 0; /* Index of new segment */ |
+ sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ |
+ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ |
+ Fts3SegFilter filter; /* Segment term filter condition */ |
+ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ |
+ int bIgnoreEmpty = 0; /* True to ignore empty segments */ |
+ i64 iMaxLevel = 0; /* Max level number for this index/langid */ |
+ |
+ assert( iLevel==FTS3_SEGCURSOR_ALL |
+ || iLevel==FTS3_SEGCURSOR_PENDING |
+ || iLevel>=0 |
+ ); |
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); |
+ assert( iIndex>=0 && iIndex<p->nIndex ); |
+ |
+ rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); |
+ if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; |
+ |
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){ |
+ rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); |
+ if( rc!=SQLITE_OK ) goto finished; |
+ } |
+ |
+ if( iLevel==FTS3_SEGCURSOR_ALL ){ |
+ /* This call is to merge all segments in the database to a single |
+ ** segment. The level of the new segment is equal to the numerically |
+ ** greatest segment level currently present in the database for this |
+ ** index. The idx of the new segment is always 0. */ |
+ if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ |
+ rc = SQLITE_DONE; |
+ goto finished; |
+ } |
+ iNewLevel = iMaxLevel; |
+ bIgnoreEmpty = 1; |
+ |
+ }else{ |
+ /* This call is to merge all segments at level iLevel. find the next |
+ ** available segment index at level iLevel+1. The call to |
+ ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to |
+ ** a single iLevel+2 segment if necessary. */ |
+ assert( FTS3_SEGCURSOR_PENDING==-1 ); |
+ iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); |
+ rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); |
+ bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); |
+ } |
+ if( rc!=SQLITE_OK ) goto finished; |
+ |
+ assert( csr.nSegment>0 ); |
+ assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); |
+ assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) ); |
+ |
+ memset(&filter, 0, sizeof(Fts3SegFilter)); |
+ filter.flags = FTS3_SEGMENT_REQUIRE_POS; |
+ filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); |
+ |
+ rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); |
+ while( SQLITE_OK==rc ){ |
+ rc = sqlite3Fts3SegReaderStep(p, &csr); |
+ if( rc!=SQLITE_ROW ) break; |
+ rc = fts3SegWriterAdd(p, &pWriter, 1, |
+ csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); |
+ } |
+ if( rc!=SQLITE_OK ) goto finished; |
+ assert( pWriter || bIgnoreEmpty ); |
+ |
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){ |
+ rc = fts3DeleteSegdir( |
+ p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment |
+ ); |
+ if( rc!=SQLITE_OK ) goto finished; |
+ } |
+ if( pWriter ){ |
+ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); |
+ if( rc==SQLITE_OK ){ |
+ if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevel<iMaxLevel ){ |
+ rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); |
+ } |
+ } |
+ } |
+ |
+ finished: |
+ fts3SegWriterFree(pWriter); |
+ sqlite3Fts3SegReaderFinish(&csr); |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Flush the contents of pendingTerms to level 0 segments. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ |
+ int rc = SQLITE_OK; |
+ int i; |
+ |
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ |
+ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); |
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
+ } |
+ sqlite3Fts3PendingTermsClear(p); |
+ |
+ /* Determine the auto-incr-merge setting if unknown. If enabled, |
+ ** estimate the number of leaf blocks of content to be written |
+ */ |
+ if( rc==SQLITE_OK && p->bHasStat |
+ && p->nAutoincrmerge==0xff && p->nLeafAdd>0 |
+ ){ |
+ sqlite3_stmt *pStmt = 0; |
+ rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); |
+ rc = sqlite3_step(pStmt); |
+ if( rc==SQLITE_ROW ){ |
+ p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); |
+ if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; |
+ }else if( rc==SQLITE_DONE ){ |
+ p->nAutoincrmerge = 0; |
+ } |
+ rc = sqlite3_reset(pStmt); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Encode N integers as varints into a blob. |
+*/ |
+static void fts3EncodeIntArray( |
+ int N, /* The number of integers to encode */ |
+ u32 *a, /* The integer values */ |
+ char *zBuf, /* Write the BLOB here */ |
+ int *pNBuf /* Write number of bytes if zBuf[] used here */ |
+){ |
+ int i, j; |
+ for(i=j=0; i<N; i++){ |
+ j += sqlite3Fts3PutVarint(&zBuf[j], (sqlite3_int64)a[i]); |
+ } |
+ *pNBuf = j; |
+} |
+ |
+/* |
+** Decode a blob of varints into N integers |
+*/ |
+static void fts3DecodeIntArray( |
+ int N, /* The number of integers to decode */ |
+ u32 *a, /* Write the integer values */ |
+ const char *zBuf, /* The BLOB containing the varints */ |
+ int nBuf /* size of the BLOB */ |
+){ |
+ int i, j; |
+ UNUSED_PARAMETER(nBuf); |
+ for(i=j=0; i<N; i++){ |
+ sqlite3_int64 x; |
+ j += sqlite3Fts3GetVarint(&zBuf[j], &x); |
+ assert(j<=nBuf); |
+ a[i] = (u32)(x & 0xffffffff); |
+ } |
+} |
+ |
+/* |
+** Insert the sizes (in tokens) for each column of the document |
+** with docid equal to p->iPrevDocid. The sizes are encoded as |
+** a blob of varints. |
+*/ |
+static void fts3InsertDocsize( |
+ int *pRC, /* Result code */ |
+ Fts3Table *p, /* Table into which to insert */ |
+ u32 *aSz /* Sizes of each column, in tokens */ |
+){ |
+ char *pBlob; /* The BLOB encoding of the document size */ |
+ int nBlob; /* Number of bytes in the BLOB */ |
+ sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ |
+ int rc; /* Result code from subfunctions */ |
+ |
+ if( *pRC ) return; |
+ pBlob = sqlite3_malloc( 10*p->nColumn ); |
+ if( pBlob==0 ){ |
+ *pRC = SQLITE_NOMEM; |
+ return; |
+ } |
+ fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); |
+ rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); |
+ if( rc ){ |
+ sqlite3_free(pBlob); |
+ *pRC = rc; |
+ return; |
+ } |
+ sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); |
+ sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); |
+ sqlite3_step(pStmt); |
+ *pRC = sqlite3_reset(pStmt); |
+} |
+ |
+/* |
+** Record 0 of the %_stat table contains a blob consisting of N varints, |
+** where N is the number of user defined columns in the fts3 table plus |
+** two. If nCol is the number of user defined columns, then values of the |
+** varints are set as follows: |
+** |
+** Varint 0: Total number of rows in the table. |
+** |
+** Varint 1..nCol: For each column, the total number of tokens stored in |
+** the column for all rows of the table. |
+** |
+** Varint 1+nCol: The total size, in bytes, of all text values in all |
+** columns of all rows of the table. |
+** |
+*/ |
+static void fts3UpdateDocTotals( |
+ int *pRC, /* The result code */ |
+ Fts3Table *p, /* Table being updated */ |
+ u32 *aSzIns, /* Size increases */ |
+ u32 *aSzDel, /* Size decreases */ |
+ int nChng /* Change in the number of documents */ |
+){ |
+ char *pBlob; /* Storage for BLOB written into %_stat */ |
+ int nBlob; /* Size of BLOB written into %_stat */ |
+ u32 *a; /* Array of integers that becomes the BLOB */ |
+ sqlite3_stmt *pStmt; /* Statement for reading and writing */ |
+ int i; /* Loop counter */ |
+ int rc; /* Result code from subfunctions */ |
+ |
+ const int nStat = p->nColumn+2; |
+ |
+ if( *pRC ) return; |
+ a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); |
+ if( a==0 ){ |
+ *pRC = SQLITE_NOMEM; |
+ return; |
+ } |
+ pBlob = (char*)&a[nStat]; |
+ rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); |
+ if( rc ){ |
+ sqlite3_free(a); |
+ *pRC = rc; |
+ return; |
+ } |
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); |
+ if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
+ fts3DecodeIntArray(nStat, a, |
+ sqlite3_column_blob(pStmt, 0), |
+ sqlite3_column_bytes(pStmt, 0)); |
+ }else{ |
+ memset(a, 0, sizeof(u32)*(nStat) ); |
+ } |
+ rc = sqlite3_reset(pStmt); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_free(a); |
+ *pRC = rc; |
+ return; |
+ } |
+ if( nChng<0 && a[0]<(u32)(-nChng) ){ |
+ a[0] = 0; |
+ }else{ |
+ a[0] += nChng; |
+ } |
+ for(i=0; i<p->nColumn+1; i++){ |
+ u32 x = a[i+1]; |
+ if( x+aSzIns[i] < aSzDel[i] ){ |
+ x = 0; |
+ }else{ |
+ x = x + aSzIns[i] - aSzDel[i]; |
+ } |
+ a[i+1] = x; |
+ } |
+ fts3EncodeIntArray(nStat, a, pBlob, &nBlob); |
+ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); |
+ if( rc ){ |
+ sqlite3_free(a); |
+ *pRC = rc; |
+ return; |
+ } |
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); |
+ sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); |
+ sqlite3_step(pStmt); |
+ *pRC = sqlite3_reset(pStmt); |
+ sqlite3_free(a); |
+} |
+ |
+/* |
+** Merge the entire database so that there is one segment for each |
+** iIndex/iLangid combination. |
+*/ |
+static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ |
+ int bSeenDone = 0; |
+ int rc; |
+ sqlite3_stmt *pAllLangid = 0; |
+ |
+ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); |
+ sqlite3_bind_int(pAllLangid, 2, p->nIndex); |
+ while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ |
+ int i; |
+ int iLangid = sqlite3_column_int(pAllLangid, 0); |
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ |
+ rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); |
+ if( rc==SQLITE_DONE ){ |
+ bSeenDone = 1; |
+ rc = SQLITE_OK; |
+ } |
+ } |
+ } |
+ rc2 = sqlite3_reset(pAllLangid); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ sqlite3Fts3SegmentsClose(p); |
+ sqlite3Fts3PendingTermsClear(p); |
+ |
+ return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; |
+} |
+ |
+/* |
+** This function is called when the user executes the following statement: |
+** |
+** INSERT INTO <tbl>(<tbl>) VALUES('rebuild'); |
+** |
+** The entire FTS index is discarded and rebuilt. If the table is one |
+** created using the content=xxx option, then the new index is based on |
+** the current contents of the xxx table. Otherwise, it is rebuilt based |
+** on the contents of the %_content table. |
+*/ |
+static int fts3DoRebuild(Fts3Table *p){ |
+ int rc; /* Return Code */ |
+ |
+ rc = fts3DeleteAll(p, 0); |
+ if( rc==SQLITE_OK ){ |
+ u32 *aSz = 0; |
+ u32 *aSzIns = 0; |
+ u32 *aSzDel = 0; |
+ sqlite3_stmt *pStmt = 0; |
+ int nEntry = 0; |
+ |
+ /* Compose and prepare an SQL statement to loop through the content table */ |
+ char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); |
+ if( !zSql ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); |
+ sqlite3_free(zSql); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ int nByte = sizeof(u32) * (p->nColumn+1)*3; |
+ aSz = (u32 *)sqlite3_malloc(nByte); |
+ if( aSz==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(aSz, 0, nByte); |
+ aSzIns = &aSz[p->nColumn+1]; |
+ aSzDel = &aSzIns[p->nColumn+1]; |
+ } |
+ } |
+ |
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ int iCol; |
+ int iLangid = langidFromSelect(p, pStmt); |
+ rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); |
+ memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); |
+ for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){ |
+ if( p->abNotindexed[iCol]==0 ){ |
+ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); |
+ rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); |
+ aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); |
+ } |
+ } |
+ if( p->bHasDocsize ){ |
+ fts3InsertDocsize(&rc, p, aSz); |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_finalize(pStmt); |
+ pStmt = 0; |
+ }else{ |
+ nEntry++; |
+ for(iCol=0; iCol<=p->nColumn; iCol++){ |
+ aSzIns[iCol] += aSz[iCol]; |
+ } |
+ } |
+ } |
+ if( p->bFts4 ){ |
+ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); |
+ } |
+ sqlite3_free(aSz); |
+ |
+ if( pStmt ){ |
+ int rc2 = sqlite3_finalize(pStmt); |
+ if( rc==SQLITE_OK ){ |
+ rc = rc2; |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+/* |
+** This function opens a cursor used to read the input data for an |
+** incremental merge operation. Specifically, it opens a cursor to scan |
+** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute |
+** level iAbsLevel. |
+*/ |
+static int fts3IncrmergeCsr( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute level to open */ |
+ int nSeg, /* Number of segments to merge */ |
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */ |
+){ |
+ int rc; /* Return Code */ |
+ sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ |
+ int nByte; /* Bytes allocated at pCsr->apSegment[] */ |
+ |
+ /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ |
+ memset(pCsr, 0, sizeof(*pCsr)); |
+ nByte = sizeof(Fts3SegReader *) * nSeg; |
+ pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); |
+ |
+ if( pCsr->apSegment==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(pCsr->apSegment, 0, nByte); |
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ int i; |
+ int rc2; |
+ sqlite3_bind_int64(pStmt, 1, iAbsLevel); |
+ assert( pCsr->nSegment==0 ); |
+ for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<nSeg; i++){ |
+ rc = sqlite3Fts3SegReaderNew(i, 0, |
+ sqlite3_column_int64(pStmt, 1), /* segdir.start_block */ |
+ sqlite3_column_int64(pStmt, 2), /* segdir.leaves_end_block */ |
+ sqlite3_column_int64(pStmt, 3), /* segdir.end_block */ |
+ sqlite3_column_blob(pStmt, 4), /* segdir.root */ |
+ sqlite3_column_bytes(pStmt, 4), /* segdir.root */ |
+ &pCsr->apSegment[i] |
+ ); |
+ pCsr->nSegment++; |
+ } |
+ rc2 = sqlite3_reset(pStmt); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ return rc; |
+} |
+ |
+typedef struct IncrmergeWriter IncrmergeWriter; |
+typedef struct NodeWriter NodeWriter; |
+typedef struct Blob Blob; |
+typedef struct NodeReader NodeReader; |
+ |
+/* |
+** An instance of the following structure is used as a dynamic buffer |
+** to build up nodes or other blobs of data in. |
+** |
+** The function blobGrowBuffer() is used to extend the allocation. |
+*/ |
+struct Blob { |
+ char *a; /* Pointer to allocation */ |
+ int n; /* Number of valid bytes of data in a[] */ |
+ int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ |
+}; |
+ |
+/* |
+** This structure is used to build up buffers containing segment b-tree |
+** nodes (blocks). |
+*/ |
+struct NodeWriter { |
+ sqlite3_int64 iBlock; /* Current block id */ |
+ Blob key; /* Last key written to the current block */ |
+ Blob block; /* Current block image */ |
+}; |
+ |
+/* |
+** An object of this type contains the state required to create or append |
+** to an appendable b-tree segment. |
+*/ |
+struct IncrmergeWriter { |
+ int nLeafEst; /* Space allocated for leaf blocks */ |
+ int nWork; /* Number of leaf pages flushed */ |
+ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ |
+ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ |
+ sqlite3_int64 iStart; /* Block number of first allocated block */ |
+ sqlite3_int64 iEnd; /* Block number of last allocated block */ |
+ sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ |
+ u8 bNoLeafData; /* If true, store 0 for segment size */ |
+ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; |
+}; |
+ |
+/* |
+** An object of the following type is used to read data from a single |
+** FTS segment node. See the following functions: |
+** |
+** nodeReaderInit() |
+** nodeReaderNext() |
+** nodeReaderRelease() |
+*/ |
+struct NodeReader { |
+ const char *aNode; |
+ int nNode; |
+ int iOff; /* Current offset within aNode[] */ |
+ |
+ /* Output variables. Containing the current node entry. */ |
+ sqlite3_int64 iChild; /* Pointer to child node */ |
+ Blob term; /* Current term */ |
+ const char *aDoclist; /* Pointer to doclist */ |
+ int nDoclist; /* Size of doclist in bytes */ |
+}; |
+ |
+/* |
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op. |
+** Otherwise, if the allocation at pBlob->a is not already at least nMin |
+** bytes in size, extend (realloc) it to be so. |
+** |
+** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a |
+** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc |
+** to reflect the new size of the pBlob->a[] buffer. |
+*/ |
+static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ |
+ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ |
+ int nAlloc = nMin; |
+ char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); |
+ if( a ){ |
+ pBlob->nAlloc = nAlloc; |
+ pBlob->a = a; |
+ }else{ |
+ *pRc = SQLITE_NOMEM; |
+ } |
+ } |
+} |
+ |
+/* |
+** Attempt to advance the node-reader object passed as the first argument to |
+** the next entry on the node. |
+** |
+** Return an error code if an error occurs (SQLITE_NOMEM is possible). |
+** Otherwise return SQLITE_OK. If there is no next entry on the node |
+** (e.g. because the current entry is the last) set NodeReader->aNode to |
+** NULL to indicate EOF. Otherwise, populate the NodeReader structure output |
+** variables for the new entry. |
+*/ |
+static int nodeReaderNext(NodeReader *p){ |
+ int bFirst = (p->term.n==0); /* True for first term on the node */ |
+ int nPrefix = 0; /* Bytes to copy from previous term */ |
+ int nSuffix = 0; /* Bytes to append to the prefix */ |
+ int rc = SQLITE_OK; /* Return code */ |
+ |
+ assert( p->aNode ); |
+ if( p->iChild && bFirst==0 ) p->iChild++; |
+ if( p->iOff>=p->nNode ){ |
+ /* EOF */ |
+ p->aNode = 0; |
+ }else{ |
+ if( bFirst==0 ){ |
+ p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); |
+ } |
+ p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); |
+ |
+ blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); |
+ if( rc==SQLITE_OK ){ |
+ memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); |
+ p->term.n = nPrefix+nSuffix; |
+ p->iOff += nSuffix; |
+ if( p->iChild==0 ){ |
+ p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); |
+ p->aDoclist = &p->aNode[p->iOff]; |
+ p->iOff += p->nDoclist; |
+ } |
+ } |
+ } |
+ |
+ assert( p->iOff<=p->nNode ); |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Release all dynamic resources held by node-reader object *p. |
+*/ |
+static void nodeReaderRelease(NodeReader *p){ |
+ sqlite3_free(p->term.a); |
+} |
+ |
+/* |
+** Initialize a node-reader object to read the node in buffer aNode/nNode. |
+** |
+** If successful, SQLITE_OK is returned and the NodeReader object set to |
+** point to the first entry on the node (if any). Otherwise, an SQLite |
+** error code is returned. |
+*/ |
+static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ |
+ memset(p, 0, sizeof(NodeReader)); |
+ p->aNode = aNode; |
+ p->nNode = nNode; |
+ |
+ /* Figure out if this is a leaf or an internal node. */ |
+ if( p->aNode[0] ){ |
+ /* An internal node. */ |
+ p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); |
+ }else{ |
+ p->iOff = 1; |
+ } |
+ |
+ return nodeReaderNext(p); |
+} |
+ |
+/* |
+** This function is called while writing an FTS segment each time a leaf o |
+** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed |
+** to be greater than the largest key on the node just written, but smaller |
+** than or equal to the first key that will be written to the next leaf |
+** node. |
+** |
+** The block id of the leaf node just written to disk may be found in |
+** (pWriter->aNodeWriter[0].iBlock) when this function is called. |
+*/ |
+static int fts3IncrmergePush( |
+ Fts3Table *p, /* Fts3 table handle */ |
+ IncrmergeWriter *pWriter, /* Writer object */ |
+ const char *zTerm, /* Term to write to internal node */ |
+ int nTerm /* Bytes at zTerm */ |
+){ |
+ sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; |
+ int iLayer; |
+ |
+ assert( nTerm>0 ); |
+ for(iLayer=1; ALWAYS(iLayer<FTS_MAX_APPENDABLE_HEIGHT); iLayer++){ |
+ sqlite3_int64 iNextPtr = 0; |
+ NodeWriter *pNode = &pWriter->aNodeWriter[iLayer]; |
+ int rc = SQLITE_OK; |
+ int nPrefix; |
+ int nSuffix; |
+ int nSpace; |
+ |
+ /* Figure out how much space the key will consume if it is written to |
+ ** the current node of layer iLayer. Due to the prefix compression, |
+ ** the space required changes depending on which node the key is to |
+ ** be added to. */ |
+ nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); |
+ nSuffix = nTerm - nPrefix; |
+ nSpace = sqlite3Fts3VarintLen(nPrefix); |
+ nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; |
+ |
+ if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ |
+ /* If the current node of layer iLayer contains zero keys, or if adding |
+ ** the key to it will not cause it to grow to larger than nNodeSize |
+ ** bytes in size, write the key here. */ |
+ |
+ Blob *pBlk = &pNode->block; |
+ if( pBlk->n==0 ){ |
+ blobGrowBuffer(pBlk, p->nNodeSize, &rc); |
+ if( rc==SQLITE_OK ){ |
+ pBlk->a[0] = (char)iLayer; |
+ pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); |
+ } |
+ } |
+ blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); |
+ blobGrowBuffer(&pNode->key, nTerm, &rc); |
+ |
+ if( rc==SQLITE_OK ){ |
+ if( pNode->key.n ){ |
+ pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); |
+ } |
+ pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); |
+ memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); |
+ pBlk->n += nSuffix; |
+ |
+ memcpy(pNode->key.a, zTerm, nTerm); |
+ pNode->key.n = nTerm; |
+ } |
+ }else{ |
+ /* Otherwise, flush the current node of layer iLayer to disk. |
+ ** Then allocate a new, empty sibling node. The key will be written |
+ ** into the parent of this node. */ |
+ rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); |
+ |
+ assert( pNode->block.nAlloc>=p->nNodeSize ); |
+ pNode->block.a[0] = (char)iLayer; |
+ pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); |
+ |
+ iNextPtr = pNode->iBlock; |
+ pNode->iBlock++; |
+ pNode->key.n = 0; |
+ } |
+ |
+ if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; |
+ iPtr = iNextPtr; |
+ } |
+ |
+ assert( 0 ); |
+ return 0; |
+} |
+ |
+/* |
+** Append a term and (optionally) doclist to the FTS segment node currently |
+** stored in blob *pNode. The node need not contain any terms, but the |
+** header must be written before this function is called. |
+** |
+** A node header is a single 0x00 byte for a leaf node, or a height varint |
+** followed by the left-hand-child varint for an internal node. |
+** |
+** The term to be appended is passed via arguments zTerm/nTerm. For a |
+** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal |
+** node, both aDoclist and nDoclist must be passed 0. |
+** |
+** If the size of the value in blob pPrev is zero, then this is the first |
+** term written to the node. Otherwise, pPrev contains a copy of the |
+** previous term. Before this function returns, it is updated to contain a |
+** copy of zTerm/nTerm. |
+** |
+** It is assumed that the buffer associated with pNode is already large |
+** enough to accommodate the new entry. The buffer associated with pPrev |
+** is extended by this function if requrired. |
+** |
+** If an error (i.e. OOM condition) occurs, an SQLite error code is |
+** returned. Otherwise, SQLITE_OK. |
+*/ |
+static int fts3AppendToNode( |
+ Blob *pNode, /* Current node image to append to */ |
+ Blob *pPrev, /* Buffer containing previous term written */ |
+ const char *zTerm, /* New term to write */ |
+ int nTerm, /* Size of zTerm in bytes */ |
+ const char *aDoclist, /* Doclist (or NULL) to write */ |
+ int nDoclist /* Size of aDoclist in bytes */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int bFirst = (pPrev->n==0); /* True if this is the first term written */ |
+ int nPrefix; /* Size of term prefix in bytes */ |
+ int nSuffix; /* Size of term suffix in bytes */ |
+ |
+ /* Node must have already been started. There must be a doclist for a |
+ ** leaf node, and there must not be a doclist for an internal node. */ |
+ assert( pNode->n>0 ); |
+ assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); |
+ |
+ blobGrowBuffer(pPrev, nTerm, &rc); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); |
+ nSuffix = nTerm - nPrefix; |
+ memcpy(pPrev->a, zTerm, nTerm); |
+ pPrev->n = nTerm; |
+ |
+ if( bFirst==0 ){ |
+ pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); |
+ } |
+ pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); |
+ memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); |
+ pNode->n += nSuffix; |
+ |
+ if( aDoclist ){ |
+ pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); |
+ memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); |
+ pNode->n += nDoclist; |
+ } |
+ |
+ assert( pNode->n<=pNode->nAlloc ); |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Append the current term and doclist pointed to by cursor pCsr to the |
+** appendable b-tree segment opened for writing by pWriter. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code otherwise. |
+*/ |
+static int fts3IncrmergeAppend( |
+ Fts3Table *p, /* Fts3 table handle */ |
+ IncrmergeWriter *pWriter, /* Writer object */ |
+ Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ |
+){ |
+ const char *zTerm = pCsr->zTerm; |
+ int nTerm = pCsr->nTerm; |
+ const char *aDoclist = pCsr->aDoclist; |
+ int nDoclist = pCsr->nDoclist; |
+ int rc = SQLITE_OK; /* Return code */ |
+ int nSpace; /* Total space in bytes required on leaf */ |
+ int nPrefix; /* Size of prefix shared with previous term */ |
+ int nSuffix; /* Size of suffix (nTerm - nPrefix) */ |
+ NodeWriter *pLeaf; /* Object used to write leaf nodes */ |
+ |
+ pLeaf = &pWriter->aNodeWriter[0]; |
+ nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); |
+ nSuffix = nTerm - nPrefix; |
+ |
+ nSpace = sqlite3Fts3VarintLen(nPrefix); |
+ nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; |
+ nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; |
+ |
+ /* If the current block is not empty, and if adding this term/doclist |
+ ** to the current block would make it larger than Fts3Table.nNodeSize |
+ ** bytes, write this block out to the database. */ |
+ if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ |
+ rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); |
+ pWriter->nWork++; |
+ |
+ /* Add the current term to the parent node. The term added to the |
+ ** parent must: |
+ ** |
+ ** a) be greater than the largest term on the leaf node just written |
+ ** to the database (still available in pLeaf->key), and |
+ ** |
+ ** b) be less than or equal to the term about to be added to the new |
+ ** leaf node (zTerm/nTerm). |
+ ** |
+ ** In other words, it must be the prefix of zTerm 1 byte longer than |
+ ** the common prefix (if any) of zTerm and pWriter->zTerm. |
+ */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); |
+ } |
+ |
+ /* Advance to the next output block */ |
+ pLeaf->iBlock++; |
+ pLeaf->key.n = 0; |
+ pLeaf->block.n = 0; |
+ |
+ nSuffix = nTerm; |
+ nSpace = 1; |
+ nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; |
+ nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; |
+ } |
+ |
+ pWriter->nLeafData += nSpace; |
+ blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); |
+ if( rc==SQLITE_OK ){ |
+ if( pLeaf->block.n==0 ){ |
+ pLeaf->block.n = 1; |
+ pLeaf->block.a[0] = '\0'; |
+ } |
+ rc = fts3AppendToNode( |
+ &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist |
+ ); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This function is called to release all dynamic resources held by the |
+** merge-writer object pWriter, and if no error has occurred, to flush |
+** all outstanding node buffers held by pWriter to disk. |
+** |
+** If *pRc is not SQLITE_OK when this function is called, then no attempt |
+** is made to write any data to disk. Instead, this function serves only |
+** to release outstanding resources. |
+** |
+** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while |
+** flushing buffers to disk, *pRc is set to an SQLite error code before |
+** returning. |
+*/ |
+static void fts3IncrmergeRelease( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ IncrmergeWriter *pWriter, /* Merge-writer object */ |
+ int *pRc /* IN/OUT: Error code */ |
+){ |
+ int i; /* Used to iterate through non-root layers */ |
+ int iRoot; /* Index of root in pWriter->aNodeWriter */ |
+ NodeWriter *pRoot; /* NodeWriter for root node */ |
+ int rc = *pRc; /* Error code */ |
+ |
+ /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment |
+ ** root node. If the segment fits entirely on a single leaf node, iRoot |
+ ** will be set to 0. If the root node is the parent of the leaves, iRoot |
+ ** will be 1. And so on. */ |
+ for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ |
+ NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; |
+ if( pNode->block.n>0 ) break; |
+ assert( *pRc || pNode->block.nAlloc==0 ); |
+ assert( *pRc || pNode->key.nAlloc==0 ); |
+ sqlite3_free(pNode->block.a); |
+ sqlite3_free(pNode->key.a); |
+ } |
+ |
+ /* Empty output segment. This is a no-op. */ |
+ if( iRoot<0 ) return; |
+ |
+ /* The entire output segment fits on a single node. Normally, this means |
+ ** the node would be stored as a blob in the "root" column of the %_segdir |
+ ** table. However, this is not permitted in this case. The problem is that |
+ ** space has already been reserved in the %_segments table, and so the |
+ ** start_block and end_block fields of the %_segdir table must be populated. |
+ ** And, by design or by accident, released versions of FTS cannot handle |
+ ** segments that fit entirely on the root node with start_block!=0. |
+ ** |
+ ** Instead, create a synthetic root node that contains nothing but a |
+ ** pointer to the single content node. So that the segment consists of a |
+ ** single leaf and a single interior (root) node. |
+ ** |
+ ** Todo: Better might be to defer allocating space in the %_segments |
+ ** table until we are sure it is needed. |
+ */ |
+ if( iRoot==0 ){ |
+ Blob *pBlock = &pWriter->aNodeWriter[1].block; |
+ blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); |
+ if( rc==SQLITE_OK ){ |
+ pBlock->a[0] = 0x01; |
+ pBlock->n = 1 + sqlite3Fts3PutVarint( |
+ &pBlock->a[1], pWriter->aNodeWriter[0].iBlock |
+ ); |
+ } |
+ iRoot = 1; |
+ } |
+ pRoot = &pWriter->aNodeWriter[iRoot]; |
+ |
+ /* Flush all currently outstanding nodes to disk. */ |
+ for(i=0; i<iRoot; i++){ |
+ NodeWriter *pNode = &pWriter->aNodeWriter[i]; |
+ if( pNode->block.n>0 && rc==SQLITE_OK ){ |
+ rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); |
+ } |
+ sqlite3_free(pNode->block.a); |
+ sqlite3_free(pNode->key.a); |
+ } |
+ |
+ /* Write the %_segdir record. */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3WriteSegdir(p, |
+ pWriter->iAbsLevel+1, /* level */ |
+ pWriter->iIdx, /* idx */ |
+ pWriter->iStart, /* start_block */ |
+ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ |
+ pWriter->iEnd, /* end_block */ |
+ (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ |
+ pRoot->block.a, pRoot->block.n /* root */ |
+ ); |
+ } |
+ sqlite3_free(pRoot->block.a); |
+ sqlite3_free(pRoot->key.a); |
+ |
+ *pRc = rc; |
+} |
+ |
+/* |
+** Compare the term in buffer zLhs (size in bytes nLhs) with that in |
+** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of |
+** the other, it is considered to be smaller than the other. |
+** |
+** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve |
+** if it is greater. |
+*/ |
+static int fts3TermCmp( |
+ const char *zLhs, int nLhs, /* LHS of comparison */ |
+ const char *zRhs, int nRhs /* RHS of comparison */ |
+){ |
+ int nCmp = MIN(nLhs, nRhs); |
+ int res; |
+ |
+ res = memcmp(zLhs, zRhs, nCmp); |
+ if( res==0 ) res = nLhs - nRhs; |
+ |
+ return res; |
+} |
+ |
+ |
+/* |
+** Query to see if the entry in the %_segments table with blockid iEnd is |
+** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before |
+** returning. Otherwise, set *pbRes to 0. |
+** |
+** Or, if an error occurs while querying the database, return an SQLite |
+** error code. The final value of *pbRes is undefined in this case. |
+** |
+** This is used to test if a segment is an "appendable" segment. If it |
+** is, then a NULL entry has been inserted into the %_segments table |
+** with blockid %_segdir.end_block. |
+*/ |
+static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ |
+ int bRes = 0; /* Result to set *pbRes to */ |
+ sqlite3_stmt *pCheck = 0; /* Statement to query database with */ |
+ int rc; /* Return code */ |
+ |
+ rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pCheck, 1, iEnd); |
+ if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; |
+ rc = sqlite3_reset(pCheck); |
+ } |
+ |
+ *pbRes = bRes; |
+ return rc; |
+} |
+ |
+/* |
+** This function is called when initializing an incremental-merge operation. |
+** It checks if the existing segment with index value iIdx at absolute level |
+** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the |
+** merge-writer object *pWriter is initialized to write to it. |
+** |
+** An existing segment can be appended to by an incremental merge if: |
+** |
+** * It was initially created as an appendable segment (with all required |
+** space pre-allocated), and |
+** |
+** * The first key read from the input (arguments zKey and nKey) is |
+** greater than the largest key currently stored in the potential |
+** output segment. |
+*/ |
+static int fts3IncrmergeLoad( |
+ Fts3Table *p, /* Fts3 table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ |
+ int iIdx, /* Index of candidate output segment */ |
+ const char *zKey, /* First key to write */ |
+ int nKey, /* Number of bytes in nKey */ |
+ IncrmergeWriter *pWriter /* Populate this object */ |
+){ |
+ int rc; /* Return code */ |
+ sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ |
+ |
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ |
+ sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ |
+ sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ |
+ const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ |
+ int nRoot = 0; /* Size of aRoot[] in bytes */ |
+ int rc2; /* Return code from sqlite3_reset() */ |
+ int bAppendable = 0; /* Set to true if segment is appendable */ |
+ |
+ /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ |
+ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); |
+ sqlite3_bind_int(pSelect, 2, iIdx); |
+ if( sqlite3_step(pSelect)==SQLITE_ROW ){ |
+ iStart = sqlite3_column_int64(pSelect, 1); |
+ iLeafEnd = sqlite3_column_int64(pSelect, 2); |
+ fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); |
+ if( pWriter->nLeafData<0 ){ |
+ pWriter->nLeafData = pWriter->nLeafData * -1; |
+ } |
+ pWriter->bNoLeafData = (pWriter->nLeafData==0); |
+ nRoot = sqlite3_column_bytes(pSelect, 4); |
+ aRoot = sqlite3_column_blob(pSelect, 4); |
+ }else{ |
+ return sqlite3_reset(pSelect); |
+ } |
+ |
+ /* Check for the zero-length marker in the %_segments table */ |
+ rc = fts3IsAppendable(p, iEnd, &bAppendable); |
+ |
+ /* Check that zKey/nKey is larger than the largest key the candidate */ |
+ if( rc==SQLITE_OK && bAppendable ){ |
+ char *aLeaf = 0; |
+ int nLeaf = 0; |
+ |
+ rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); |
+ if( rc==SQLITE_OK ){ |
+ NodeReader reader; |
+ for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); |
+ rc==SQLITE_OK && reader.aNode; |
+ rc = nodeReaderNext(&reader) |
+ ){ |
+ assert( reader.aNode ); |
+ } |
+ if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ |
+ bAppendable = 0; |
+ } |
+ nodeReaderRelease(&reader); |
+ } |
+ sqlite3_free(aLeaf); |
+ } |
+ |
+ if( rc==SQLITE_OK && bAppendable ){ |
+ /* It is possible to append to this segment. Set up the IncrmergeWriter |
+ ** object to do so. */ |
+ int i; |
+ int nHeight = (int)aRoot[0]; |
+ NodeWriter *pNode; |
+ |
+ pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; |
+ pWriter->iStart = iStart; |
+ pWriter->iEnd = iEnd; |
+ pWriter->iAbsLevel = iAbsLevel; |
+ pWriter->iIdx = iIdx; |
+ |
+ for(i=nHeight+1; i<FTS_MAX_APPENDABLE_HEIGHT; i++){ |
+ pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; |
+ } |
+ |
+ pNode = &pWriter->aNodeWriter[nHeight]; |
+ pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; |
+ blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); |
+ if( rc==SQLITE_OK ){ |
+ memcpy(pNode->block.a, aRoot, nRoot); |
+ pNode->block.n = nRoot; |
+ } |
+ |
+ for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ |
+ NodeReader reader; |
+ pNode = &pWriter->aNodeWriter[i]; |
+ |
+ rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); |
+ while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); |
+ blobGrowBuffer(&pNode->key, reader.term.n, &rc); |
+ if( rc==SQLITE_OK ){ |
+ memcpy(pNode->key.a, reader.term.a, reader.term.n); |
+ pNode->key.n = reader.term.n; |
+ if( i>0 ){ |
+ char *aBlock = 0; |
+ int nBlock = 0; |
+ pNode = &pWriter->aNodeWriter[i-1]; |
+ pNode->iBlock = reader.iChild; |
+ rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); |
+ blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); |
+ if( rc==SQLITE_OK ){ |
+ memcpy(pNode->block.a, aBlock, nBlock); |
+ pNode->block.n = nBlock; |
+ } |
+ sqlite3_free(aBlock); |
+ } |
+ } |
+ nodeReaderRelease(&reader); |
+ } |
+ } |
+ |
+ rc2 = sqlite3_reset(pSelect); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Determine the largest segment index value that exists within absolute |
+** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus |
+** one before returning SQLITE_OK. Or, if there are no segments at all |
+** within level iAbsLevel, set *piIdx to zero. |
+** |
+** If an error occurs, return an SQLite error code. The final value of |
+** *piIdx is undefined in this case. |
+*/ |
+static int fts3IncrmergeOutputIdx( |
+ Fts3Table *p, /* FTS Table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ |
+ int *piIdx /* OUT: Next free index at iAbsLevel+1 */ |
+){ |
+ int rc; |
+ sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ |
+ |
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); |
+ sqlite3_step(pOutputIdx); |
+ *piIdx = sqlite3_column_int(pOutputIdx, 0); |
+ rc = sqlite3_reset(pOutputIdx); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Allocate an appendable output segment on absolute level iAbsLevel+1 |
+** with idx value iIdx. |
+** |
+** In the %_segdir table, a segment is defined by the values in three |
+** columns: |
+** |
+** start_block |
+** leaves_end_block |
+** end_block |
+** |
+** When an appendable segment is allocated, it is estimated that the |
+** maximum number of leaf blocks that may be required is the sum of the |
+** number of leaf blocks consumed by the input segments, plus the number |
+** of input segments, multiplied by two. This value is stored in stack |
+** variable nLeafEst. |
+** |
+** A total of 16*nLeafEst blocks are allocated when an appendable segment |
+** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous |
+** array of leaf nodes starts at the first block allocated. The array |
+** of interior nodes that are parents of the leaf nodes start at block |
+** (start_block + (1 + end_block - start_block) / 16). And so on. |
+** |
+** In the actual code below, the value "16" is replaced with the |
+** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. |
+*/ |
+static int fts3IncrmergeWriter( |
+ Fts3Table *p, /* Fts3 table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ |
+ int iIdx, /* Index of new output segment */ |
+ Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ |
+ IncrmergeWriter *pWriter /* Populate this object */ |
+){ |
+ int rc; /* Return Code */ |
+ int i; /* Iterator variable */ |
+ int nLeafEst = 0; /* Blocks allocated for leaf nodes */ |
+ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ |
+ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ |
+ |
+ /* Calculate nLeafEst. */ |
+ rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); |
+ sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); |
+ if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ |
+ nLeafEst = sqlite3_column_int(pLeafEst, 0); |
+ } |
+ rc = sqlite3_reset(pLeafEst); |
+ } |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ /* Calculate the first block to use in the output segment */ |
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); |
+ if( rc==SQLITE_OK ){ |
+ if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ |
+ pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); |
+ pWriter->iEnd = pWriter->iStart - 1; |
+ pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; |
+ } |
+ rc = sqlite3_reset(pFirstBlock); |
+ } |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ /* Insert the marker in the %_segments table to make sure nobody tries |
+ ** to steal the space just allocated. This is also used to identify |
+ ** appendable segments. */ |
+ rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); |
+ if( rc!=SQLITE_OK ) return rc; |
+ |
+ pWriter->iAbsLevel = iAbsLevel; |
+ pWriter->nLeafEst = nLeafEst; |
+ pWriter->iIdx = iIdx; |
+ |
+ /* Set up the array of NodeWriter objects */ |
+ for(i=0; i<FTS_MAX_APPENDABLE_HEIGHT; i++){ |
+ pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Remove an entry from the %_segdir table. This involves running the |
+** following two statements: |
+** |
+** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx |
+** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx |
+** |
+** The DELETE statement removes the specific %_segdir level. The UPDATE |
+** statement ensures that the remaining segments have contiguously allocated |
+** idx values. |
+*/ |
+static int fts3RemoveSegdirEntry( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ |
+ int iIdx /* Index of %_segdir entry to delete */ |
+){ |
+ int rc; /* Return code */ |
+ sqlite3_stmt *pDelete = 0; /* DELETE statement */ |
+ |
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pDelete, 1, iAbsLevel); |
+ sqlite3_bind_int(pDelete, 2, iIdx); |
+ sqlite3_step(pDelete); |
+ rc = sqlite3_reset(pDelete); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** One or more segments have just been removed from absolute level iAbsLevel. |
+** Update the 'idx' values of the remaining segments in the level so that |
+** the idx values are a contiguous sequence starting from 0. |
+*/ |
+static int fts3RepackSegdirLevel( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ sqlite3_int64 iAbsLevel /* Absolute level to repack */ |
+){ |
+ int rc; /* Return code */ |
+ int *aIdx = 0; /* Array of remaining idx values */ |
+ int nIdx = 0; /* Valid entries in aIdx[] */ |
+ int nAlloc = 0; /* Allocated size of aIdx[] */ |
+ int i; /* Iterator variable */ |
+ sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ |
+ sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ |
+ |
+ rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ sqlite3_bind_int64(pSelect, 1, iAbsLevel); |
+ while( SQLITE_ROW==sqlite3_step(pSelect) ){ |
+ if( nIdx>=nAlloc ){ |
+ int *aNew; |
+ nAlloc += 16; |
+ aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); |
+ if( !aNew ){ |
+ rc = SQLITE_NOMEM; |
+ break; |
+ } |
+ aIdx = aNew; |
+ } |
+ aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); |
+ } |
+ rc2 = sqlite3_reset(pSelect); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pUpdate, 2, iAbsLevel); |
+ } |
+ |
+ assert( p->bIgnoreSavepoint==0 ); |
+ p->bIgnoreSavepoint = 1; |
+ for(i=0; rc==SQLITE_OK && i<nIdx; i++){ |
+ if( aIdx[i]!=i ){ |
+ sqlite3_bind_int(pUpdate, 3, aIdx[i]); |
+ sqlite3_bind_int(pUpdate, 1, i); |
+ sqlite3_step(pUpdate); |
+ rc = sqlite3_reset(pUpdate); |
+ } |
+ } |
+ p->bIgnoreSavepoint = 0; |
+ |
+ sqlite3_free(aIdx); |
+ return rc; |
+} |
+ |
+static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ |
+ pNode->a[0] = (char)iHeight; |
+ if( iChild ){ |
+ assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); |
+ pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); |
+ }else{ |
+ assert( pNode->nAlloc>=1 ); |
+ pNode->n = 1; |
+ } |
+} |
+ |
+/* |
+** The first two arguments are a pointer to and the size of a segment b-tree |
+** node. The node may be a leaf or an internal node. |
+** |
+** This function creates a new node image in blob object *pNew by copying |
+** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) |
+** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. |
+*/ |
+static int fts3TruncateNode( |
+ const char *aNode, /* Current node image */ |
+ int nNode, /* Size of aNode in bytes */ |
+ Blob *pNew, /* OUT: Write new node image here */ |
+ const char *zTerm, /* Omit all terms smaller than this */ |
+ int nTerm, /* Size of zTerm in bytes */ |
+ sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ |
+){ |
+ NodeReader reader; /* Reader object */ |
+ Blob prev = {0, 0, 0}; /* Previous term written to new node */ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ |
+ |
+ /* Allocate required output space */ |
+ blobGrowBuffer(pNew, nNode, &rc); |
+ if( rc!=SQLITE_OK ) return rc; |
+ pNew->n = 0; |
+ |
+ /* Populate new node buffer */ |
+ for(rc = nodeReaderInit(&reader, aNode, nNode); |
+ rc==SQLITE_OK && reader.aNode; |
+ rc = nodeReaderNext(&reader) |
+ ){ |
+ if( pNew->n==0 ){ |
+ int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); |
+ if( res<0 || (bLeaf==0 && res==0) ) continue; |
+ fts3StartNode(pNew, (int)aNode[0], reader.iChild); |
+ *piBlock = reader.iChild; |
+ } |
+ rc = fts3AppendToNode( |
+ pNew, &prev, reader.term.a, reader.term.n, |
+ reader.aDoclist, reader.nDoclist |
+ ); |
+ if( rc!=SQLITE_OK ) break; |
+ } |
+ if( pNew->n==0 ){ |
+ fts3StartNode(pNew, (int)aNode[0], reader.iChild); |
+ *piBlock = reader.iChild; |
+ } |
+ assert( pNew->n<=pNew->nAlloc ); |
+ |
+ nodeReaderRelease(&reader); |
+ sqlite3_free(prev.a); |
+ return rc; |
+} |
+ |
+/* |
+** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute |
+** level iAbsLevel. This may involve deleting entries from the %_segments |
+** table, and modifying existing entries in both the %_segments and %_segdir |
+** tables. |
+** |
+** SQLITE_OK is returned if the segment is updated successfully. Or an |
+** SQLite error code otherwise. |
+*/ |
+static int fts3TruncateSegment( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ |
+ int iIdx, /* Index within level of segment to modify */ |
+ const char *zTerm, /* Remove terms smaller than this */ |
+ int nTerm /* Number of bytes in buffer zTerm */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ Blob root = {0,0,0}; /* New root page image */ |
+ Blob block = {0,0,0}; /* Buffer used for any other block */ |
+ sqlite3_int64 iBlock = 0; /* Block id */ |
+ sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ |
+ sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ |
+ sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ |
+ |
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; /* sqlite3_reset() return code */ |
+ sqlite3_bind_int64(pFetch, 1, iAbsLevel); |
+ sqlite3_bind_int(pFetch, 2, iIdx); |
+ if( SQLITE_ROW==sqlite3_step(pFetch) ){ |
+ const char *aRoot = sqlite3_column_blob(pFetch, 4); |
+ int nRoot = sqlite3_column_bytes(pFetch, 4); |
+ iOldStart = sqlite3_column_int64(pFetch, 1); |
+ rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); |
+ } |
+ rc2 = sqlite3_reset(pFetch); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ while( rc==SQLITE_OK && iBlock ){ |
+ char *aBlock = 0; |
+ int nBlock = 0; |
+ iNewStart = iBlock; |
+ |
+ rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3WriteSegment(p, iNewStart, block.a, block.n); |
+ } |
+ sqlite3_free(aBlock); |
+ } |
+ |
+ /* Variable iNewStart now contains the first valid leaf node. */ |
+ if( rc==SQLITE_OK && iNewStart ){ |
+ sqlite3_stmt *pDel = 0; |
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pDel, 1, iOldStart); |
+ sqlite3_bind_int64(pDel, 2, iNewStart-1); |
+ sqlite3_step(pDel); |
+ rc = sqlite3_reset(pDel); |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_stmt *pChomp = 0; |
+ rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pChomp, 1, iNewStart); |
+ sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); |
+ sqlite3_bind_int64(pChomp, 3, iAbsLevel); |
+ sqlite3_bind_int(pChomp, 4, iIdx); |
+ sqlite3_step(pChomp); |
+ rc = sqlite3_reset(pChomp); |
+ } |
+ } |
+ |
+ sqlite3_free(root.a); |
+ sqlite3_free(block.a); |
+ return rc; |
+} |
+ |
+/* |
+** This function is called after an incrmental-merge operation has run to |
+** merge (or partially merge) two or more segments from absolute level |
+** iAbsLevel. |
+** |
+** Each input segment is either removed from the db completely (if all of |
+** its data was copied to the output segment by the incrmerge operation) |
+** or modified in place so that it no longer contains those entries that |
+** have been duplicated in the output segment. |
+*/ |
+static int fts3IncrmergeChomp( |
+ Fts3Table *p, /* FTS table handle */ |
+ sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ |
+ Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ |
+ int *pnRem /* Number of segments not deleted */ |
+){ |
+ int i; |
+ int nRem = 0; |
+ int rc = SQLITE_OK; |
+ |
+ for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ |
+ Fts3SegReader *pSeg = 0; |
+ int j; |
+ |
+ /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding |
+ ** somewhere in the pCsr->apSegment[] array. */ |
+ for(j=0; ALWAYS(j<pCsr->nSegment); j++){ |
+ pSeg = pCsr->apSegment[j]; |
+ if( pSeg->iIdx==i ) break; |
+ } |
+ assert( j<pCsr->nSegment && pSeg->iIdx==i ); |
+ |
+ if( pSeg->aNode==0 ){ |
+ /* Seg-reader is at EOF. Remove the entire input segment. */ |
+ rc = fts3DeleteSegment(p, pSeg); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); |
+ } |
+ *pnRem = 0; |
+ }else{ |
+ /* The incremental merge did not copy all the data from this |
+ ** segment to the upper level. The segment is modified in place |
+ ** so that it contains no keys smaller than zTerm/nTerm. */ |
+ const char *zTerm = pSeg->zTerm; |
+ int nTerm = pSeg->nTerm; |
+ rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); |
+ nRem++; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ |
+ rc = fts3RepackSegdirLevel(p, iAbsLevel); |
+ } |
+ |
+ *pnRem = nRem; |
+ return rc; |
+} |
+ |
+/* |
+** Store an incr-merge hint in the database. |
+*/ |
+static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ |
+ sqlite3_stmt *pReplace = 0; |
+ int rc; /* Return code */ |
+ |
+ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); |
+ sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); |
+ sqlite3_step(pReplace); |
+ rc = sqlite3_reset(pReplace); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Load an incr-merge hint from the database. The incr-merge hint, if one |
+** exists, is stored in the rowid==1 row of the %_stat table. |
+** |
+** If successful, populate blob *pHint with the value read from the %_stat |
+** table and return SQLITE_OK. Otherwise, if an error occurs, return an |
+** SQLite error code. |
+*/ |
+static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ |
+ sqlite3_stmt *pSelect = 0; |
+ int rc; |
+ |
+ pHint->n = 0; |
+ rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); |
+ if( SQLITE_ROW==sqlite3_step(pSelect) ){ |
+ const char *aHint = sqlite3_column_blob(pSelect, 0); |
+ int nHint = sqlite3_column_bytes(pSelect, 0); |
+ if( aHint ){ |
+ blobGrowBuffer(pHint, nHint, &rc); |
+ if( rc==SQLITE_OK ){ |
+ memcpy(pHint->a, aHint, nHint); |
+ pHint->n = nHint; |
+ } |
+ } |
+ } |
+ rc2 = sqlite3_reset(pSelect); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op. |
+** Otherwise, append an entry to the hint stored in blob *pHint. Each entry |
+** consists of two varints, the absolute level number of the input segments |
+** and the number of input segments. |
+** |
+** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, |
+** set *pRc to an SQLite error code before returning. |
+*/ |
+static void fts3IncrmergeHintPush( |
+ Blob *pHint, /* Hint blob to append to */ |
+ i64 iAbsLevel, /* First varint to store in hint */ |
+ int nInput, /* Second varint to store in hint */ |
+ int *pRc /* IN/OUT: Error code */ |
+){ |
+ blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); |
+ if( *pRc==SQLITE_OK ){ |
+ pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); |
+ pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); |
+ } |
+} |
+ |
+/* |
+** Read the last entry (most recently pushed) from the hint blob *pHint |
+** and then remove the entry. Write the two values read to *piAbsLevel and |
+** *pnInput before returning. |
+** |
+** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does |
+** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. |
+*/ |
+static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ |
+ const int nHint = pHint->n; |
+ int i; |
+ |
+ i = pHint->n-2; |
+ while( i>0 && (pHint->a[i-1] & 0x80) ) i--; |
+ while( i>0 && (pHint->a[i-1] & 0x80) ) i--; |
+ |
+ pHint->n = i; |
+ i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); |
+ i += fts3GetVarint32(&pHint->a[i], pnInput); |
+ if( i!=nHint ) return FTS_CORRUPT_VTAB; |
+ |
+ return SQLITE_OK; |
+} |
+ |
+ |
+/* |
+** Attempt an incremental merge that writes nMerge leaf blocks. |
+** |
+** Incremental merges happen nMin segments at a time. The segments |
+** to be merged are the nMin oldest segments (the ones with the smallest |
+** values for the _segdir.idx field) in the highest level that contains |
+** at least nMin segments. Multiple merges might occur in an attempt to |
+** write the quota of nMerge leaf blocks. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ |
+ int rc; /* Return code */ |
+ int nRem = nMerge; /* Number of leaf pages yet to be written */ |
+ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ |
+ Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ |
+ IncrmergeWriter *pWriter; /* Writer object */ |
+ int nSeg = 0; /* Number of input segments */ |
+ sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ |
+ Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ |
+ int bDirtyHint = 0; /* True if blob 'hint' has been modified */ |
+ |
+ /* Allocate space for the cursor, filter and writer objects */ |
+ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); |
+ pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); |
+ if( !pWriter ) return SQLITE_NOMEM; |
+ pFilter = (Fts3SegFilter *)&pWriter[1]; |
+ pCsr = (Fts3MultiSegReader *)&pFilter[1]; |
+ |
+ rc = fts3IncrmergeHintLoad(p, &hint); |
+ while( rc==SQLITE_OK && nRem>0 ){ |
+ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; |
+ sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ |
+ int bUseHint = 0; /* True if attempting to append */ |
+ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ |
+ |
+ /* Search the %_segdir table for the absolute level with the smallest |
+ ** relative level number that contains at least nMin segments, if any. |
+ ** If one is found, set iAbsLevel to the absolute level number and |
+ ** nSeg to nMin. If no level with at least nMin segments can be found, |
+ ** set nSeg to -1. |
+ */ |
+ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); |
+ sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); |
+ if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ |
+ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); |
+ nSeg = sqlite3_column_int(pFindLevel, 1); |
+ assert( nSeg>=2 ); |
+ }else{ |
+ nSeg = -1; |
+ } |
+ rc = sqlite3_reset(pFindLevel); |
+ |
+ /* If the hint read from the %_stat table is not empty, check if the |
+ ** last entry in it specifies a relative level smaller than or equal |
+ ** to the level identified by the block above (if any). If so, this |
+ ** iteration of the loop will work on merging at the hinted level. |
+ */ |
+ if( rc==SQLITE_OK && hint.n ){ |
+ int nHint = hint.n; |
+ sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ |
+ int nHintSeg = 0; /* Hint number of segments */ |
+ |
+ rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); |
+ if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ |
+ iAbsLevel = iHintAbsLevel; |
+ nSeg = nHintSeg; |
+ bUseHint = 1; |
+ bDirtyHint = 1; |
+ }else{ |
+ /* This undoes the effect of the HintPop() above - so that no entry |
+ ** is removed from the hint blob. */ |
+ hint.n = nHint; |
+ } |
+ } |
+ |
+ /* If nSeg is less that zero, then there is no level with at least |
+ ** nMin segments and no hint in the %_stat table. No work to do. |
+ ** Exit early in this case. */ |
+ if( nSeg<0 ) break; |
+ |
+ /* Open a cursor to iterate through the contents of the oldest nSeg |
+ ** indexes of absolute level iAbsLevel. If this cursor is opened using |
+ ** the 'hint' parameters, it is possible that there are less than nSeg |
+ ** segments available in level iAbsLevel. In this case, no work is |
+ ** done on iAbsLevel - fall through to the next iteration of the loop |
+ ** to start work on some other level. */ |
+ memset(pWriter, 0, nAlloc); |
+ pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); |
+ assert( bUseHint==1 || bUseHint==0 ); |
+ if( iIdx==0 || (bUseHint && iIdx==1) ){ |
+ int bIgnore = 0; |
+ rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); |
+ if( bIgnore ){ |
+ pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; |
+ } |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); |
+ } |
+ if( SQLITE_OK==rc && pCsr->nSegment==nSeg |
+ && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) |
+ && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) |
+ ){ |
+ if( bUseHint && iIdx>0 ){ |
+ const char *zKey = pCsr->zTerm; |
+ int nKey = pCsr->nTerm; |
+ rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); |
+ }else{ |
+ rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); |
+ } |
+ |
+ if( rc==SQLITE_OK && pWriter->nLeafEst ){ |
+ fts3LogMerge(nSeg, iAbsLevel); |
+ do { |
+ rc = fts3IncrmergeAppend(p, pWriter, pCsr); |
+ if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); |
+ if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; |
+ }while( rc==SQLITE_ROW ); |
+ |
+ /* Update or delete the input segments */ |
+ if( rc==SQLITE_OK ){ |
+ nRem -= (1 + pWriter->nWork); |
+ rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); |
+ if( nSeg!=0 ){ |
+ bDirtyHint = 1; |
+ fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); |
+ } |
+ } |
+ } |
+ |
+ if( nSeg!=0 ){ |
+ pWriter->nLeafData = pWriter->nLeafData * -1; |
+ } |
+ fts3IncrmergeRelease(p, pWriter, &rc); |
+ if( nSeg==0 && pWriter->bNoLeafData==0 ){ |
+ fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); |
+ } |
+ } |
+ |
+ sqlite3Fts3SegReaderFinish(pCsr); |
+ } |
+ |
+ /* Write the hint values into the %_stat table for the next incr-merger */ |
+ if( bDirtyHint && rc==SQLITE_OK ){ |
+ rc = fts3IncrmergeHintStore(p, &hint); |
+ } |
+ |
+ sqlite3_free(pWriter); |
+ sqlite3_free(hint.a); |
+ return rc; |
+} |
+ |
+/* |
+** Convert the text beginning at *pz into an integer and return |
+** its value. Advance *pz to point to the first character past |
+** the integer. |
+*/ |
+static int fts3Getint(const char **pz){ |
+ const char *z = *pz; |
+ int i = 0; |
+ while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; |
+ *pz = z; |
+ return i; |
+} |
+ |
+/* |
+** Process statements of the form: |
+** |
+** INSERT INTO table(table) VALUES('merge=A,B'); |
+** |
+** A and B are integers that decode to be the number of leaf pages |
+** written for the merge, and the minimum number of segments on a level |
+** before it will be selected for a merge, respectively. |
+*/ |
+static int fts3DoIncrmerge( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ const char *zParam /* Nul-terminated string containing "A,B" */ |
+){ |
+ int rc; |
+ int nMin = (FTS3_MERGE_COUNT / 2); |
+ int nMerge = 0; |
+ const char *z = zParam; |
+ |
+ /* Read the first integer value */ |
+ nMerge = fts3Getint(&z); |
+ |
+ /* If the first integer value is followed by a ',', read the second |
+ ** integer value. */ |
+ if( z[0]==',' && z[1]!='\0' ){ |
+ z++; |
+ nMin = fts3Getint(&z); |
+ } |
+ |
+ if( z[0]!='\0' || nMin<2 ){ |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ rc = SQLITE_OK; |
+ if( !p->bHasStat ){ |
+ assert( p->bFts4==0 ); |
+ sqlite3Fts3CreateStatTable(&rc, p); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); |
+ } |
+ sqlite3Fts3SegmentsClose(p); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Process statements of the form: |
+** |
+** INSERT INTO table(table) VALUES('automerge=X'); |
+** |
+** where X is an integer. X==0 means to turn automerge off. X!=0 means |
+** turn it on. The setting is persistent. |
+*/ |
+static int fts3DoAutoincrmerge( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ const char *zParam /* Nul-terminated string containing boolean */ |
+){ |
+ int rc = SQLITE_OK; |
+ sqlite3_stmt *pStmt = 0; |
+ p->nAutoincrmerge = fts3Getint(&zParam); |
+ if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ |
+ p->nAutoincrmerge = 8; |
+ } |
+ if( !p->bHasStat ){ |
+ assert( p->bFts4==0 ); |
+ sqlite3Fts3CreateStatTable(&rc, p); |
+ if( rc ) return rc; |
+ } |
+ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); |
+ if( rc ) return rc; |
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); |
+ sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); |
+ sqlite3_step(pStmt); |
+ rc = sqlite3_reset(pStmt); |
+ return rc; |
+} |
+ |
+/* |
+** Return a 64-bit checksum for the FTS index entry specified by the |
+** arguments to this function. |
+*/ |
+static u64 fts3ChecksumEntry( |
+ const char *zTerm, /* Pointer to buffer containing term */ |
+ int nTerm, /* Size of zTerm in bytes */ |
+ int iLangid, /* Language id for current row */ |
+ int iIndex, /* Index (0..Fts3Table.nIndex-1) */ |
+ i64 iDocid, /* Docid for current row. */ |
+ int iCol, /* Column number */ |
+ int iPos /* Position */ |
+){ |
+ int i; |
+ u64 ret = (u64)iDocid; |
+ |
+ ret += (ret<<3) + iLangid; |
+ ret += (ret<<3) + iIndex; |
+ ret += (ret<<3) + iCol; |
+ ret += (ret<<3) + iPos; |
+ for(i=0; i<nTerm; i++) ret += (ret<<3) + zTerm[i]; |
+ |
+ return ret; |
+} |
+ |
+/* |
+** Return a checksum of all entries in the FTS index that correspond to |
+** language id iLangid. The checksum is calculated by XORing the checksums |
+** of each individual entry (see fts3ChecksumEntry()) together. |
+** |
+** If successful, the checksum value is returned and *pRc set to SQLITE_OK. |
+** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The |
+** return value is undefined in this case. |
+*/ |
+static u64 fts3ChecksumIndex( |
+ Fts3Table *p, /* FTS3 table handle */ |
+ int iLangid, /* Language id to return cksum for */ |
+ int iIndex, /* Index to cksum (0..p->nIndex-1) */ |
+ int *pRc /* OUT: Return code */ |
+){ |
+ Fts3SegFilter filter; |
+ Fts3MultiSegReader csr; |
+ int rc; |
+ u64 cksum = 0; |
+ |
+ assert( *pRc==SQLITE_OK ); |
+ |
+ memset(&filter, 0, sizeof(filter)); |
+ memset(&csr, 0, sizeof(csr)); |
+ filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; |
+ filter.flags |= FTS3_SEGMENT_SCAN; |
+ |
+ rc = sqlite3Fts3SegReaderCursor( |
+ p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr |
+ ); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ |
+ char *pCsr = csr.aDoclist; |
+ char *pEnd = &pCsr[csr.nDoclist]; |
+ |
+ i64 iDocid = 0; |
+ i64 iCol = 0; |
+ i64 iPos = 0; |
+ |
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); |
+ while( pCsr<pEnd ){ |
+ i64 iVal = 0; |
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iVal); |
+ if( pCsr<pEnd ){ |
+ if( iVal==0 || iVal==1 ){ |
+ iCol = 0; |
+ iPos = 0; |
+ if( iVal ){ |
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); |
+ }else{ |
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iVal); |
+ iDocid += iVal; |
+ } |
+ }else{ |
+ iPos += (iVal - 2); |
+ cksum = cksum ^ fts3ChecksumEntry( |
+ csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, |
+ (int)iCol, (int)iPos |
+ ); |
+ } |
+ } |
+ } |
+ } |
+ } |
+ sqlite3Fts3SegReaderFinish(&csr); |
+ |
+ *pRc = rc; |
+ return cksum; |
+} |
+ |
+/* |
+** Check if the contents of the FTS index match the current contents of the |
+** content table. If no error occurs and the contents do match, set *pbOk |
+** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk |
+** to false before returning. |
+** |
+** If an error occurs (e.g. an OOM or IO error), return an SQLite error |
+** code. The final value of *pbOk is undefined in this case. |
+*/ |
+static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ u64 cksum1 = 0; /* Checksum based on FTS index contents */ |
+ u64 cksum2 = 0; /* Checksum based on %_content contents */ |
+ sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ |
+ |
+ /* This block calculates the checksum according to the FTS index. */ |
+ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); |
+ sqlite3_bind_int(pAllLangid, 2, p->nIndex); |
+ while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ |
+ int iLangid = sqlite3_column_int(pAllLangid, 0); |
+ int i; |
+ for(i=0; i<p->nIndex; i++){ |
+ cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); |
+ } |
+ } |
+ rc2 = sqlite3_reset(pAllLangid); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ /* This block calculates the checksum according to the %_content table */ |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; |
+ sqlite3_stmt *pStmt = 0; |
+ char *zSql; |
+ |
+ zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); |
+ if( !zSql ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); |
+ sqlite3_free(zSql); |
+ } |
+ |
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ i64 iDocid = sqlite3_column_int64(pStmt, 0); |
+ int iLang = langidFromSelect(p, pStmt); |
+ int iCol; |
+ |
+ for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){ |
+ if( p->abNotindexed[iCol]==0 ){ |
+ const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); |
+ int nText = sqlite3_column_bytes(pStmt, iCol+1); |
+ sqlite3_tokenizer_cursor *pT = 0; |
+ |
+ rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); |
+ while( rc==SQLITE_OK ){ |
+ char const *zToken; /* Buffer containing token */ |
+ int nToken = 0; /* Number of bytes in token */ |
+ int iDum1 = 0, iDum2 = 0; /* Dummy variables */ |
+ int iPos = 0; /* Position of token in zText */ |
+ |
+ rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); |
+ if( rc==SQLITE_OK ){ |
+ int i; |
+ cksum2 = cksum2 ^ fts3ChecksumEntry( |
+ zToken, nToken, iLang, 0, iDocid, iCol, iPos |
+ ); |
+ for(i=1; i<p->nIndex; i++){ |
+ if( p->aIndex[i].nPrefix<=nToken ){ |
+ cksum2 = cksum2 ^ fts3ChecksumEntry( |
+ zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos |
+ ); |
+ } |
+ } |
+ } |
+ } |
+ if( pT ) pModule->xClose(pT); |
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
+ } |
+ } |
+ } |
+ |
+ sqlite3_finalize(pStmt); |
+ } |
+ |
+ *pbOk = (cksum1==cksum2); |
+ return rc; |
+} |
+ |
+/* |
+** Run the integrity-check. If no error occurs and the current contents of |
+** the FTS index are correct, return SQLITE_OK. Or, if the contents of the |
+** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. |
+** |
+** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite |
+** error code. |
+** |
+** The integrity-check works as follows. For each token and indexed token |
+** prefix in the document set, a 64-bit checksum is calculated (by code |
+** in fts3ChecksumEntry()) based on the following: |
+** |
+** + The index number (0 for the main index, 1 for the first prefix |
+** index etc.), |
+** + The token (or token prefix) text itself, |
+** + The language-id of the row it appears in, |
+** + The docid of the row it appears in, |
+** + The column it appears in, and |
+** + The tokens position within that column. |
+** |
+** The checksums for all entries in the index are XORed together to create |
+** a single checksum for the entire index. |
+** |
+** The integrity-check code calculates the same checksum in two ways: |
+** |
+** 1. By scanning the contents of the FTS index, and |
+** 2. By scanning and tokenizing the content table. |
+** |
+** If the two checksums are identical, the integrity-check is deemed to have |
+** passed. |
+*/ |
+static int fts3DoIntegrityCheck( |
+ Fts3Table *p /* FTS3 table handle */ |
+){ |
+ int rc; |
+ int bOk = 0; |
+ rc = fts3IntegrityCheck(p, &bOk); |
+ if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; |
+ return rc; |
+} |
+ |
+/* |
+** Handle a 'special' INSERT of the form: |
+** |
+** "INSERT INTO tbl(tbl) VALUES(<expr>)" |
+** |
+** Argument pVal contains the result of <expr>. Currently the only |
+** meaningful value to insert is the text 'optimize'. |
+*/ |
+static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ |
+ int rc; /* Return Code */ |
+ const char *zVal = (const char *)sqlite3_value_text(pVal); |
+ int nVal = sqlite3_value_bytes(pVal); |
+ |
+ if( !zVal ){ |
+ return SQLITE_NOMEM; |
+ }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ |
+ rc = fts3DoOptimize(p, 0); |
+ }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ |
+ rc = fts3DoRebuild(p); |
+ }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ |
+ rc = fts3DoIntegrityCheck(p); |
+ }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ |
+ rc = fts3DoIncrmerge(p, &zVal[6]); |
+ }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ |
+ rc = fts3DoAutoincrmerge(p, &zVal[10]); |
+#ifdef SQLITE_TEST |
+ }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ |
+ p->nNodeSize = atoi(&zVal[9]); |
+ rc = SQLITE_OK; |
+ }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ |
+ p->nMaxPendingData = atoi(&zVal[11]); |
+ rc = SQLITE_OK; |
+ }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ |
+ p->bNoIncrDoclist = atoi(&zVal[21]); |
+ rc = SQLITE_OK; |
+#endif |
+ }else{ |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ return rc; |
+} |
+ |
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED |
+/* |
+** Delete all cached deferred doclists. Deferred doclists are cached |
+** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ |
+ Fts3DeferredToken *pDef; |
+ for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ |
+ fts3PendingListDelete(pDef->pList); |
+ pDef->pList = 0; |
+ } |
+} |
+ |
+/* |
+** Free all entries in the pCsr->pDeffered list. Entries are added to |
+** this list using sqlite3Fts3DeferToken(). |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ |
+ Fts3DeferredToken *pDef; |
+ Fts3DeferredToken *pNext; |
+ for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ |
+ pNext = pDef->pNext; |
+ fts3PendingListDelete(pDef->pList); |
+ sqlite3_free(pDef); |
+ } |
+ pCsr->pDeferred = 0; |
+} |
+ |
+/* |
+** Generate deferred-doclists for all tokens in the pCsr->pDeferred list |
+** based on the row that pCsr currently points to. |
+** |
+** A deferred-doclist is like any other doclist with position information |
+** included, except that it only contains entries for a single row of the |
+** table, not for all rows. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ if( pCsr->pDeferred ){ |
+ int i; /* Used to iterate through table columns */ |
+ sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ |
+ Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ |
+ |
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; |
+ sqlite3_tokenizer *pT = p->pTokenizer; |
+ sqlite3_tokenizer_module const *pModule = pT->pModule; |
+ |
+ assert( pCsr->isRequireSeek==0 ); |
+ iDocid = sqlite3_column_int64(pCsr->pStmt, 0); |
+ |
+ for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){ |
+ if( p->abNotindexed[i]==0 ){ |
+ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); |
+ sqlite3_tokenizer_cursor *pTC = 0; |
+ |
+ rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); |
+ while( rc==SQLITE_OK ){ |
+ char const *zToken; /* Buffer containing token */ |
+ int nToken = 0; /* Number of bytes in token */ |
+ int iDum1 = 0, iDum2 = 0; /* Dummy variables */ |
+ int iPos = 0; /* Position of token in zText */ |
+ |
+ rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); |
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ |
+ Fts3PhraseToken *pPT = pDef->pToken; |
+ if( (pDef->iCol>=p->nColumn || pDef->iCol==i) |
+ && (pPT->bFirst==0 || iPos==0) |
+ && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken)) |
+ && (0==memcmp(zToken, pPT->z, pPT->n)) |
+ ){ |
+ fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); |
+ } |
+ } |
+ } |
+ if( pTC ) pModule->xClose(pTC); |
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
+ } |
+ } |
+ |
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ |
+ if( pDef->pList ){ |
+ rc = fts3PendingListAppendVarint(&pDef->pList, 0); |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( |
+ Fts3DeferredToken *p, |
+ char **ppData, |
+ int *pnData |
+){ |
+ char *pRet; |
+ int nSkip; |
+ sqlite3_int64 dummy; |
+ |
+ *ppData = 0; |
+ *pnData = 0; |
+ |
+ if( p->pList==0 ){ |
+ return SQLITE_OK; |
+ } |
+ |
+ pRet = (char *)sqlite3_malloc(p->pList->nData); |
+ if( !pRet ) return SQLITE_NOMEM; |
+ |
+ nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); |
+ *pnData = p->pList->nData - nSkip; |
+ *ppData = pRet; |
+ |
+ memcpy(pRet, &p->pList->aData[nSkip], *pnData); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Add an entry for token pToken to the pCsr->pDeferred list. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3DeferToken( |
+ Fts3Cursor *pCsr, /* Fts3 table cursor */ |
+ Fts3PhraseToken *pToken, /* Token to defer */ |
+ int iCol /* Column that token must appear in (or -1) */ |
+){ |
+ Fts3DeferredToken *pDeferred; |
+ pDeferred = sqlite3_malloc(sizeof(*pDeferred)); |
+ if( !pDeferred ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pDeferred, 0, sizeof(*pDeferred)); |
+ pDeferred->pToken = pToken; |
+ pDeferred->pNext = pCsr->pDeferred; |
+ pDeferred->iCol = iCol; |
+ pCsr->pDeferred = pDeferred; |
+ |
+ assert( pToken->pDeferred==0 ); |
+ pToken->pDeferred = pDeferred; |
+ |
+ return SQLITE_OK; |
+} |
+#endif |
+ |
+/* |
+** SQLite value pRowid contains the rowid of a row that may or may not be |
+** present in the FTS3 table. If it is, delete it and adjust the contents |
+** of subsiduary data structures accordingly. |
+*/ |
+static int fts3DeleteByRowid( |
+ Fts3Table *p, |
+ sqlite3_value *pRowid, |
+ int *pnChng, /* IN/OUT: Decrement if row is deleted */ |
+ u32 *aSzDel |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int bFound = 0; /* True if *pRowid really is in the table */ |
+ |
+ fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); |
+ if( bFound && rc==SQLITE_OK ){ |
+ int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ |
+ rc = fts3IsEmpty(p, pRowid, &isEmpty); |
+ if( rc==SQLITE_OK ){ |
+ if( isEmpty ){ |
+ /* Deleting this row means the whole table is empty. In this case |
+ ** delete the contents of all three tables and throw away any |
+ ** data in the pendingTerms hash table. */ |
+ rc = fts3DeleteAll(p, 1); |
+ *pnChng = 0; |
+ memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); |
+ }else{ |
+ *pnChng = *pnChng - 1; |
+ if( p->zContentTbl==0 ){ |
+ fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); |
+ } |
+ if( p->bHasDocsize ){ |
+ fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); |
+ } |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This function does the work for the xUpdate method of FTS3 virtual |
+** tables. The schema of the virtual table being: |
+** |
+** CREATE TABLE <table name>( |
+** <user columns>, |
+** <table name> HIDDEN, |
+** docid HIDDEN, |
+** <langid> HIDDEN |
+** ); |
+** |
+** |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( |
+ sqlite3_vtab *pVtab, /* FTS3 vtab object */ |
+ int nArg, /* Size of argument array */ |
+ sqlite3_value **apVal, /* Array of arguments */ |
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ |
+){ |
+ Fts3Table *p = (Fts3Table *)pVtab; |
+ int rc = SQLITE_OK; /* Return Code */ |
+ int isRemove = 0; /* True for an UPDATE or DELETE */ |
+ u32 *aSzIns = 0; /* Sizes of inserted documents */ |
+ u32 *aSzDel = 0; /* Sizes of deleted documents */ |
+ int nChng = 0; /* Net change in number of documents */ |
+ int bInsertDone = 0; |
+ |
+ /* At this point it must be known if the %_stat table exists or not. |
+ ** So bHasStat may not be 2. */ |
+ assert( p->bHasStat==0 || p->bHasStat==1 ); |
+ |
+ assert( p->pSegments==0 ); |
+ assert( |
+ nArg==1 /* DELETE operations */ |
+ || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ |
+ ); |
+ |
+ /* Check for a "special" INSERT operation. One of the form: |
+ ** |
+ ** INSERT INTO xyz(xyz) VALUES('command'); |
+ */ |
+ if( nArg>1 |
+ && sqlite3_value_type(apVal[0])==SQLITE_NULL |
+ && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL |
+ ){ |
+ rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); |
+ goto update_out; |
+ } |
+ |
+ if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ |
+ rc = SQLITE_CONSTRAINT; |
+ goto update_out; |
+ } |
+ |
+ /* Allocate space to hold the change in document sizes */ |
+ aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); |
+ if( aSzDel==0 ){ |
+ rc = SQLITE_NOMEM; |
+ goto update_out; |
+ } |
+ aSzIns = &aSzDel[p->nColumn+1]; |
+ memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); |
+ |
+ rc = fts3Writelock(p); |
+ if( rc!=SQLITE_OK ) goto update_out; |
+ |
+ /* If this is an INSERT operation, or an UPDATE that modifies the rowid |
+ ** value, then this operation requires constraint handling. |
+ ** |
+ ** If the on-conflict mode is REPLACE, this means that the existing row |
+ ** should be deleted from the database before inserting the new row. Or, |
+ ** if the on-conflict mode is other than REPLACE, then this method must |
+ ** detect the conflict and return SQLITE_CONSTRAINT before beginning to |
+ ** modify the database file. |
+ */ |
+ if( nArg>1 && p->zContentTbl==0 ){ |
+ /* Find the value object that holds the new rowid value. */ |
+ sqlite3_value *pNewRowid = apVal[3+p->nColumn]; |
+ if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ |
+ pNewRowid = apVal[1]; |
+ } |
+ |
+ if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( |
+ sqlite3_value_type(apVal[0])==SQLITE_NULL |
+ || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) |
+ )){ |
+ /* The new rowid is not NULL (in this case the rowid will be |
+ ** automatically assigned and there is no chance of a conflict), and |
+ ** the statement is either an INSERT or an UPDATE that modifies the |
+ ** rowid column. So if the conflict mode is REPLACE, then delete any |
+ ** existing row with rowid=pNewRowid. |
+ ** |
+ ** Or, if the conflict mode is not REPLACE, insert the new record into |
+ ** the %_content table. If we hit the duplicate rowid constraint (or any |
+ ** other error) while doing so, return immediately. |
+ ** |
+ ** This branch may also run if pNewRowid contains a value that cannot |
+ ** be losslessly converted to an integer. In this case, the eventual |
+ ** call to fts3InsertData() (either just below or further on in this |
+ ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is |
+ ** invoked, it will delete zero rows (since no row will have |
+ ** docid=$pNewRowid if $pNewRowid is not an integer value). |
+ */ |
+ if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ |
+ rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); |
+ }else{ |
+ rc = fts3InsertData(p, apVal, pRowid); |
+ bInsertDone = 1; |
+ } |
+ } |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ goto update_out; |
+ } |
+ |
+ /* If this is a DELETE or UPDATE operation, remove the old record. */ |
+ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ |
+ assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); |
+ rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); |
+ isRemove = 1; |
+ } |
+ |
+ /* If this is an INSERT or UPDATE operation, insert the new record. */ |
+ if( nArg>1 && rc==SQLITE_OK ){ |
+ int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); |
+ if( bInsertDone==0 ){ |
+ rc = fts3InsertData(p, apVal, pRowid); |
+ if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ |
+ rc = FTS_CORRUPT_VTAB; |
+ } |
+ } |
+ if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ |
+ rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ assert( p->iPrevDocid==*pRowid ); |
+ rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); |
+ } |
+ if( p->bHasDocsize ){ |
+ fts3InsertDocsize(&rc, p, aSzIns); |
+ } |
+ nChng++; |
+ } |
+ |
+ if( p->bFts4 ){ |
+ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); |
+ } |
+ |
+ update_out: |
+ sqlite3_free(aSzDel); |
+ sqlite3Fts3SegmentsClose(p); |
+ return rc; |
+} |
+ |
+/* |
+** Flush any data in the pending-terms hash table to disk. If successful, |
+** merge all segments in the database (including the new segment, if |
+** there was any data to flush) into a single segment. |
+*/ |
+SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ |
+ int rc; |
+ rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3DoOptimize(p, 1); |
+ if( rc==SQLITE_OK || rc==SQLITE_DONE ){ |
+ int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); |
+ if( rc2!=SQLITE_OK ) rc = rc2; |
+ }else{ |
+ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); |
+ sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); |
+ } |
+ } |
+ sqlite3Fts3SegmentsClose(p); |
+ return rc; |
+} |
+ |
+#endif |
+ |
+/************** End of fts3_write.c ******************************************/ |
+/************** Begin file fts3_snippet.c ************************************/ |
+/* |
+** 2009 Oct 23 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+*/ |
+ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <string.h> */ |
+/* #include <assert.h> */ |
+ |
+/* |
+** Characters that may appear in the second argument to matchinfo(). |
+*/ |
+#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ |
+#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ |
+#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ |
+#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ |
+#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ |
+#define FTS3_MATCHINFO_LCS 's' /* nCol values */ |
+#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ |
+#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ |
+#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ |
+ |
+/* |
+** The default value for the second argument to matchinfo(). |
+*/ |
+#define FTS3_MATCHINFO_DEFAULT "pcx" |
+ |
+ |
+/* |
+** Used as an fts3ExprIterate() context when loading phrase doclists to |
+** Fts3Expr.aDoclist[]/nDoclist. |
+*/ |
+typedef struct LoadDoclistCtx LoadDoclistCtx; |
+struct LoadDoclistCtx { |
+ Fts3Cursor *pCsr; /* FTS3 Cursor */ |
+ int nPhrase; /* Number of phrases seen so far */ |
+ int nToken; /* Number of tokens seen so far */ |
+}; |
+ |
+/* |
+** The following types are used as part of the implementation of the |
+** fts3BestSnippet() routine. |
+*/ |
+typedef struct SnippetIter SnippetIter; |
+typedef struct SnippetPhrase SnippetPhrase; |
+typedef struct SnippetFragment SnippetFragment; |
+ |
+struct SnippetIter { |
+ Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ |
+ int iCol; /* Extract snippet from this column */ |
+ int nSnippet; /* Requested snippet length (in tokens) */ |
+ int nPhrase; /* Number of phrases in query */ |
+ SnippetPhrase *aPhrase; /* Array of size nPhrase */ |
+ int iCurrent; /* First token of current snippet */ |
+}; |
+ |
+struct SnippetPhrase { |
+ int nToken; /* Number of tokens in phrase */ |
+ char *pList; /* Pointer to start of phrase position list */ |
+ int iHead; /* Next value in position list */ |
+ char *pHead; /* Position list data following iHead */ |
+ int iTail; /* Next value in trailing position list */ |
+ char *pTail; /* Position list data following iTail */ |
+}; |
+ |
+struct SnippetFragment { |
+ int iCol; /* Column snippet is extracted from */ |
+ int iPos; /* Index of first token in snippet */ |
+ u64 covered; /* Mask of query phrases covered */ |
+ u64 hlmask; /* Mask of snippet terms to highlight */ |
+}; |
+ |
+/* |
+** This type is used as an fts3ExprIterate() context object while |
+** accumulating the data returned by the matchinfo() function. |
+*/ |
+typedef struct MatchInfo MatchInfo; |
+struct MatchInfo { |
+ Fts3Cursor *pCursor; /* FTS3 Cursor */ |
+ int nCol; /* Number of columns in table */ |
+ int nPhrase; /* Number of matchable phrases in query */ |
+ sqlite3_int64 nDoc; /* Number of docs in database */ |
+ char flag; |
+ u32 *aMatchinfo; /* Pre-allocated buffer */ |
+}; |
+ |
+/* |
+** An instance of this structure is used to manage a pair of buffers, each |
+** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below |
+** for details. |
+*/ |
+struct MatchinfoBuffer { |
+ u8 aRef[3]; |
+ int nElem; |
+ int bGlobal; /* Set if global data is loaded */ |
+ char *zMatchinfo; |
+ u32 aMatchinfo[1]; |
+}; |
+ |
+ |
+/* |
+** The snippet() and offsets() functions both return text values. An instance |
+** of the following structure is used to accumulate those values while the |
+** functions are running. See fts3StringAppend() for details. |
+*/ |
+typedef struct StrBuffer StrBuffer; |
+struct StrBuffer { |
+ char *z; /* Pointer to buffer containing string */ |
+ int n; /* Length of z in bytes (excl. nul-term) */ |
+ int nAlloc; /* Allocated size of buffer z in bytes */ |
+}; |
+ |
+ |
+/************************************************************************* |
+** Start of MatchinfoBuffer code. |
+*/ |
+ |
+/* |
+** Allocate a two-slot MatchinfoBuffer object. |
+*/ |
+static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ |
+ MatchinfoBuffer *pRet; |
+ int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); |
+ int nStr = (int)strlen(zMatchinfo); |
+ |
+ pRet = sqlite3_malloc(nByte + nStr+1); |
+ if( pRet ){ |
+ memset(pRet, 0, nByte); |
+ pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; |
+ pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); |
+ pRet->nElem = nElem; |
+ pRet->zMatchinfo = ((char*)pRet) + nByte; |
+ memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); |
+ pRet->aRef[0] = 1; |
+ } |
+ |
+ return pRet; |
+} |
+ |
+static void fts3MIBufferFree(void *p){ |
+ MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); |
+ |
+ assert( (u32*)p==&pBuf->aMatchinfo[1] |
+ || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] |
+ ); |
+ if( (u32*)p==&pBuf->aMatchinfo[1] ){ |
+ pBuf->aRef[1] = 0; |
+ }else{ |
+ pBuf->aRef[2] = 0; |
+ } |
+ |
+ if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ |
+ sqlite3_free(pBuf); |
+ } |
+} |
+ |
+static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ |
+ void (*xRet)(void*) = 0; |
+ u32 *aOut = 0; |
+ |
+ if( p->aRef[1]==0 ){ |
+ p->aRef[1] = 1; |
+ aOut = &p->aMatchinfo[1]; |
+ xRet = fts3MIBufferFree; |
+ } |
+ else if( p->aRef[2]==0 ){ |
+ p->aRef[2] = 1; |
+ aOut = &p->aMatchinfo[p->nElem+2]; |
+ xRet = fts3MIBufferFree; |
+ }else{ |
+ aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); |
+ if( aOut ){ |
+ xRet = sqlite3_free; |
+ if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); |
+ } |
+ } |
+ |
+ *paOut = aOut; |
+ return xRet; |
+} |
+ |
+static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ |
+ p->bGlobal = 1; |
+ memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); |
+} |
+ |
+/* |
+** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ |
+ if( p ){ |
+ assert( p->aRef[0]==1 ); |
+ p->aRef[0] = 0; |
+ if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ |
+ sqlite3_free(p); |
+ } |
+ } |
+} |
+ |
+/* |
+** End of MatchinfoBuffer code. |
+*************************************************************************/ |
+ |
+ |
+/* |
+** This function is used to help iterate through a position-list. A position |
+** list is a list of unique integers, sorted from smallest to largest. Each |
+** element of the list is represented by an FTS3 varint that takes the value |
+** of the difference between the current element and the previous one plus |
+** two. For example, to store the position-list: |
+** |
+** 4 9 113 |
+** |
+** the three varints: |
+** |
+** 6 7 106 |
+** |
+** are encoded. |
+** |
+** When this function is called, *pp points to the start of an element of |
+** the list. *piPos contains the value of the previous entry in the list. |
+** After it returns, *piPos contains the value of the next element of the |
+** list and *pp is advanced to the following varint. |
+*/ |
+static void fts3GetDeltaPosition(char **pp, int *piPos){ |
+ int iVal; |
+ *pp += fts3GetVarint32(*pp, &iVal); |
+ *piPos += (iVal-2); |
+} |
+ |
+/* |
+** Helper function for fts3ExprIterate() (see below). |
+*/ |
+static int fts3ExprIterate2( |
+ Fts3Expr *pExpr, /* Expression to iterate phrases of */ |
+ int *piPhrase, /* Pointer to phrase counter */ |
+ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ |
+ void *pCtx /* Second argument to pass to callback */ |
+){ |
+ int rc; /* Return code */ |
+ int eType = pExpr->eType; /* Type of expression node pExpr */ |
+ |
+ if( eType!=FTSQUERY_PHRASE ){ |
+ assert( pExpr->pLeft && pExpr->pRight ); |
+ rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); |
+ if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ |
+ rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); |
+ } |
+ }else{ |
+ rc = x(pExpr, *piPhrase, pCtx); |
+ (*piPhrase)++; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Iterate through all phrase nodes in an FTS3 query, except those that |
+** are part of a sub-tree that is the right-hand-side of a NOT operator. |
+** For each phrase node found, the supplied callback function is invoked. |
+** |
+** If the callback function returns anything other than SQLITE_OK, |
+** the iteration is abandoned and the error code returned immediately. |
+** Otherwise, SQLITE_OK is returned after a callback has been made for |
+** all eligible phrase nodes. |
+*/ |
+static int fts3ExprIterate( |
+ Fts3Expr *pExpr, /* Expression to iterate phrases of */ |
+ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ |
+ void *pCtx /* Second argument to pass to callback */ |
+){ |
+ int iPhrase = 0; /* Variable used as the phrase counter */ |
+ return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); |
+} |
+ |
+ |
+/* |
+** This is an fts3ExprIterate() callback used while loading the doclists |
+** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also |
+** fts3ExprLoadDoclists(). |
+*/ |
+static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ |
+ int rc = SQLITE_OK; |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; |
+ |
+ UNUSED_PARAMETER(iPhrase); |
+ |
+ p->nPhrase++; |
+ p->nToken += pPhrase->nToken; |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Load the doclists for each phrase in the query associated with FTS3 cursor |
+** pCsr. |
+** |
+** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable |
+** phrases in the expression (all phrases except those directly or |
+** indirectly descended from the right-hand-side of a NOT operator). If |
+** pnToken is not NULL, then it is set to the number of tokens in all |
+** matchable phrases of the expression. |
+*/ |
+static int fts3ExprLoadDoclists( |
+ Fts3Cursor *pCsr, /* Fts3 cursor for current query */ |
+ int *pnPhrase, /* OUT: Number of phrases in query */ |
+ int *pnToken /* OUT: Number of tokens in query */ |
+){ |
+ int rc; /* Return Code */ |
+ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ |
+ sCtx.pCsr = pCsr; |
+ rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); |
+ if( pnPhrase ) *pnPhrase = sCtx.nPhrase; |
+ if( pnToken ) *pnToken = sCtx.nToken; |
+ return rc; |
+} |
+ |
+static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ |
+ (*(int *)ctx)++; |
+ pExpr->iPhrase = iPhrase; |
+ return SQLITE_OK; |
+} |
+static int fts3ExprPhraseCount(Fts3Expr *pExpr){ |
+ int nPhrase = 0; |
+ (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); |
+ return nPhrase; |
+} |
+ |
+/* |
+** Advance the position list iterator specified by the first two |
+** arguments so that it points to the first element with a value greater |
+** than or equal to parameter iNext. |
+*/ |
+static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ |
+ char *pIter = *ppIter; |
+ if( pIter ){ |
+ int iIter = *piIter; |
+ |
+ while( iIter<iNext ){ |
+ if( 0==(*pIter & 0xFE) ){ |
+ iIter = -1; |
+ pIter = 0; |
+ break; |
+ } |
+ fts3GetDeltaPosition(&pIter, &iIter); |
+ } |
+ |
+ *piIter = iIter; |
+ *ppIter = pIter; |
+ } |
+} |
+ |
+/* |
+** Advance the snippet iterator to the next candidate snippet. |
+*/ |
+static int fts3SnippetNextCandidate(SnippetIter *pIter){ |
+ int i; /* Loop counter */ |
+ |
+ if( pIter->iCurrent<0 ){ |
+ /* The SnippetIter object has just been initialized. The first snippet |
+ ** candidate always starts at offset 0 (even if this candidate has a |
+ ** score of 0.0). |
+ */ |
+ pIter->iCurrent = 0; |
+ |
+ /* Advance the 'head' iterator of each phrase to the first offset that |
+ ** is greater than or equal to (iNext+nSnippet). |
+ */ |
+ for(i=0; i<pIter->nPhrase; i++){ |
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; |
+ fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); |
+ } |
+ }else{ |
+ int iStart; |
+ int iEnd = 0x7FFFFFFF; |
+ |
+ for(i=0; i<pIter->nPhrase; i++){ |
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; |
+ if( pPhrase->pHead && pPhrase->iHead<iEnd ){ |
+ iEnd = pPhrase->iHead; |
+ } |
+ } |
+ if( iEnd==0x7FFFFFFF ){ |
+ return 1; |
+ } |
+ |
+ pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; |
+ for(i=0; i<pIter->nPhrase; i++){ |
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; |
+ fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); |
+ fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); |
+ } |
+ } |
+ |
+ return 0; |
+} |
+ |
+/* |
+** Retrieve information about the current candidate snippet of snippet |
+** iterator pIter. |
+*/ |
+static void fts3SnippetDetails( |
+ SnippetIter *pIter, /* Snippet iterator */ |
+ u64 mCovered, /* Bitmask of phrases already covered */ |
+ int *piToken, /* OUT: First token of proposed snippet */ |
+ int *piScore, /* OUT: "Score" for this snippet */ |
+ u64 *pmCover, /* OUT: Bitmask of phrases covered */ |
+ u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ |
+){ |
+ int iStart = pIter->iCurrent; /* First token of snippet */ |
+ int iScore = 0; /* Score of this snippet */ |
+ int i; /* Loop counter */ |
+ u64 mCover = 0; /* Mask of phrases covered by this snippet */ |
+ u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ |
+ |
+ for(i=0; i<pIter->nPhrase; i++){ |
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; |
+ if( pPhrase->pTail ){ |
+ char *pCsr = pPhrase->pTail; |
+ int iCsr = pPhrase->iTail; |
+ |
+ while( iCsr<(iStart+pIter->nSnippet) ){ |
+ int j; |
+ u64 mPhrase = (u64)1 << i; |
+ u64 mPos = (u64)1 << (iCsr - iStart); |
+ assert( iCsr>=iStart ); |
+ if( (mCover|mCovered)&mPhrase ){ |
+ iScore++; |
+ }else{ |
+ iScore += 1000; |
+ } |
+ mCover |= mPhrase; |
+ |
+ for(j=0; j<pPhrase->nToken; j++){ |
+ mHighlight |= (mPos>>j); |
+ } |
+ |
+ if( 0==(*pCsr & 0x0FE) ) break; |
+ fts3GetDeltaPosition(&pCsr, &iCsr); |
+ } |
+ } |
+ } |
+ |
+ /* Set the output variables before returning. */ |
+ *piToken = iStart; |
+ *piScore = iScore; |
+ *pmCover = mCover; |
+ *pmHighlight = mHighlight; |
+} |
+ |
+/* |
+** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). |
+** Each invocation populates an element of the SnippetIter.aPhrase[] array. |
+*/ |
+static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ |
+ SnippetIter *p = (SnippetIter *)ctx; |
+ SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; |
+ char *pCsr; |
+ int rc; |
+ |
+ pPhrase->nToken = pExpr->pPhrase->nToken; |
+ rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); |
+ assert( rc==SQLITE_OK || pCsr==0 ); |
+ if( pCsr ){ |
+ int iFirst = 0; |
+ pPhrase->pList = pCsr; |
+ fts3GetDeltaPosition(&pCsr, &iFirst); |
+ assert( iFirst>=0 ); |
+ pPhrase->pHead = pCsr; |
+ pPhrase->pTail = pCsr; |
+ pPhrase->iHead = iFirst; |
+ pPhrase->iTail = iFirst; |
+ }else{ |
+ assert( rc!=SQLITE_OK || ( |
+ pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 |
+ )); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Select the fragment of text consisting of nFragment contiguous tokens |
+** from column iCol that represent the "best" snippet. The best snippet |
+** is the snippet with the highest score, where scores are calculated |
+** by adding: |
+** |
+** (a) +1 point for each occurrence of a matchable phrase in the snippet. |
+** |
+** (b) +1000 points for the first occurrence of each matchable phrase in |
+** the snippet for which the corresponding mCovered bit is not set. |
+** |
+** The selected snippet parameters are stored in structure *pFragment before |
+** returning. The score of the selected snippet is stored in *piScore |
+** before returning. |
+*/ |
+static int fts3BestSnippet( |
+ int nSnippet, /* Desired snippet length */ |
+ Fts3Cursor *pCsr, /* Cursor to create snippet for */ |
+ int iCol, /* Index of column to create snippet from */ |
+ u64 mCovered, /* Mask of phrases already covered */ |
+ u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ |
+ SnippetFragment *pFragment, /* OUT: Best snippet found */ |
+ int *piScore /* OUT: Score of snippet pFragment */ |
+){ |
+ int rc; /* Return Code */ |
+ int nList; /* Number of phrases in expression */ |
+ SnippetIter sIter; /* Iterates through snippet candidates */ |
+ int nByte; /* Number of bytes of space to allocate */ |
+ int iBestScore = -1; /* Best snippet score found so far */ |
+ int i; /* Loop counter */ |
+ |
+ memset(&sIter, 0, sizeof(sIter)); |
+ |
+ /* Iterate through the phrases in the expression to count them. The same |
+ ** callback makes sure the doclists are loaded for each phrase. |
+ */ |
+ rc = fts3ExprLoadDoclists(pCsr, &nList, 0); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ /* Now that it is known how many phrases there are, allocate and zero |
+ ** the required space using malloc(). |
+ */ |
+ nByte = sizeof(SnippetPhrase) * nList; |
+ sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); |
+ if( !sIter.aPhrase ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(sIter.aPhrase, 0, nByte); |
+ |
+ /* Initialize the contents of the SnippetIter object. Then iterate through |
+ ** the set of phrases in the expression to populate the aPhrase[] array. |
+ */ |
+ sIter.pCsr = pCsr; |
+ sIter.iCol = iCol; |
+ sIter.nSnippet = nSnippet; |
+ sIter.nPhrase = nList; |
+ sIter.iCurrent = -1; |
+ rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); |
+ if( rc==SQLITE_OK ){ |
+ |
+ /* Set the *pmSeen output variable. */ |
+ for(i=0; i<nList; i++){ |
+ if( sIter.aPhrase[i].pHead ){ |
+ *pmSeen |= (u64)1 << i; |
+ } |
+ } |
+ |
+ /* Loop through all candidate snippets. Store the best snippet in |
+ ** *pFragment. Store its associated 'score' in iBestScore. |
+ */ |
+ pFragment->iCol = iCol; |
+ while( !fts3SnippetNextCandidate(&sIter) ){ |
+ int iPos; |
+ int iScore; |
+ u64 mCover; |
+ u64 mHighlite; |
+ fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); |
+ assert( iScore>=0 ); |
+ if( iScore>iBestScore ){ |
+ pFragment->iPos = iPos; |
+ pFragment->hlmask = mHighlite; |
+ pFragment->covered = mCover; |
+ iBestScore = iScore; |
+ } |
+ } |
+ |
+ *piScore = iBestScore; |
+ } |
+ sqlite3_free(sIter.aPhrase); |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Append a string to the string-buffer passed as the first argument. |
+** |
+** If nAppend is negative, then the length of the string zAppend is |
+** determined using strlen(). |
+*/ |
+static int fts3StringAppend( |
+ StrBuffer *pStr, /* Buffer to append to */ |
+ const char *zAppend, /* Pointer to data to append to buffer */ |
+ int nAppend /* Size of zAppend in bytes (or -1) */ |
+){ |
+ if( nAppend<0 ){ |
+ nAppend = (int)strlen(zAppend); |
+ } |
+ |
+ /* If there is insufficient space allocated at StrBuffer.z, use realloc() |
+ ** to grow the buffer until so that it is big enough to accomadate the |
+ ** appended data. |
+ */ |
+ if( pStr->n+nAppend+1>=pStr->nAlloc ){ |
+ int nAlloc = pStr->nAlloc+nAppend+100; |
+ char *zNew = sqlite3_realloc(pStr->z, nAlloc); |
+ if( !zNew ){ |
+ return SQLITE_NOMEM; |
+ } |
+ pStr->z = zNew; |
+ pStr->nAlloc = nAlloc; |
+ } |
+ assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); |
+ |
+ /* Append the data to the string buffer. */ |
+ memcpy(&pStr->z[pStr->n], zAppend, nAppend); |
+ pStr->n += nAppend; |
+ pStr->z[pStr->n] = '\0'; |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The fts3BestSnippet() function often selects snippets that end with a |
+** query term. That is, the final term of the snippet is always a term |
+** that requires highlighting. For example, if 'X' is a highlighted term |
+** and '.' is a non-highlighted term, BestSnippet() may select: |
+** |
+** ........X.....X |
+** |
+** This function "shifts" the beginning of the snippet forward in the |
+** document so that there are approximately the same number of |
+** non-highlighted terms to the right of the final highlighted term as there |
+** are to the left of the first highlighted term. For example, to this: |
+** |
+** ....X.....X.... |
+** |
+** This is done as part of extracting the snippet text, not when selecting |
+** the snippet. Snippet selection is done based on doclists only, so there |
+** is no way for fts3BestSnippet() to know whether or not the document |
+** actually contains terms that follow the final highlighted term. |
+*/ |
+static int fts3SnippetShift( |
+ Fts3Table *pTab, /* FTS3 table snippet comes from */ |
+ int iLangid, /* Language id to use in tokenizing */ |
+ int nSnippet, /* Number of tokens desired for snippet */ |
+ const char *zDoc, /* Document text to extract snippet from */ |
+ int nDoc, /* Size of buffer zDoc in bytes */ |
+ int *piPos, /* IN/OUT: First token of snippet */ |
+ u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ |
+){ |
+ u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ |
+ |
+ if( hlmask ){ |
+ int nLeft; /* Tokens to the left of first highlight */ |
+ int nRight; /* Tokens to the right of last highlight */ |
+ int nDesired; /* Ideal number of tokens to shift forward */ |
+ |
+ for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); |
+ for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); |
+ nDesired = (nLeft-nRight)/2; |
+ |
+ /* Ideally, the start of the snippet should be pushed forward in the |
+ ** document nDesired tokens. This block checks if there are actually |
+ ** nDesired tokens to the right of the snippet. If so, *piPos and |
+ ** *pHlMask are updated to shift the snippet nDesired tokens to the |
+ ** right. Otherwise, the snippet is shifted by the number of tokens |
+ ** available. |
+ */ |
+ if( nDesired>0 ){ |
+ int nShift; /* Number of tokens to shift snippet by */ |
+ int iCurrent = 0; /* Token counter */ |
+ int rc; /* Return Code */ |
+ sqlite3_tokenizer_module *pMod; |
+ sqlite3_tokenizer_cursor *pC; |
+ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; |
+ |
+ /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) |
+ ** or more tokens in zDoc/nDoc. |
+ */ |
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ |
+ const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; |
+ rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); |
+ } |
+ pMod->xClose(pC); |
+ if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } |
+ |
+ nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; |
+ assert( nShift<=nDesired ); |
+ if( nShift>0 ){ |
+ *piPos += nShift; |
+ *pHlmask = hlmask >> nShift; |
+ } |
+ } |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Extract the snippet text for fragment pFragment from cursor pCsr and |
+** append it to string buffer pOut. |
+*/ |
+static int fts3SnippetText( |
+ Fts3Cursor *pCsr, /* FTS3 Cursor */ |
+ SnippetFragment *pFragment, /* Snippet to extract */ |
+ int iFragment, /* Fragment number */ |
+ int isLast, /* True for final fragment in snippet */ |
+ int nSnippet, /* Number of tokens in extracted snippet */ |
+ const char *zOpen, /* String inserted before highlighted term */ |
+ const char *zClose, /* String inserted after highlighted term */ |
+ const char *zEllipsis, /* String inserted between snippets */ |
+ StrBuffer *pOut /* Write output here */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int rc; /* Return code */ |
+ const char *zDoc; /* Document text to extract snippet from */ |
+ int nDoc; /* Size of zDoc in bytes */ |
+ int iCurrent = 0; /* Current token number of document */ |
+ int iEnd = 0; /* Byte offset of end of current token */ |
+ int isShiftDone = 0; /* True after snippet is shifted */ |
+ int iPos = pFragment->iPos; /* First token of snippet */ |
+ u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ |
+ int iCol = pFragment->iCol+1; /* Query column to extract text from */ |
+ sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ |
+ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ |
+ |
+ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); |
+ if( zDoc==0 ){ |
+ if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ |
+ return SQLITE_NOMEM; |
+ } |
+ return SQLITE_OK; |
+ } |
+ nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); |
+ |
+ /* Open a token cursor on the document. */ |
+ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; |
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ while( rc==SQLITE_OK ){ |
+ const char *ZDUMMY; /* Dummy argument used with tokenizer */ |
+ int DUMMY1 = -1; /* Dummy argument used with tokenizer */ |
+ int iBegin = 0; /* Offset in zDoc of start of token */ |
+ int iFin = 0; /* Offset in zDoc of end of token */ |
+ int isHighlight = 0; /* True for highlighted terms */ |
+ |
+ /* Variable DUMMY1 is initialized to a negative value above. Elsewhere |
+ ** in the FTS code the variable that the third argument to xNext points to |
+ ** is initialized to zero before the first (*but not necessarily |
+ ** subsequent*) call to xNext(). This is done for a particular application |
+ ** that needs to know whether or not the tokenizer is being used for |
+ ** snippet generation or for some other purpose. |
+ ** |
+ ** Extreme care is required when writing code to depend on this |
+ ** initialization. It is not a documented part of the tokenizer interface. |
+ ** If a tokenizer is used directly by any code outside of FTS, this |
+ ** convention might not be respected. */ |
+ rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); |
+ if( rc!=SQLITE_OK ){ |
+ if( rc==SQLITE_DONE ){ |
+ /* Special case - the last token of the snippet is also the last token |
+ ** of the column. Append any punctuation that occurred between the end |
+ ** of the previous token and the end of the document to the output. |
+ ** Then break out of the loop. */ |
+ rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); |
+ } |
+ break; |
+ } |
+ if( iCurrent<iPos ){ continue; } |
+ |
+ if( !isShiftDone ){ |
+ int n = nDoc - iBegin; |
+ rc = fts3SnippetShift( |
+ pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask |
+ ); |
+ isShiftDone = 1; |
+ |
+ /* Now that the shift has been done, check if the initial "..." are |
+ ** required. They are required if (a) this is not the first fragment, |
+ ** or (b) this fragment does not begin at position 0 of its column. |
+ */ |
+ if( rc==SQLITE_OK ){ |
+ if( iPos>0 || iFragment>0 ){ |
+ rc = fts3StringAppend(pOut, zEllipsis, -1); |
+ }else if( iBegin ){ |
+ rc = fts3StringAppend(pOut, zDoc, iBegin); |
+ } |
+ } |
+ if( rc!=SQLITE_OK || iCurrent<iPos ) continue; |
+ } |
+ |
+ if( iCurrent>=(iPos+nSnippet) ){ |
+ if( isLast ){ |
+ rc = fts3StringAppend(pOut, zEllipsis, -1); |
+ } |
+ break; |
+ } |
+ |
+ /* Set isHighlight to true if this term should be highlighted. */ |
+ isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; |
+ |
+ if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); |
+ if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); |
+ if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); |
+ if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); |
+ |
+ iEnd = iFin; |
+ } |
+ |
+ pMod->xClose(pC); |
+ return rc; |
+} |
+ |
+ |
+/* |
+** This function is used to count the entries in a column-list (a |
+** delta-encoded list of term offsets within a single column of a single |
+** row). When this function is called, *ppCollist should point to the |
+** beginning of the first varint in the column-list (the varint that |
+** contains the position of the first matching term in the column data). |
+** Before returning, *ppCollist is set to point to the first byte after |
+** the last varint in the column-list (either the 0x00 signifying the end |
+** of the position-list, or the 0x01 that precedes the column number of |
+** the next column in the position-list). |
+** |
+** The number of elements in the column-list is returned. |
+*/ |
+static int fts3ColumnlistCount(char **ppCollist){ |
+ char *pEnd = *ppCollist; |
+ char c = 0; |
+ int nEntry = 0; |
+ |
+ /* A column-list is terminated by either a 0x01 or 0x00. */ |
+ while( 0xFE & (*pEnd | c) ){ |
+ c = *pEnd++ & 0x80; |
+ if( !c ) nEntry++; |
+ } |
+ |
+ *ppCollist = pEnd; |
+ return nEntry; |
+} |
+ |
+/* |
+** This function gathers 'y' or 'b' data for a single phrase. |
+*/ |
+static void fts3ExprLHits( |
+ Fts3Expr *pExpr, /* Phrase expression node */ |
+ MatchInfo *p /* Matchinfo context */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; |
+ int iStart; |
+ Fts3Phrase *pPhrase = pExpr->pPhrase; |
+ char *pIter = pPhrase->doclist.pList; |
+ int iCol = 0; |
+ |
+ assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); |
+ if( p->flag==FTS3_MATCHINFO_LHITS ){ |
+ iStart = pExpr->iPhrase * p->nCol; |
+ }else{ |
+ iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); |
+ } |
+ |
+ while( 1 ){ |
+ int nHit = fts3ColumnlistCount(&pIter); |
+ if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ |
+ if( p->flag==FTS3_MATCHINFO_LHITS ){ |
+ p->aMatchinfo[iStart + iCol] = (u32)nHit; |
+ }else if( nHit ){ |
+ p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); |
+ } |
+ } |
+ assert( *pIter==0x00 || *pIter==0x01 ); |
+ if( *pIter!=0x01 ) break; |
+ pIter++; |
+ pIter += fts3GetVarint32(pIter, &iCol); |
+ } |
+} |
+ |
+/* |
+** Gather the results for matchinfo directives 'y' and 'b'. |
+*/ |
+static void fts3ExprLHitGather( |
+ Fts3Expr *pExpr, |
+ MatchInfo *p |
+){ |
+ assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); |
+ if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ |
+ if( pExpr->pLeft ){ |
+ fts3ExprLHitGather(pExpr->pLeft, p); |
+ fts3ExprLHitGather(pExpr->pRight, p); |
+ }else{ |
+ fts3ExprLHits(pExpr, p); |
+ } |
+ } |
+} |
+ |
+/* |
+** fts3ExprIterate() callback used to collect the "global" matchinfo stats |
+** for a single query. |
+** |
+** fts3ExprIterate() callback to load the 'global' elements of a |
+** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements |
+** of the matchinfo array that are constant for all rows returned by the |
+** current query. |
+** |
+** Argument pCtx is actually a pointer to a struct of type MatchInfo. This |
+** function populates Matchinfo.aMatchinfo[] as follows: |
+** |
+** for(iCol=0; iCol<nCol; iCol++){ |
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 1] = X; |
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 2] = Y; |
+** } |
+** |
+** where X is the number of matches for phrase iPhrase is column iCol of all |
+** rows of the table. Y is the number of rows for which column iCol contains |
+** at least one instance of phrase iPhrase. |
+** |
+** If the phrase pExpr consists entirely of deferred tokens, then all X and |
+** Y values are set to nDoc, where nDoc is the number of documents in the |
+** file system. This is done because the full-text index doclist is required |
+** to calculate these values properly, and the full-text index doclist is |
+** not available for deferred tokens. |
+*/ |
+static int fts3ExprGlobalHitsCb( |
+ Fts3Expr *pExpr, /* Phrase expression node */ |
+ int iPhrase, /* Phrase number (numbered from zero) */ |
+ void *pCtx /* Pointer to MatchInfo structure */ |
+){ |
+ MatchInfo *p = (MatchInfo *)pCtx; |
+ return sqlite3Fts3EvalPhraseStats( |
+ p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] |
+ ); |
+} |
+ |
+/* |
+** fts3ExprIterate() callback used to collect the "local" part of the |
+** FTS3_MATCHINFO_HITS array. The local stats are those elements of the |
+** array that are different for each row returned by the query. |
+*/ |
+static int fts3ExprLocalHitsCb( |
+ Fts3Expr *pExpr, /* Phrase expression node */ |
+ int iPhrase, /* Phrase number */ |
+ void *pCtx /* Pointer to MatchInfo structure */ |
+){ |
+ int rc = SQLITE_OK; |
+ MatchInfo *p = (MatchInfo *)pCtx; |
+ int iStart = iPhrase * p->nCol * 3; |
+ int i; |
+ |
+ for(i=0; i<p->nCol && rc==SQLITE_OK; i++){ |
+ char *pCsr; |
+ rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); |
+ if( pCsr ){ |
+ p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); |
+ }else{ |
+ p->aMatchinfo[iStart+i*3] = 0; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+static int fts3MatchinfoCheck( |
+ Fts3Table *pTab, |
+ char cArg, |
+ char **pzErr |
+){ |
+ if( (cArg==FTS3_MATCHINFO_NPHRASE) |
+ || (cArg==FTS3_MATCHINFO_NCOL) |
+ || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) |
+ || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) |
+ || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) |
+ || (cArg==FTS3_MATCHINFO_LCS) |
+ || (cArg==FTS3_MATCHINFO_HITS) |
+ || (cArg==FTS3_MATCHINFO_LHITS) |
+ || (cArg==FTS3_MATCHINFO_LHITS_BM) |
+ ){ |
+ return SQLITE_OK; |
+ } |
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); |
+ return SQLITE_ERROR; |
+} |
+ |
+static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ |
+ int nVal; /* Number of integers output by cArg */ |
+ |
+ switch( cArg ){ |
+ case FTS3_MATCHINFO_NDOC: |
+ case FTS3_MATCHINFO_NPHRASE: |
+ case FTS3_MATCHINFO_NCOL: |
+ nVal = 1; |
+ break; |
+ |
+ case FTS3_MATCHINFO_AVGLENGTH: |
+ case FTS3_MATCHINFO_LENGTH: |
+ case FTS3_MATCHINFO_LCS: |
+ nVal = pInfo->nCol; |
+ break; |
+ |
+ case FTS3_MATCHINFO_LHITS: |
+ nVal = pInfo->nCol * pInfo->nPhrase; |
+ break; |
+ |
+ case FTS3_MATCHINFO_LHITS_BM: |
+ nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); |
+ break; |
+ |
+ default: |
+ assert( cArg==FTS3_MATCHINFO_HITS ); |
+ nVal = pInfo->nCol * pInfo->nPhrase * 3; |
+ break; |
+ } |
+ |
+ return nVal; |
+} |
+ |
+static int fts3MatchinfoSelectDoctotal( |
+ Fts3Table *pTab, |
+ sqlite3_stmt **ppStmt, |
+ sqlite3_int64 *pnDoc, |
+ const char **paLen |
+){ |
+ sqlite3_stmt *pStmt; |
+ const char *a; |
+ sqlite3_int64 nDoc; |
+ |
+ if( !*ppStmt ){ |
+ int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ pStmt = *ppStmt; |
+ assert( sqlite3_data_count(pStmt)==1 ); |
+ |
+ a = sqlite3_column_blob(pStmt, 0); |
+ a += sqlite3Fts3GetVarint(a, &nDoc); |
+ if( nDoc==0 ) return FTS_CORRUPT_VTAB; |
+ *pnDoc = (u32)nDoc; |
+ |
+ if( paLen ) *paLen = a; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** An instance of the following structure is used to store state while |
+** iterating through a multi-column position-list corresponding to the |
+** hits for a single phrase on a single row in order to calculate the |
+** values for a matchinfo() FTS3_MATCHINFO_LCS request. |
+*/ |
+typedef struct LcsIterator LcsIterator; |
+struct LcsIterator { |
+ Fts3Expr *pExpr; /* Pointer to phrase expression */ |
+ int iPosOffset; /* Tokens count up to end of this phrase */ |
+ char *pRead; /* Cursor used to iterate through aDoclist */ |
+ int iPos; /* Current position */ |
+}; |
+ |
+/* |
+** If LcsIterator.iCol is set to the following value, the iterator has |
+** finished iterating through all offsets for all columns. |
+*/ |
+#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; |
+ |
+static int fts3MatchinfoLcsCb( |
+ Fts3Expr *pExpr, /* Phrase expression node */ |
+ int iPhrase, /* Phrase number (numbered from zero) */ |
+ void *pCtx /* Pointer to MatchInfo structure */ |
+){ |
+ LcsIterator *aIter = (LcsIterator *)pCtx; |
+ aIter[iPhrase].pExpr = pExpr; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Advance the iterator passed as an argument to the next position. Return |
+** 1 if the iterator is at EOF or if it now points to the start of the |
+** position list for the next column. |
+*/ |
+static int fts3LcsIteratorAdvance(LcsIterator *pIter){ |
+ char *pRead = pIter->pRead; |
+ sqlite3_int64 iRead; |
+ int rc = 0; |
+ |
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead); |
+ if( iRead==0 || iRead==1 ){ |
+ pRead = 0; |
+ rc = 1; |
+ }else{ |
+ pIter->iPos += (int)(iRead-2); |
+ } |
+ |
+ pIter->pRead = pRead; |
+ return rc; |
+} |
+ |
+/* |
+** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. |
+** |
+** If the call is successful, the longest-common-substring lengths for each |
+** column are written into the first nCol elements of the pInfo->aMatchinfo[] |
+** array before returning. SQLITE_OK is returned in this case. |
+** |
+** Otherwise, if an error occurs, an SQLite error code is returned and the |
+** data written to the first nCol elements of pInfo->aMatchinfo[] is |
+** undefined. |
+*/ |
+static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ |
+ LcsIterator *aIter; |
+ int i; |
+ int iCol; |
+ int nToken = 0; |
+ |
+ /* Allocate and populate the array of LcsIterator objects. The array |
+ ** contains one element for each matchable phrase in the query. |
+ **/ |
+ aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); |
+ if( !aIter ) return SQLITE_NOMEM; |
+ memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); |
+ (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); |
+ |
+ for(i=0; i<pInfo->nPhrase; i++){ |
+ LcsIterator *pIter = &aIter[i]; |
+ nToken -= pIter->pExpr->pPhrase->nToken; |
+ pIter->iPosOffset = nToken; |
+ } |
+ |
+ for(iCol=0; iCol<pInfo->nCol; iCol++){ |
+ int nLcs = 0; /* LCS value for this column */ |
+ int nLive = 0; /* Number of iterators in aIter not at EOF */ |
+ |
+ for(i=0; i<pInfo->nPhrase; i++){ |
+ int rc; |
+ LcsIterator *pIt = &aIter[i]; |
+ rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); |
+ if( rc!=SQLITE_OK ) return rc; |
+ if( pIt->pRead ){ |
+ pIt->iPos = pIt->iPosOffset; |
+ fts3LcsIteratorAdvance(&aIter[i]); |
+ nLive++; |
+ } |
+ } |
+ |
+ while( nLive>0 ){ |
+ LcsIterator *pAdv = 0; /* The iterator to advance by one position */ |
+ int nThisLcs = 0; /* LCS for the current iterator positions */ |
+ |
+ for(i=0; i<pInfo->nPhrase; i++){ |
+ LcsIterator *pIter = &aIter[i]; |
+ if( pIter->pRead==0 ){ |
+ /* This iterator is already at EOF for this column. */ |
+ nThisLcs = 0; |
+ }else{ |
+ if( pAdv==0 || pIter->iPos<pAdv->iPos ){ |
+ pAdv = pIter; |
+ } |
+ if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ |
+ nThisLcs++; |
+ }else{ |
+ nThisLcs = 1; |
+ } |
+ if( nThisLcs>nLcs ) nLcs = nThisLcs; |
+ } |
+ } |
+ if( fts3LcsIteratorAdvance(pAdv) ) nLive--; |
+ } |
+ |
+ pInfo->aMatchinfo[iCol] = nLcs; |
+ } |
+ |
+ sqlite3_free(aIter); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Populate the buffer pInfo->aMatchinfo[] with an array of integers to |
+** be returned by the matchinfo() function. Argument zArg contains the |
+** format string passed as the second argument to matchinfo (or the |
+** default value "pcx" if no second argument was specified). The format |
+** string has already been validated and the pInfo->aMatchinfo[] array |
+** is guaranteed to be large enough for the output. |
+** |
+** If bGlobal is true, then populate all fields of the matchinfo() output. |
+** If it is false, then assume that those fields that do not change between |
+** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) |
+** have already been populated. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code if an error |
+** occurs. If a value other than SQLITE_OK is returned, the state the |
+** pInfo->aMatchinfo[] buffer is left in is undefined. |
+*/ |
+static int fts3MatchinfoValues( |
+ Fts3Cursor *pCsr, /* FTS3 cursor object */ |
+ int bGlobal, /* True to grab the global stats */ |
+ MatchInfo *pInfo, /* Matchinfo context object */ |
+ const char *zArg /* Matchinfo format string */ |
+){ |
+ int rc = SQLITE_OK; |
+ int i; |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ sqlite3_stmt *pSelect = 0; |
+ |
+ for(i=0; rc==SQLITE_OK && zArg[i]; i++){ |
+ pInfo->flag = zArg[i]; |
+ switch( zArg[i] ){ |
+ case FTS3_MATCHINFO_NPHRASE: |
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; |
+ break; |
+ |
+ case FTS3_MATCHINFO_NCOL: |
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; |
+ break; |
+ |
+ case FTS3_MATCHINFO_NDOC: |
+ if( bGlobal ){ |
+ sqlite3_int64 nDoc = 0; |
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); |
+ pInfo->aMatchinfo[0] = (u32)nDoc; |
+ } |
+ break; |
+ |
+ case FTS3_MATCHINFO_AVGLENGTH: |
+ if( bGlobal ){ |
+ sqlite3_int64 nDoc; /* Number of rows in table */ |
+ const char *a; /* Aggregate column length array */ |
+ |
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); |
+ if( rc==SQLITE_OK ){ |
+ int iCol; |
+ for(iCol=0; iCol<pInfo->nCol; iCol++){ |
+ u32 iVal; |
+ sqlite3_int64 nToken; |
+ a += sqlite3Fts3GetVarint(a, &nToken); |
+ iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); |
+ pInfo->aMatchinfo[iCol] = iVal; |
+ } |
+ } |
+ } |
+ break; |
+ |
+ case FTS3_MATCHINFO_LENGTH: { |
+ sqlite3_stmt *pSelectDocsize = 0; |
+ rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); |
+ if( rc==SQLITE_OK ){ |
+ int iCol; |
+ const char *a = sqlite3_column_blob(pSelectDocsize, 0); |
+ for(iCol=0; iCol<pInfo->nCol; iCol++){ |
+ sqlite3_int64 nToken; |
+ a += sqlite3Fts3GetVarint(a, &nToken); |
+ pInfo->aMatchinfo[iCol] = (u32)nToken; |
+ } |
+ } |
+ sqlite3_reset(pSelectDocsize); |
+ break; |
+ } |
+ |
+ case FTS3_MATCHINFO_LCS: |
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts3MatchinfoLcs(pCsr, pInfo); |
+ } |
+ break; |
+ |
+ case FTS3_MATCHINFO_LHITS_BM: |
+ case FTS3_MATCHINFO_LHITS: { |
+ int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); |
+ memset(pInfo->aMatchinfo, 0, nZero); |
+ fts3ExprLHitGather(pCsr->pExpr, pInfo); |
+ break; |
+ } |
+ |
+ default: { |
+ Fts3Expr *pExpr; |
+ assert( zArg[i]==FTS3_MATCHINFO_HITS ); |
+ pExpr = pCsr->pExpr; |
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0); |
+ if( rc!=SQLITE_OK ) break; |
+ if( bGlobal ){ |
+ if( pCsr->pDeferred ){ |
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); |
+ if( rc!=SQLITE_OK ) break; |
+ } |
+ rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); |
+ sqlite3Fts3EvalTestDeferred(pCsr, &rc); |
+ if( rc!=SQLITE_OK ) break; |
+ } |
+ (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); |
+ break; |
+ } |
+ } |
+ |
+ pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); |
+ } |
+ |
+ sqlite3_reset(pSelect); |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Populate pCsr->aMatchinfo[] with data for the current row. The |
+** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). |
+*/ |
+static void fts3GetMatchinfo( |
+ sqlite3_context *pCtx, /* Return results here */ |
+ Fts3Cursor *pCsr, /* FTS3 Cursor object */ |
+ const char *zArg /* Second argument to matchinfo() function */ |
+){ |
+ MatchInfo sInfo; |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int rc = SQLITE_OK; |
+ int bGlobal = 0; /* Collect 'global' stats as well as local */ |
+ |
+ u32 *aOut = 0; |
+ void (*xDestroyOut)(void*) = 0; |
+ |
+ memset(&sInfo, 0, sizeof(MatchInfo)); |
+ sInfo.pCursor = pCsr; |
+ sInfo.nCol = pTab->nColumn; |
+ |
+ /* If there is cached matchinfo() data, but the format string for the |
+ ** cache does not match the format string for this request, discard |
+ ** the cached data. */ |
+ if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ |
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); |
+ pCsr->pMIBuffer = 0; |
+ } |
+ |
+ /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the |
+ ** matchinfo function has been called for this query. In this case |
+ ** allocate the array used to accumulate the matchinfo data and |
+ ** initialize those elements that are constant for every row. |
+ */ |
+ if( pCsr->pMIBuffer==0 ){ |
+ int nMatchinfo = 0; /* Number of u32 elements in match-info */ |
+ int i; /* Used to iterate through zArg */ |
+ |
+ /* Determine the number of phrases in the query */ |
+ pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); |
+ sInfo.nPhrase = pCsr->nPhrase; |
+ |
+ /* Determine the number of integers in the buffer returned by this call. */ |
+ for(i=0; zArg[i]; i++){ |
+ char *zErr = 0; |
+ if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ |
+ sqlite3_result_error(pCtx, zErr, -1); |
+ sqlite3_free(zErr); |
+ return; |
+ } |
+ nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); |
+ } |
+ |
+ /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ |
+ pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); |
+ if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; |
+ |
+ pCsr->isMatchinfoNeeded = 1; |
+ bGlobal = 1; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); |
+ if( xDestroyOut==0 ){ |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ sInfo.aMatchinfo = aOut; |
+ sInfo.nPhrase = pCsr->nPhrase; |
+ rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); |
+ if( bGlobal ){ |
+ fts3MIBufferSetGlobal(pCsr->pMIBuffer); |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_result_error_code(pCtx, rc); |
+ if( xDestroyOut ) xDestroyOut(aOut); |
+ }else{ |
+ int n = pCsr->pMIBuffer->nElem * sizeof(u32); |
+ sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); |
+ } |
+} |
+ |
+/* |
+** Implementation of snippet() function. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3Snippet( |
+ sqlite3_context *pCtx, /* SQLite function call context */ |
+ Fts3Cursor *pCsr, /* Cursor object */ |
+ const char *zStart, /* Snippet start text - "<b>" */ |
+ const char *zEnd, /* Snippet end text - "</b>" */ |
+ const char *zEllipsis, /* Snippet ellipsis text - "<b>...</b>" */ |
+ int iCol, /* Extract snippet from this column */ |
+ int nToken /* Approximate number of tokens in snippet */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ int rc = SQLITE_OK; |
+ int i; |
+ StrBuffer res = {0, 0, 0}; |
+ |
+ /* The returned text includes up to four fragments of text extracted from |
+ ** the data in the current row. The first iteration of the for(...) loop |
+ ** below attempts to locate a single fragment of text nToken tokens in |
+ ** size that contains at least one instance of all phrases in the query |
+ ** expression that appear in the current row. If such a fragment of text |
+ ** cannot be found, the second iteration of the loop attempts to locate |
+ ** a pair of fragments, and so on. |
+ */ |
+ int nSnippet = 0; /* Number of fragments in this snippet */ |
+ SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ |
+ int nFToken = -1; /* Number of tokens in each fragment */ |
+ |
+ if( !pCsr->pExpr ){ |
+ sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); |
+ return; |
+ } |
+ |
+ for(nSnippet=1; 1; nSnippet++){ |
+ |
+ int iSnip; /* Loop counter 0..nSnippet-1 */ |
+ u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ |
+ u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ |
+ |
+ if( nToken>=0 ){ |
+ nFToken = (nToken+nSnippet-1) / nSnippet; |
+ }else{ |
+ nFToken = -1 * nToken; |
+ } |
+ |
+ for(iSnip=0; iSnip<nSnippet; iSnip++){ |
+ int iBestScore = -1; /* Best score of columns checked so far */ |
+ int iRead; /* Used to iterate through columns */ |
+ SnippetFragment *pFragment = &aSnippet[iSnip]; |
+ |
+ memset(pFragment, 0, sizeof(*pFragment)); |
+ |
+ /* Loop through all columns of the table being considered for snippets. |
+ ** If the iCol argument to this function was negative, this means all |
+ ** columns of the FTS3 table. Otherwise, only column iCol is considered. |
+ */ |
+ for(iRead=0; iRead<pTab->nColumn; iRead++){ |
+ SnippetFragment sF = {0, 0, 0, 0}; |
+ int iS = 0; |
+ if( iCol>=0 && iRead!=iCol ) continue; |
+ |
+ /* Find the best snippet of nFToken tokens in column iRead. */ |
+ rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); |
+ if( rc!=SQLITE_OK ){ |
+ goto snippet_out; |
+ } |
+ if( iS>iBestScore ){ |
+ *pFragment = sF; |
+ iBestScore = iS; |
+ } |
+ } |
+ |
+ mCovered |= pFragment->covered; |
+ } |
+ |
+ /* If all query phrases seen by fts3BestSnippet() are present in at least |
+ ** one of the nSnippet snippet fragments, break out of the loop. |
+ */ |
+ assert( (mCovered&mSeen)==mCovered ); |
+ if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; |
+ } |
+ |
+ assert( nFToken>0 ); |
+ |
+ for(i=0; i<nSnippet && rc==SQLITE_OK; i++){ |
+ rc = fts3SnippetText(pCsr, &aSnippet[i], |
+ i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res |
+ ); |
+ } |
+ |
+ snippet_out: |
+ sqlite3Fts3SegmentsClose(pTab); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_result_error_code(pCtx, rc); |
+ sqlite3_free(res.z); |
+ }else{ |
+ sqlite3_result_text(pCtx, res.z, -1, sqlite3_free); |
+ } |
+} |
+ |
+ |
+typedef struct TermOffset TermOffset; |
+typedef struct TermOffsetCtx TermOffsetCtx; |
+ |
+struct TermOffset { |
+ char *pList; /* Position-list */ |
+ int iPos; /* Position just read from pList */ |
+ int iOff; /* Offset of this term from read positions */ |
+}; |
+ |
+struct TermOffsetCtx { |
+ Fts3Cursor *pCsr; |
+ int iCol; /* Column of table to populate aTerm for */ |
+ int iTerm; |
+ sqlite3_int64 iDocid; |
+ TermOffset *aTerm; |
+}; |
+ |
+/* |
+** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets(). |
+*/ |
+static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ |
+ TermOffsetCtx *p = (TermOffsetCtx *)ctx; |
+ int nTerm; /* Number of tokens in phrase */ |
+ int iTerm; /* For looping through nTerm phrase terms */ |
+ char *pList; /* Pointer to position list for phrase */ |
+ int iPos = 0; /* First position in position-list */ |
+ int rc; |
+ |
+ UNUSED_PARAMETER(iPhrase); |
+ rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList); |
+ nTerm = pExpr->pPhrase->nToken; |
+ if( pList ){ |
+ fts3GetDeltaPosition(&pList, &iPos); |
+ assert( iPos>=0 ); |
+ } |
+ |
+ for(iTerm=0; iTerm<nTerm; iTerm++){ |
+ TermOffset *pT = &p->aTerm[p->iTerm++]; |
+ pT->iOff = nTerm-iTerm-1; |
+ pT->pList = pList; |
+ pT->iPos = iPos; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of offsets() function. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3Offsets( |
+ sqlite3_context *pCtx, /* SQLite function call context */ |
+ Fts3Cursor *pCsr /* Cursor object */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; |
+ int rc; /* Return Code */ |
+ int nToken; /* Number of tokens in query */ |
+ int iCol; /* Column currently being processed */ |
+ StrBuffer res = {0, 0, 0}; /* Result string */ |
+ TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ |
+ |
+ if( !pCsr->pExpr ){ |
+ sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); |
+ return; |
+ } |
+ |
+ memset(&sCtx, 0, sizeof(sCtx)); |
+ assert( pCsr->isRequireSeek==0 ); |
+ |
+ /* Count the number of terms in the query */ |
+ rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); |
+ if( rc!=SQLITE_OK ) goto offsets_out; |
+ |
+ /* Allocate the array of TermOffset iterators. */ |
+ sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken); |
+ if( 0==sCtx.aTerm ){ |
+ rc = SQLITE_NOMEM; |
+ goto offsets_out; |
+ } |
+ sCtx.iDocid = pCsr->iPrevId; |
+ sCtx.pCsr = pCsr; |
+ |
+ /* Loop through the table columns, appending offset information to |
+ ** string-buffer res for each column. |
+ */ |
+ for(iCol=0; iCol<pTab->nColumn; iCol++){ |
+ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ |
+ const char *ZDUMMY; /* Dummy argument used with xNext() */ |
+ int NDUMMY = 0; /* Dummy argument used with xNext() */ |
+ int iStart = 0; |
+ int iEnd = 0; |
+ int iCurrent = 0; |
+ const char *zDoc; |
+ int nDoc; |
+ |
+ /* Initialize the contents of sCtx.aTerm[] for column iCol. There is |
+ ** no way that this operation can fail, so the return code from |
+ ** fts3ExprIterate() can be discarded. |
+ */ |
+ sCtx.iCol = iCol; |
+ sCtx.iTerm = 0; |
+ (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); |
+ |
+ /* Retreive the text stored in column iCol. If an SQL NULL is stored |
+ ** in column iCol, jump immediately to the next iteration of the loop. |
+ ** If an OOM occurs while retrieving the data (this can happen if SQLite |
+ ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM |
+ ** to the caller. |
+ */ |
+ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); |
+ nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); |
+ if( zDoc==0 ){ |
+ if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ |
+ continue; |
+ } |
+ rc = SQLITE_NOMEM; |
+ goto offsets_out; |
+ } |
+ |
+ /* Initialize a tokenizer iterator to iterate through column iCol. */ |
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, |
+ zDoc, nDoc, &pC |
+ ); |
+ if( rc!=SQLITE_OK ) goto offsets_out; |
+ |
+ rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); |
+ while( rc==SQLITE_OK ){ |
+ int i; /* Used to loop through terms */ |
+ int iMinPos = 0x7FFFFFFF; /* Position of next token */ |
+ TermOffset *pTerm = 0; /* TermOffset associated with next token */ |
+ |
+ for(i=0; i<nToken; i++){ |
+ TermOffset *pT = &sCtx.aTerm[i]; |
+ if( pT->pList && (pT->iPos-pT->iOff)<iMinPos ){ |
+ iMinPos = pT->iPos-pT->iOff; |
+ pTerm = pT; |
+ } |
+ } |
+ |
+ if( !pTerm ){ |
+ /* All offsets for this column have been gathered. */ |
+ rc = SQLITE_DONE; |
+ }else{ |
+ assert( iCurrent<=iMinPos ); |
+ if( 0==(0xFE&*pTerm->pList) ){ |
+ pTerm->pList = 0; |
+ }else{ |
+ fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); |
+ } |
+ while( rc==SQLITE_OK && iCurrent<iMinPos ){ |
+ rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ char aBuffer[64]; |
+ sqlite3_snprintf(sizeof(aBuffer), aBuffer, |
+ "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart |
+ ); |
+ rc = fts3StringAppend(&res, aBuffer, -1); |
+ }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ |
+ rc = FTS_CORRUPT_VTAB; |
+ } |
+ } |
+ } |
+ if( rc==SQLITE_DONE ){ |
+ rc = SQLITE_OK; |
+ } |
+ |
+ pMod->xClose(pC); |
+ if( rc!=SQLITE_OK ) goto offsets_out; |
+ } |
+ |
+ offsets_out: |
+ sqlite3_free(sCtx.aTerm); |
+ assert( rc!=SQLITE_DONE ); |
+ sqlite3Fts3SegmentsClose(pTab); |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_result_error_code(pCtx, rc); |
+ sqlite3_free(res.z); |
+ }else{ |
+ sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); |
+ } |
+ return; |
+} |
+ |
+/* |
+** Implementation of matchinfo() function. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3Matchinfo( |
+ sqlite3_context *pContext, /* Function call context */ |
+ Fts3Cursor *pCsr, /* FTS3 table cursor */ |
+ const char *zArg /* Second arg to matchinfo() function */ |
+){ |
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
+ const char *zFormat; |
+ |
+ if( zArg ){ |
+ zFormat = zArg; |
+ }else{ |
+ zFormat = FTS3_MATCHINFO_DEFAULT; |
+ } |
+ |
+ if( !pCsr->pExpr ){ |
+ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); |
+ return; |
+ }else{ |
+ /* Retrieve matchinfo() data. */ |
+ fts3GetMatchinfo(pContext, pCsr, zFormat); |
+ sqlite3Fts3SegmentsClose(pTab); |
+ } |
+} |
+ |
+#endif |
+ |
+/************** End of fts3_snippet.c ****************************************/ |
+/************** Begin file fts3_unicode.c ************************************/ |
+/* |
+** 2012 May 24 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** Implementation of the "unicode" full-text-search tokenizer. |
+*/ |
+ |
+#ifndef SQLITE_DISABLE_FTS3_UNICODE |
+ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+ |
+/* #include <assert.h> */ |
+/* #include <stdlib.h> */ |
+/* #include <stdio.h> */ |
+/* #include <string.h> */ |
+ |
+/* #include "fts3_tokenizer.h" */ |
+ |
+/* |
+** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied |
+** from the sqlite3 source file utf.c. If this file is compiled as part |
+** of the amalgamation, they are not required. |
+*/ |
+#ifndef SQLITE_AMALGAMATION |
+ |
+static const unsigned char sqlite3Utf8Trans1[] = { |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, |
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, |
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, |
+}; |
+ |
+#define READ_UTF8(zIn, zTerm, c) \ |
+ c = *(zIn++); \ |
+ if( c>=0xc0 ){ \ |
+ c = sqlite3Utf8Trans1[c-0xc0]; \ |
+ while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ |
+ c = (c<<6) + (0x3f & *(zIn++)); \ |
+ } \ |
+ if( c<0x80 \ |
+ || (c&0xFFFFF800)==0xD800 \ |
+ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ |
+ } |
+ |
+#define WRITE_UTF8(zOut, c) { \ |
+ if( c<0x00080 ){ \ |
+ *zOut++ = (u8)(c&0xFF); \ |
+ } \ |
+ else if( c<0x00800 ){ \ |
+ *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ |
+ *zOut++ = 0x80 + (u8)(c & 0x3F); \ |
+ } \ |
+ else if( c<0x10000 ){ \ |
+ *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ |
+ *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ |
+ *zOut++ = 0x80 + (u8)(c & 0x3F); \ |
+ }else{ \ |
+ *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ |
+ *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ |
+ *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ |
+ *zOut++ = 0x80 + (u8)(c & 0x3F); \ |
+ } \ |
+} |
+ |
+#endif /* ifndef SQLITE_AMALGAMATION */ |
+ |
+typedef struct unicode_tokenizer unicode_tokenizer; |
+typedef struct unicode_cursor unicode_cursor; |
+ |
+struct unicode_tokenizer { |
+ sqlite3_tokenizer base; |
+ int bRemoveDiacritic; |
+ int nException; |
+ int *aiException; |
+}; |
+ |
+struct unicode_cursor { |
+ sqlite3_tokenizer_cursor base; |
+ const unsigned char *aInput; /* Input text being tokenized */ |
+ int nInput; /* Size of aInput[] in bytes */ |
+ int iOff; /* Current offset within aInput[] */ |
+ int iToken; /* Index of next token to be returned */ |
+ char *zToken; /* storage for current token */ |
+ int nAlloc; /* space allocated at zToken */ |
+}; |
+ |
+ |
+/* |
+** Destroy a tokenizer allocated by unicodeCreate(). |
+*/ |
+static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ |
+ if( pTokenizer ){ |
+ unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; |
+ sqlite3_free(p->aiException); |
+ sqlite3_free(p); |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE |
+** statement has specified that the tokenizer for this table shall consider |
+** all characters in string zIn/nIn to be separators (if bAlnum==0) or |
+** token characters (if bAlnum==1). |
+** |
+** For each codepoint in the zIn/nIn string, this function checks if the |
+** sqlite3FtsUnicodeIsalnum() function already returns the desired result. |
+** If so, no action is taken. Otherwise, the codepoint is added to the |
+** unicode_tokenizer.aiException[] array. For the purposes of tokenization, |
+** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all |
+** codepoints in the aiException[] array. |
+** |
+** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() |
+** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. |
+** It is not possible to change the behavior of the tokenizer with respect |
+** to these codepoints. |
+*/ |
+static int unicodeAddExceptions( |
+ unicode_tokenizer *p, /* Tokenizer to add exceptions to */ |
+ int bAlnum, /* Replace Isalnum() return value with this */ |
+ const char *zIn, /* Array of characters to make exceptions */ |
+ int nIn /* Length of z in bytes */ |
+){ |
+ const unsigned char *z = (const unsigned char *)zIn; |
+ const unsigned char *zTerm = &z[nIn]; |
+ int iCode; |
+ int nEntry = 0; |
+ |
+ assert( bAlnum==0 || bAlnum==1 ); |
+ |
+ while( z<zTerm ){ |
+ READ_UTF8(z, zTerm, iCode); |
+ assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); |
+ if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum |
+ && sqlite3FtsUnicodeIsdiacritic(iCode)==0 |
+ ){ |
+ nEntry++; |
+ } |
+ } |
+ |
+ if( nEntry ){ |
+ int *aNew; /* New aiException[] array */ |
+ int nNew; /* Number of valid entries in array aNew[] */ |
+ |
+ aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int)); |
+ if( aNew==0 ) return SQLITE_NOMEM; |
+ nNew = p->nException; |
+ |
+ z = (const unsigned char *)zIn; |
+ while( z<zTerm ){ |
+ READ_UTF8(z, zTerm, iCode); |
+ if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum |
+ && sqlite3FtsUnicodeIsdiacritic(iCode)==0 |
+ ){ |
+ int i, j; |
+ for(i=0; i<nNew && aNew[i]<iCode; i++); |
+ for(j=nNew; j>i; j--) aNew[j] = aNew[j-1]; |
+ aNew[i] = iCode; |
+ nNew++; |
+ } |
+ } |
+ p->aiException = aNew; |
+ p->nException = nNew; |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Return true if the p->aiException[] array contains the value iCode. |
+*/ |
+static int unicodeIsException(unicode_tokenizer *p, int iCode){ |
+ if( p->nException>0 ){ |
+ int *a = p->aiException; |
+ int iLo = 0; |
+ int iHi = p->nException-1; |
+ |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ if( iCode==a[iTest] ){ |
+ return 1; |
+ }else if( iCode>a[iTest] ){ |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ } |
+ |
+ return 0; |
+} |
+ |
+/* |
+** Return true if, for the purposes of tokenization, codepoint iCode is |
+** considered a token character (not a separator). |
+*/ |
+static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ |
+ assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); |
+ return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); |
+} |
+ |
+/* |
+** Create a new tokenizer instance. |
+*/ |
+static int unicodeCreate( |
+ int nArg, /* Size of array argv[] */ |
+ const char * const *azArg, /* Tokenizer creation arguments */ |
+ sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ |
+){ |
+ unicode_tokenizer *pNew; /* New tokenizer object */ |
+ int i; |
+ int rc = SQLITE_OK; |
+ |
+ pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); |
+ if( pNew==NULL ) return SQLITE_NOMEM; |
+ memset(pNew, 0, sizeof(unicode_tokenizer)); |
+ pNew->bRemoveDiacritic = 1; |
+ |
+ for(i=0; rc==SQLITE_OK && i<nArg; i++){ |
+ const char *z = azArg[i]; |
+ int n = (int)strlen(z); |
+ |
+ if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){ |
+ pNew->bRemoveDiacritic = 1; |
+ } |
+ else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ |
+ pNew->bRemoveDiacritic = 0; |
+ } |
+ else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ |
+ rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); |
+ } |
+ else if( n>=11 && memcmp("separators=", z, 11)==0 ){ |
+ rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); |
+ } |
+ else{ |
+ /* Unrecognized argument */ |
+ rc = SQLITE_ERROR; |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ unicodeDestroy((sqlite3_tokenizer *)pNew); |
+ pNew = 0; |
+ } |
+ *pp = (sqlite3_tokenizer *)pNew; |
+ return rc; |
+} |
+ |
+/* |
+** Prepare to begin tokenizing a particular string. The input |
+** string to be tokenized is pInput[0..nBytes-1]. A cursor |
+** used to incrementally tokenize this string is returned in |
+** *ppCursor. |
+*/ |
+static int unicodeOpen( |
+ sqlite3_tokenizer *p, /* The tokenizer */ |
+ const char *aInput, /* Input string */ |
+ int nInput, /* Size of string aInput in bytes */ |
+ sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ |
+){ |
+ unicode_cursor *pCsr; |
+ |
+ pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); |
+ if( pCsr==0 ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pCsr, 0, sizeof(unicode_cursor)); |
+ |
+ pCsr->aInput = (const unsigned char *)aInput; |
+ if( aInput==0 ){ |
+ pCsr->nInput = 0; |
+ }else if( nInput<0 ){ |
+ pCsr->nInput = (int)strlen(aInput); |
+ }else{ |
+ pCsr->nInput = nInput; |
+ } |
+ |
+ *pp = &pCsr->base; |
+ UNUSED_PARAMETER(p); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Close a tokenization cursor previously opened by a call to |
+** simpleOpen() above. |
+*/ |
+static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ |
+ unicode_cursor *pCsr = (unicode_cursor *) pCursor; |
+ sqlite3_free(pCsr->zToken); |
+ sqlite3_free(pCsr); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Extract the next token from a tokenization cursor. The cursor must |
+** have been opened by a prior call to simpleOpen(). |
+*/ |
+static int unicodeNext( |
+ sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ |
+ const char **paToken, /* OUT: Token text */ |
+ int *pnToken, /* OUT: Number of bytes at *paToken */ |
+ int *piStart, /* OUT: Starting offset of token */ |
+ int *piEnd, /* OUT: Ending offset of token */ |
+ int *piPos /* OUT: Position integer of token */ |
+){ |
+ unicode_cursor *pCsr = (unicode_cursor *)pC; |
+ unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); |
+ int iCode = 0; |
+ char *zOut; |
+ const unsigned char *z = &pCsr->aInput[pCsr->iOff]; |
+ const unsigned char *zStart = z; |
+ const unsigned char *zEnd; |
+ const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; |
+ |
+ /* Scan past any delimiter characters before the start of the next token. |
+ ** Return SQLITE_DONE early if this takes us all the way to the end of |
+ ** the input. */ |
+ while( z<zTerm ){ |
+ READ_UTF8(z, zTerm, iCode); |
+ if( unicodeIsAlnum(p, iCode) ) break; |
+ zStart = z; |
+ } |
+ if( zStart>=zTerm ) return SQLITE_DONE; |
+ |
+ zOut = pCsr->zToken; |
+ do { |
+ int iOut; |
+ |
+ /* Grow the output buffer if required. */ |
+ if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ |
+ char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); |
+ if( !zNew ) return SQLITE_NOMEM; |
+ zOut = &zNew[zOut - pCsr->zToken]; |
+ pCsr->zToken = zNew; |
+ pCsr->nAlloc += 64; |
+ } |
+ |
+ /* Write the folded case of the last character read to the output */ |
+ zEnd = z; |
+ iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); |
+ if( iOut ){ |
+ WRITE_UTF8(zOut, iOut); |
+ } |
+ |
+ /* If the cursor is not at EOF, read the next character */ |
+ if( z>=zTerm ) break; |
+ READ_UTF8(z, zTerm, iCode); |
+ }while( unicodeIsAlnum(p, iCode) |
+ || sqlite3FtsUnicodeIsdiacritic(iCode) |
+ ); |
+ |
+ /* Set the output variables and return. */ |
+ pCsr->iOff = (int)(z - pCsr->aInput); |
+ *paToken = pCsr->zToken; |
+ *pnToken = (int)(zOut - pCsr->zToken); |
+ *piStart = (int)(zStart - pCsr->aInput); |
+ *piEnd = (int)(zEnd - pCsr->aInput); |
+ *piPos = pCsr->iToken++; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Set *ppModule to a pointer to the sqlite3_tokenizer_module |
+** structure for the unicode tokenizer. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ |
+ static const sqlite3_tokenizer_module module = { |
+ 0, |
+ unicodeCreate, |
+ unicodeDestroy, |
+ unicodeOpen, |
+ unicodeClose, |
+ unicodeNext, |
+ 0, |
+ }; |
+ *ppModule = &module; |
+} |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ |
+ |
+/************** End of fts3_unicode.c ****************************************/ |
+/************** Begin file fts3_unicode2.c ***********************************/ |
+/* |
+** 2012 May 25 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+*/ |
+ |
+/* |
+** DO NOT EDIT THIS MACHINE GENERATED FILE. |
+*/ |
+ |
+#ifndef SQLITE_DISABLE_FTS3_UNICODE |
+#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) |
+ |
+/* #include <assert.h> */ |
+ |
+/* |
+** Return true if the argument corresponds to a unicode codepoint |
+** classified as either a letter or a number. Otherwise false. |
+** |
+** The results are undefined if the value passed to this function |
+** is less than zero. |
+*/ |
+SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ |
+ /* Each unsigned integer in the following array corresponds to a contiguous |
+ ** range of unicode codepoints that are not either letters or numbers (i.e. |
+ ** codepoints for which this function should return 0). |
+ ** |
+ ** The most significant 22 bits in each 32-bit value contain the first |
+ ** codepoint in the range. The least significant 10 bits are used to store |
+ ** the size of the range (always at least 1). In other words, the value |
+ ** ((C<<22) + N) represents a range of N codepoints starting with codepoint |
+ ** C. It is not possible to represent a range larger than 1023 codepoints |
+ ** using this format. |
+ */ |
+ static const unsigned int aEntry[] = { |
+ 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, |
+ 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, |
+ 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, |
+ 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, |
+ 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, |
+ 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, |
+ 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, |
+ 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, |
+ 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, |
+ 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, |
+ 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, |
+ 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, |
+ 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, |
+ 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, |
+ 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, |
+ 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, |
+ 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, |
+ 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, |
+ 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, |
+ 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, |
+ 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, |
+ 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, |
+ 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, |
+ 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, |
+ 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, |
+ 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, |
+ 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, |
+ 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, |
+ 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, |
+ 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, |
+ 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, |
+ 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, |
+ 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, |
+ 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, |
+ 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, |
+ 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, |
+ 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, |
+ 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, |
+ 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, |
+ 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, |
+ 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, |
+ 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, |
+ 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, |
+ 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, |
+ 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, |
+ 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, |
+ 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, |
+ 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, |
+ 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, |
+ 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, |
+ 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, |
+ 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, |
+ 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, |
+ 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, |
+ 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, |
+ 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, |
+ 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, |
+ 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, |
+ 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, |
+ 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, |
+ 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, |
+ 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, |
+ 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, |
+ 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, |
+ 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, |
+ 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, |
+ 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, |
+ 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, |
+ 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, |
+ 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, |
+ 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, |
+ 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, |
+ 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, |
+ 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, |
+ 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, |
+ 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, |
+ 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, |
+ 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, |
+ 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, |
+ 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, |
+ 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, |
+ 0x380400F0, |
+ }; |
+ static const unsigned int aAscii[4] = { |
+ 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, |
+ }; |
+ |
+ if( c<128 ){ |
+ return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); |
+ }else if( c<(1<<22) ){ |
+ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; |
+ int iRes = 0; |
+ int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; |
+ int iLo = 0; |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ if( key >= aEntry[iTest] ){ |
+ iRes = iTest; |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ assert( aEntry[0]<key ); |
+ assert( key>=aEntry[iRes] ); |
+ return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); |
+ } |
+ return 1; |
+} |
+ |
+ |
+/* |
+** If the argument is a codepoint corresponding to a lowercase letter |
+** in the ASCII range with a diacritic added, return the codepoint |
+** of the ASCII letter only. For example, if passed 235 - "LATIN |
+** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER |
+** E"). The resuls of passing a codepoint that corresponds to an |
+** uppercase letter are undefined. |
+*/ |
+static int remove_diacritic(int c){ |
+ unsigned short aDia[] = { |
+ 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, |
+ 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, |
+ 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, |
+ 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, |
+ 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, |
+ 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, |
+ 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, |
+ 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, |
+ 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, |
+ 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, |
+ 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, |
+ 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, |
+ 62924, 63050, 63082, 63274, 63390, |
+ }; |
+ char aChar[] = { |
+ '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', |
+ 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', |
+ 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', |
+ 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', |
+ 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', |
+ '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', |
+ 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', |
+ 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', |
+ 'e', 'i', 'o', 'u', 'y', |
+ }; |
+ |
+ unsigned int key = (((unsigned int)c)<<3) | 0x00000007; |
+ int iRes = 0; |
+ int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; |
+ int iLo = 0; |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ if( key >= aDia[iTest] ){ |
+ iRes = iTest; |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ assert( key>=aDia[iRes] ); |
+ return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); |
+} |
+ |
+ |
+/* |
+** Return true if the argument interpreted as a unicode codepoint |
+** is a diacritical modifier character. |
+*/ |
+SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ |
+ unsigned int mask0 = 0x08029FDF; |
+ unsigned int mask1 = 0x000361F8; |
+ if( c<768 || c>817 ) return 0; |
+ return (c < 768+32) ? |
+ (mask0 & (1 << (c-768))) : |
+ (mask1 & (1 << (c-768-32))); |
+} |
+ |
+ |
+/* |
+** Interpret the argument as a unicode codepoint. If the codepoint |
+** is an upper case character that has a lower case equivalent, |
+** return the codepoint corresponding to the lower case version. |
+** Otherwise, return a copy of the argument. |
+** |
+** The results are undefined if the value passed to this function |
+** is less than zero. |
+*/ |
+SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ |
+ /* Each entry in the following array defines a rule for folding a range |
+ ** of codepoints to lower case. The rule applies to a range of nRange |
+ ** codepoints starting at codepoint iCode. |
+ ** |
+ ** If the least significant bit in flags is clear, then the rule applies |
+ ** to all nRange codepoints (i.e. all nRange codepoints are upper case and |
+ ** need to be folded). Or, if it is set, then the rule only applies to |
+ ** every second codepoint in the range, starting with codepoint C. |
+ ** |
+ ** The 7 most significant bits in flags are an index into the aiOff[] |
+ ** array. If a specific codepoint C does require folding, then its lower |
+ ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). |
+ ** |
+ ** The contents of this array are generated by parsing the CaseFolding.txt |
+ ** file distributed as part of the "Unicode Character Database". See |
+ ** http://www.unicode.org for details. |
+ */ |
+ static const struct TableEntry { |
+ unsigned short iCode; |
+ unsigned char flags; |
+ unsigned char nRange; |
+ } aEntry[] = { |
+ {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, |
+ {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, |
+ {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, |
+ {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, |
+ {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, |
+ {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, |
+ {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, |
+ {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, |
+ {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, |
+ {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, |
+ {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, |
+ {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, |
+ {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, |
+ {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, |
+ {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, |
+ {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, |
+ {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, |
+ {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, |
+ {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, |
+ {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, |
+ {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, |
+ {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, |
+ {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, |
+ {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, |
+ {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, |
+ {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, |
+ {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, |
+ {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, |
+ {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, |
+ {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, |
+ {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, |
+ {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, |
+ {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, |
+ {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, |
+ {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, |
+ {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, |
+ {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, |
+ {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, |
+ {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, |
+ {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, |
+ {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, |
+ {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, |
+ {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, |
+ {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, |
+ {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, |
+ {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, |
+ {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, |
+ {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, |
+ {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, |
+ {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, |
+ {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, |
+ {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, |
+ {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, |
+ {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, |
+ {65313, 14, 26}, |
+ }; |
+ static const unsigned short aiOff[] = { |
+ 1, 2, 8, 15, 16, 26, 28, 32, |
+ 37, 38, 40, 48, 63, 64, 69, 71, |
+ 79, 80, 116, 202, 203, 205, 206, 207, |
+ 209, 210, 211, 213, 214, 217, 218, 219, |
+ 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, |
+ 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, |
+ 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, |
+ 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, |
+ 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, |
+ 65514, 65521, 65527, 65528, 65529, |
+ }; |
+ |
+ int ret = c; |
+ |
+ assert( c>=0 ); |
+ assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); |
+ |
+ if( c<128 ){ |
+ if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); |
+ }else if( c<65536 ){ |
+ int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; |
+ int iLo = 0; |
+ int iRes = -1; |
+ |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ int cmp = (c - aEntry[iTest].iCode); |
+ if( cmp>=0 ){ |
+ iRes = iTest; |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ assert( iRes<0 || c>=aEntry[iRes].iCode ); |
+ |
+ if( iRes>=0 ){ |
+ const struct TableEntry *p = &aEntry[iRes]; |
+ if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ |
+ ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; |
+ assert( ret>0 ); |
+ } |
+ } |
+ |
+ if( bRemoveDiacritic ) ret = remove_diacritic(ret); |
+ } |
+ |
+ else if( c>=66560 && c<66600 ){ |
+ ret = c + 40; |
+ } |
+ |
+ return ret; |
+} |
+#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ |
+#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ |
+ |
+/************** End of fts3_unicode2.c ***************************************/ |
+/************** Begin file rtree.c *******************************************/ |
+/* |
+** 2001 September 15 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+************************************************************************* |
+** This file contains code for implementations of the r-tree and r*-tree |
+** algorithms packaged as an SQLite virtual table module. |
+*/ |
+ |
+/* |
+** Database Format of R-Tree Tables |
+** -------------------------------- |
+** |
+** The data structure for a single virtual r-tree table is stored in three |
+** native SQLite tables declared as follows. In each case, the '%' character |
+** in the table name is replaced with the user-supplied name of the r-tree |
+** table. |
+** |
+** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) |
+** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) |
+** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) |
+** |
+** The data for each node of the r-tree structure is stored in the %_node |
+** table. For each node that is not the root node of the r-tree, there is |
+** an entry in the %_parent table associating the node with its parent. |
+** And for each row of data in the table, there is an entry in the %_rowid |
+** table that maps from the entries rowid to the id of the node that it |
+** is stored on. |
+** |
+** The root node of an r-tree always exists, even if the r-tree table is |
+** empty. The nodeno of the root node is always 1. All other nodes in the |
+** table must be the same size as the root node. The content of each node |
+** is formatted as follows: |
+** |
+** 1. If the node is the root node (node 1), then the first 2 bytes |
+** of the node contain the tree depth as a big-endian integer. |
+** For non-root nodes, the first 2 bytes are left unused. |
+** |
+** 2. The next 2 bytes contain the number of entries currently |
+** stored in the node. |
+** |
+** 3. The remainder of the node contains the node entries. Each entry |
+** consists of a single 8-byte integer followed by an even number |
+** of 4-byte coordinates. For leaf nodes the integer is the rowid |
+** of a record. For internal nodes it is the node number of a |
+** child page. |
+*/ |
+ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) |
+ |
+#ifndef SQLITE_CORE |
+/* #include "sqlite3ext.h" */ |
+ SQLITE_EXTENSION_INIT1 |
+#else |
+/* #include "sqlite3.h" */ |
+#endif |
+ |
+/* #include <string.h> */ |
+/* #include <assert.h> */ |
+/* #include <stdio.h> */ |
+ |
+#ifndef SQLITE_AMALGAMATION |
+#include "sqlite3rtree.h" |
+typedef sqlite3_int64 i64; |
+typedef sqlite3_uint64 u64; |
+typedef unsigned char u8; |
+typedef unsigned short u16; |
+typedef unsigned int u32; |
+#endif |
+ |
+/* The following macro is used to suppress compiler warnings. |
+*/ |
+#ifndef UNUSED_PARAMETER |
+# define UNUSED_PARAMETER(x) (void)(x) |
+#endif |
+ |
+typedef struct Rtree Rtree; |
+typedef struct RtreeCursor RtreeCursor; |
+typedef struct RtreeNode RtreeNode; |
+typedef struct RtreeCell RtreeCell; |
+typedef struct RtreeConstraint RtreeConstraint; |
+typedef struct RtreeMatchArg RtreeMatchArg; |
+typedef struct RtreeGeomCallback RtreeGeomCallback; |
+typedef union RtreeCoord RtreeCoord; |
+typedef struct RtreeSearchPoint RtreeSearchPoint; |
+ |
+/* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ |
+#define RTREE_MAX_DIMENSIONS 5 |
+ |
+/* Size of hash table Rtree.aHash. This hash table is not expected to |
+** ever contain very many entries, so a fixed number of buckets is |
+** used. |
+*/ |
+#define HASHSIZE 97 |
+ |
+/* The xBestIndex method of this virtual table requires an estimate of |
+** the number of rows in the virtual table to calculate the costs of |
+** various strategies. If possible, this estimate is loaded from the |
+** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). |
+** Otherwise, if no sqlite_stat1 entry is available, use |
+** RTREE_DEFAULT_ROWEST. |
+*/ |
+#define RTREE_DEFAULT_ROWEST 1048576 |
+#define RTREE_MIN_ROWEST 100 |
+ |
+/* |
+** An rtree virtual-table object. |
+*/ |
+struct Rtree { |
+ sqlite3_vtab base; /* Base class. Must be first */ |
+ sqlite3 *db; /* Host database connection */ |
+ int iNodeSize; /* Size in bytes of each node in the node table */ |
+ u8 nDim; /* Number of dimensions */ |
+ u8 nDim2; /* Twice the number of dimensions */ |
+ u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ |
+ u8 nBytesPerCell; /* Bytes consumed per cell */ |
+ u8 inWrTrans; /* True if inside write transaction */ |
+ int iDepth; /* Current depth of the r-tree structure */ |
+ char *zDb; /* Name of database containing r-tree table */ |
+ char *zName; /* Name of r-tree table */ |
+ u32 nBusy; /* Current number of users of this structure */ |
+ i64 nRowEst; /* Estimated number of rows in this table */ |
+ u32 nCursor; /* Number of open cursors */ |
+ |
+ /* List of nodes removed during a CondenseTree operation. List is |
+ ** linked together via the pointer normally used for hash chains - |
+ ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree |
+ ** headed by the node (leaf nodes have RtreeNode.iNode==0). |
+ */ |
+ RtreeNode *pDeleted; |
+ int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ |
+ |
+ /* Blob I/O on xxx_node */ |
+ sqlite3_blob *pNodeBlob; |
+ |
+ /* Statements to read/write/delete a record from xxx_node */ |
+ sqlite3_stmt *pWriteNode; |
+ sqlite3_stmt *pDeleteNode; |
+ |
+ /* Statements to read/write/delete a record from xxx_rowid */ |
+ sqlite3_stmt *pReadRowid; |
+ sqlite3_stmt *pWriteRowid; |
+ sqlite3_stmt *pDeleteRowid; |
+ |
+ /* Statements to read/write/delete a record from xxx_parent */ |
+ sqlite3_stmt *pReadParent; |
+ sqlite3_stmt *pWriteParent; |
+ sqlite3_stmt *pDeleteParent; |
+ |
+ RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ |
+}; |
+ |
+/* Possible values for Rtree.eCoordType: */ |
+#define RTREE_COORD_REAL32 0 |
+#define RTREE_COORD_INT32 1 |
+ |
+/* |
+** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will |
+** only deal with integer coordinates. No floating point operations |
+** will be done. |
+*/ |
+#ifdef SQLITE_RTREE_INT_ONLY |
+ typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ |
+ typedef int RtreeValue; /* Low accuracy coordinate */ |
+# define RTREE_ZERO 0 |
+#else |
+ typedef double RtreeDValue; /* High accuracy coordinate */ |
+ typedef float RtreeValue; /* Low accuracy coordinate */ |
+# define RTREE_ZERO 0.0 |
+#endif |
+ |
+/* |
+** When doing a search of an r-tree, instances of the following structure |
+** record intermediate results from the tree walk. |
+** |
+** The id is always a node-id. For iLevel>=1 the id is the node-id of |
+** the node that the RtreeSearchPoint represents. When iLevel==0, however, |
+** the id is of the parent node and the cell that RtreeSearchPoint |
+** represents is the iCell-th entry in the parent node. |
+*/ |
+struct RtreeSearchPoint { |
+ RtreeDValue rScore; /* The score for this node. Smallest goes first. */ |
+ sqlite3_int64 id; /* Node ID */ |
+ u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ |
+ u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ |
+ u8 iCell; /* Cell index within the node */ |
+}; |
+ |
+/* |
+** The minimum number of cells allowed for a node is a third of the |
+** maximum. In Gutman's notation: |
+** |
+** m = M/3 |
+** |
+** If an R*-tree "Reinsert" operation is required, the same number of |
+** cells are removed from the overfull node and reinserted into the tree. |
+*/ |
+#define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) |
+#define RTREE_REINSERT(p) RTREE_MINCELLS(p) |
+#define RTREE_MAXCELLS 51 |
+ |
+/* |
+** The smallest possible node-size is (512-64)==448 bytes. And the largest |
+** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). |
+** Therefore all non-root nodes must contain at least 3 entries. Since |
+** 2^40 is greater than 2^64, an r-tree structure always has a depth of |
+** 40 or less. |
+*/ |
+#define RTREE_MAX_DEPTH 40 |
+ |
+ |
+/* |
+** Number of entries in the cursor RtreeNode cache. The first entry is |
+** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining |
+** entries cache the RtreeNode for the first elements of the priority queue. |
+*/ |
+#define RTREE_CACHE_SZ 5 |
+ |
+/* |
+** An rtree cursor object. |
+*/ |
+struct RtreeCursor { |
+ sqlite3_vtab_cursor base; /* Base class. Must be first */ |
+ u8 atEOF; /* True if at end of search */ |
+ u8 bPoint; /* True if sPoint is valid */ |
+ int iStrategy; /* Copy of idxNum search parameter */ |
+ int nConstraint; /* Number of entries in aConstraint */ |
+ RtreeConstraint *aConstraint; /* Search constraints. */ |
+ int nPointAlloc; /* Number of slots allocated for aPoint[] */ |
+ int nPoint; /* Number of slots used in aPoint[] */ |
+ int mxLevel; /* iLevel value for root of the tree */ |
+ RtreeSearchPoint *aPoint; /* Priority queue for search points */ |
+ RtreeSearchPoint sPoint; /* Cached next search point */ |
+ RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ |
+ u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ |
+}; |
+ |
+/* Return the Rtree of a RtreeCursor */ |
+#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) |
+ |
+/* |
+** A coordinate can be either a floating point number or a integer. All |
+** coordinates within a single R-Tree are always of the same time. |
+*/ |
+union RtreeCoord { |
+ RtreeValue f; /* Floating point value */ |
+ int i; /* Integer value */ |
+ u32 u; /* Unsigned for byte-order conversions */ |
+}; |
+ |
+/* |
+** The argument is an RtreeCoord. Return the value stored within the RtreeCoord |
+** formatted as a RtreeDValue (double or int64). This macro assumes that local |
+** variable pRtree points to the Rtree structure associated with the |
+** RtreeCoord. |
+*/ |
+#ifdef SQLITE_RTREE_INT_ONLY |
+# define DCOORD(coord) ((RtreeDValue)coord.i) |
+#else |
+# define DCOORD(coord) ( \ |
+ (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ |
+ ((double)coord.f) : \ |
+ ((double)coord.i) \ |
+ ) |
+#endif |
+ |
+/* |
+** A search constraint. |
+*/ |
+struct RtreeConstraint { |
+ int iCoord; /* Index of constrained coordinate */ |
+ int op; /* Constraining operation */ |
+ union { |
+ RtreeDValue rValue; /* Constraint value. */ |
+ int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); |
+ int (*xQueryFunc)(sqlite3_rtree_query_info*); |
+ } u; |
+ sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ |
+}; |
+ |
+/* Possible values for RtreeConstraint.op */ |
+#define RTREE_EQ 0x41 /* A */ |
+#define RTREE_LE 0x42 /* B */ |
+#define RTREE_LT 0x43 /* C */ |
+#define RTREE_GE 0x44 /* D */ |
+#define RTREE_GT 0x45 /* E */ |
+#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ |
+#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ |
+ |
+ |
+/* |
+** An rtree structure node. |
+*/ |
+struct RtreeNode { |
+ RtreeNode *pParent; /* Parent node */ |
+ i64 iNode; /* The node number */ |
+ int nRef; /* Number of references to this node */ |
+ int isDirty; /* True if the node needs to be written to disk */ |
+ u8 *zData; /* Content of the node, as should be on disk */ |
+ RtreeNode *pNext; /* Next node in this hash collision chain */ |
+}; |
+ |
+/* Return the number of cells in a node */ |
+#define NCELL(pNode) readInt16(&(pNode)->zData[2]) |
+ |
+/* |
+** A single cell from a node, deserialized |
+*/ |
+struct RtreeCell { |
+ i64 iRowid; /* Node or entry ID */ |
+ RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ |
+}; |
+ |
+ |
+/* |
+** This object becomes the sqlite3_user_data() for the SQL functions |
+** that are created by sqlite3_rtree_geometry_callback() and |
+** sqlite3_rtree_query_callback() and which appear on the right of MATCH |
+** operators in order to constrain a search. |
+** |
+** xGeom and xQueryFunc are the callback functions. Exactly one of |
+** xGeom and xQueryFunc fields is non-NULL, depending on whether the |
+** SQL function was created using sqlite3_rtree_geometry_callback() or |
+** sqlite3_rtree_query_callback(). |
+** |
+** This object is deleted automatically by the destructor mechanism in |
+** sqlite3_create_function_v2(). |
+*/ |
+struct RtreeGeomCallback { |
+ int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); |
+ int (*xQueryFunc)(sqlite3_rtree_query_info*); |
+ void (*xDestructor)(void*); |
+ void *pContext; |
+}; |
+ |
+ |
+/* |
+** Value for the first field of every RtreeMatchArg object. The MATCH |
+** operator tests that the first field of a blob operand matches this |
+** value to avoid operating on invalid blobs (which could cause a segfault). |
+*/ |
+#define RTREE_GEOMETRY_MAGIC 0x891245AB |
+ |
+/* |
+** An instance of this structure (in the form of a BLOB) is returned by |
+** the SQL functions that sqlite3_rtree_geometry_callback() and |
+** sqlite3_rtree_query_callback() create, and is read as the right-hand |
+** operand to the MATCH operator of an R-Tree. |
+*/ |
+struct RtreeMatchArg { |
+ u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ |
+ RtreeGeomCallback cb; /* Info about the callback functions */ |
+ int nParam; /* Number of parameters to the SQL function */ |
+ sqlite3_value **apSqlParam; /* Original SQL parameter values */ |
+ RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ |
+}; |
+ |
+#ifndef MAX |
+# define MAX(x,y) ((x) < (y) ? (y) : (x)) |
+#endif |
+#ifndef MIN |
+# define MIN(x,y) ((x) > (y) ? (y) : (x)) |
+#endif |
+ |
+/* What version of GCC is being used. 0 means GCC is not being used */ |
+#ifndef GCC_VERSION |
+#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) |
+# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) |
+#else |
+# define GCC_VERSION 0 |
+#endif |
+#endif |
+ |
+/* What version of CLANG is being used. 0 means CLANG is not being used */ |
+#ifndef CLANG_VERSION |
+#if defined(__clang__) && !defined(_WIN32) && !defined(SQLITE_DISABLE_INTRINSIC) |
+# define CLANG_VERSION \ |
+ (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) |
+#else |
+# define CLANG_VERSION 0 |
+#endif |
+#endif |
+ |
+/* The testcase() macro should already be defined in the amalgamation. If |
+** it is not, make it a no-op. |
+*/ |
+#ifndef SQLITE_AMALGAMATION |
+# define testcase(X) |
+#endif |
+ |
+/* |
+** Macros to determine whether the machine is big or little endian, |
+** and whether or not that determination is run-time or compile-time. |
+** |
+** For best performance, an attempt is made to guess at the byte-order |
+** using C-preprocessor macros. If that is unsuccessful, or if |
+** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined |
+** at run-time. |
+*/ |
+#ifndef SQLITE_BYTEORDER |
+#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ |
+ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ |
+ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ |
+ defined(__arm__) |
+# define SQLITE_BYTEORDER 1234 |
+#elif defined(sparc) || defined(__ppc__) |
+# define SQLITE_BYTEORDER 4321 |
+#else |
+# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ |
+#endif |
+#endif |
+ |
+ |
+/* What version of MSVC is being used. 0 means MSVC is not being used */ |
+#ifndef MSVC_VERSION |
+#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) |
+# define MSVC_VERSION _MSC_VER |
+#else |
+# define MSVC_VERSION 0 |
+#endif |
+#endif |
+ |
+/* |
+** Functions to deserialize a 16 bit integer, 32 bit real number and |
+** 64 bit integer. The deserialized value is returned. |
+*/ |
+static int readInt16(u8 *p){ |
+ return (p[0]<<8) + p[1]; |
+} |
+static void readCoord(u8 *p, RtreeCoord *pCoord){ |
+ assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ |
+#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 |
+ pCoord->u = _byteswap_ulong(*(u32*)p); |
+#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) |
+ pCoord->u = __builtin_bswap32(*(u32*)p); |
+#elif SQLITE_BYTEORDER==4321 |
+ pCoord->u = *(u32*)p; |
+#else |
+ pCoord->u = ( |
+ (((u32)p[0]) << 24) + |
+ (((u32)p[1]) << 16) + |
+ (((u32)p[2]) << 8) + |
+ (((u32)p[3]) << 0) |
+ ); |
+#endif |
+} |
+static i64 readInt64(u8 *p){ |
+#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 |
+ u64 x; |
+ memcpy(&x, p, 8); |
+ return (i64)_byteswap_uint64(x); |
+#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) |
+ u64 x; |
+ memcpy(&x, p, 8); |
+ return (i64)__builtin_bswap64(x); |
+#elif SQLITE_BYTEORDER==4321 |
+ i64 x; |
+ memcpy(&x, p, 8); |
+ return x; |
+#else |
+ return ( |
+ (((i64)p[0]) << 56) + |
+ (((i64)p[1]) << 48) + |
+ (((i64)p[2]) << 40) + |
+ (((i64)p[3]) << 32) + |
+ (((i64)p[4]) << 24) + |
+ (((i64)p[5]) << 16) + |
+ (((i64)p[6]) << 8) + |
+ (((i64)p[7]) << 0) |
+ ); |
+#endif |
+} |
+ |
+/* |
+** Functions to serialize a 16 bit integer, 32 bit real number and |
+** 64 bit integer. The value returned is the number of bytes written |
+** to the argument buffer (always 2, 4 and 8 respectively). |
+*/ |
+static void writeInt16(u8 *p, int i){ |
+ p[0] = (i>> 8)&0xFF; |
+ p[1] = (i>> 0)&0xFF; |
+} |
+static int writeCoord(u8 *p, RtreeCoord *pCoord){ |
+ u32 i; |
+ assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ |
+ assert( sizeof(RtreeCoord)==4 ); |
+ assert( sizeof(u32)==4 ); |
+#if SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) |
+ i = __builtin_bswap32(pCoord->u); |
+ memcpy(p, &i, 4); |
+#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 |
+ i = _byteswap_ulong(pCoord->u); |
+ memcpy(p, &i, 4); |
+#elif SQLITE_BYTEORDER==4321 |
+ i = pCoord->u; |
+ memcpy(p, &i, 4); |
+#else |
+ i = pCoord->u; |
+ p[0] = (i>>24)&0xFF; |
+ p[1] = (i>>16)&0xFF; |
+ p[2] = (i>> 8)&0xFF; |
+ p[3] = (i>> 0)&0xFF; |
+#endif |
+ return 4; |
+} |
+static int writeInt64(u8 *p, i64 i){ |
+#if SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) |
+ i = (i64)__builtin_bswap64((u64)i); |
+ memcpy(p, &i, 8); |
+#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 |
+ i = (i64)_byteswap_uint64((u64)i); |
+ memcpy(p, &i, 8); |
+#elif SQLITE_BYTEORDER==4321 |
+ memcpy(p, &i, 8); |
+#else |
+ p[0] = (i>>56)&0xFF; |
+ p[1] = (i>>48)&0xFF; |
+ p[2] = (i>>40)&0xFF; |
+ p[3] = (i>>32)&0xFF; |
+ p[4] = (i>>24)&0xFF; |
+ p[5] = (i>>16)&0xFF; |
+ p[6] = (i>> 8)&0xFF; |
+ p[7] = (i>> 0)&0xFF; |
+#endif |
+ return 8; |
+} |
+ |
+/* |
+** Increment the reference count of node p. |
+*/ |
+static void nodeReference(RtreeNode *p){ |
+ if( p ){ |
+ p->nRef++; |
+ } |
+} |
+ |
+/* |
+** Clear the content of node p (set all bytes to 0x00). |
+*/ |
+static void nodeZero(Rtree *pRtree, RtreeNode *p){ |
+ memset(&p->zData[2], 0, pRtree->iNodeSize-2); |
+ p->isDirty = 1; |
+} |
+ |
+/* |
+** Given a node number iNode, return the corresponding key to use |
+** in the Rtree.aHash table. |
+*/ |
+static int nodeHash(i64 iNode){ |
+ return iNode % HASHSIZE; |
+} |
+ |
+/* |
+** Search the node hash table for node iNode. If found, return a pointer |
+** to it. Otherwise, return 0. |
+*/ |
+static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ |
+ RtreeNode *p; |
+ for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); |
+ return p; |
+} |
+ |
+/* |
+** Add node pNode to the node hash table. |
+*/ |
+static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ |
+ int iHash; |
+ assert( pNode->pNext==0 ); |
+ iHash = nodeHash(pNode->iNode); |
+ pNode->pNext = pRtree->aHash[iHash]; |
+ pRtree->aHash[iHash] = pNode; |
+} |
+ |
+/* |
+** Remove node pNode from the node hash table. |
+*/ |
+static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ |
+ RtreeNode **pp; |
+ if( pNode->iNode!=0 ){ |
+ pp = &pRtree->aHash[nodeHash(pNode->iNode)]; |
+ for( ; (*pp)!=pNode; pp = &(*pp)->pNext){ assert(*pp); } |
+ *pp = pNode->pNext; |
+ pNode->pNext = 0; |
+ } |
+} |
+ |
+/* |
+** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), |
+** indicating that node has not yet been assigned a node number. It is |
+** assigned a node number when nodeWrite() is called to write the |
+** node contents out to the database. |
+*/ |
+static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ |
+ RtreeNode *pNode; |
+ pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); |
+ if( pNode ){ |
+ memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); |
+ pNode->zData = (u8 *)&pNode[1]; |
+ pNode->nRef = 1; |
+ pNode->pParent = pParent; |
+ pNode->isDirty = 1; |
+ nodeReference(pParent); |
+ } |
+ return pNode; |
+} |
+ |
+/* |
+** Clear the Rtree.pNodeBlob object |
+*/ |
+static void nodeBlobReset(Rtree *pRtree){ |
+ if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ |
+ sqlite3_blob *pBlob = pRtree->pNodeBlob; |
+ pRtree->pNodeBlob = 0; |
+ sqlite3_blob_close(pBlob); |
+ } |
+} |
+ |
+/* |
+** Obtain a reference to an r-tree node. |
+*/ |
+static int nodeAcquire( |
+ Rtree *pRtree, /* R-tree structure */ |
+ i64 iNode, /* Node number to load */ |
+ RtreeNode *pParent, /* Either the parent node or NULL */ |
+ RtreeNode **ppNode /* OUT: Acquired node */ |
+){ |
+ int rc = SQLITE_OK; |
+ RtreeNode *pNode = 0; |
+ |
+ /* Check if the requested node is already in the hash table. If so, |
+ ** increase its reference count and return it. |
+ */ |
+ if( (pNode = nodeHashLookup(pRtree, iNode)) ){ |
+ assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); |
+ if( pParent && !pNode->pParent ){ |
+ nodeReference(pParent); |
+ pNode->pParent = pParent; |
+ } |
+ pNode->nRef++; |
+ *ppNode = pNode; |
+ return SQLITE_OK; |
+ } |
+ |
+ if( pRtree->pNodeBlob ){ |
+ sqlite3_blob *pBlob = pRtree->pNodeBlob; |
+ pRtree->pNodeBlob = 0; |
+ rc = sqlite3_blob_reopen(pBlob, iNode); |
+ pRtree->pNodeBlob = pBlob; |
+ if( rc ){ |
+ nodeBlobReset(pRtree); |
+ if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; |
+ } |
+ } |
+ if( pRtree->pNodeBlob==0 ){ |
+ char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); |
+ if( zTab==0 ) return SQLITE_NOMEM; |
+ rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, |
+ &pRtree->pNodeBlob); |
+ sqlite3_free(zTab); |
+ } |
+ if( rc ){ |
+ nodeBlobReset(pRtree); |
+ *ppNode = 0; |
+ /* If unable to open an sqlite3_blob on the desired row, that can only |
+ ** be because the shadow tables hold erroneous data. */ |
+ if( rc==SQLITE_ERROR ) rc = SQLITE_CORRUPT_VTAB; |
+ }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ |
+ pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); |
+ if( !pNode ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ pNode->pParent = pParent; |
+ pNode->zData = (u8 *)&pNode[1]; |
+ pNode->nRef = 1; |
+ pNode->iNode = iNode; |
+ pNode->isDirty = 0; |
+ pNode->pNext = 0; |
+ rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, |
+ pRtree->iNodeSize, 0); |
+ nodeReference(pParent); |
+ } |
+ } |
+ |
+ /* If the root node was just loaded, set pRtree->iDepth to the height |
+ ** of the r-tree structure. A height of zero means all data is stored on |
+ ** the root node. A height of one means the children of the root node |
+ ** are the leaves, and so on. If the depth as specified on the root node |
+ ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. |
+ */ |
+ if( pNode && iNode==1 ){ |
+ pRtree->iDepth = readInt16(pNode->zData); |
+ if( pRtree->iDepth>RTREE_MAX_DEPTH ){ |
+ rc = SQLITE_CORRUPT_VTAB; |
+ } |
+ } |
+ |
+ /* If no error has occurred so far, check if the "number of entries" |
+ ** field on the node is too large. If so, set the return code to |
+ ** SQLITE_CORRUPT_VTAB. |
+ */ |
+ if( pNode && rc==SQLITE_OK ){ |
+ if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ |
+ rc = SQLITE_CORRUPT_VTAB; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ if( pNode!=0 ){ |
+ nodeHashInsert(pRtree, pNode); |
+ }else{ |
+ rc = SQLITE_CORRUPT_VTAB; |
+ } |
+ *ppNode = pNode; |
+ }else{ |
+ sqlite3_free(pNode); |
+ *ppNode = 0; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Overwrite cell iCell of node pNode with the contents of pCell. |
+*/ |
+static void nodeOverwriteCell( |
+ Rtree *pRtree, /* The overall R-Tree */ |
+ RtreeNode *pNode, /* The node into which the cell is to be written */ |
+ RtreeCell *pCell, /* The cell to write */ |
+ int iCell /* Index into pNode into which pCell is written */ |
+){ |
+ int ii; |
+ u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; |
+ p += writeInt64(p, pCell->iRowid); |
+ for(ii=0; ii<pRtree->nDim2; ii++){ |
+ p += writeCoord(p, &pCell->aCoord[ii]); |
+ } |
+ pNode->isDirty = 1; |
+} |
+ |
+/* |
+** Remove the cell with index iCell from node pNode. |
+*/ |
+static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ |
+ u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; |
+ u8 *pSrc = &pDst[pRtree->nBytesPerCell]; |
+ int nByte = (NCELL(pNode) - iCell - 1) * pRtree->nBytesPerCell; |
+ memmove(pDst, pSrc, nByte); |
+ writeInt16(&pNode->zData[2], NCELL(pNode)-1); |
+ pNode->isDirty = 1; |
+} |
+ |
+/* |
+** Insert the contents of cell pCell into node pNode. If the insert |
+** is successful, return SQLITE_OK. |
+** |
+** If there is not enough free space in pNode, return SQLITE_FULL. |
+*/ |
+static int nodeInsertCell( |
+ Rtree *pRtree, /* The overall R-Tree */ |
+ RtreeNode *pNode, /* Write new cell into this node */ |
+ RtreeCell *pCell /* The cell to be inserted */ |
+){ |
+ int nCell; /* Current number of cells in pNode */ |
+ int nMaxCell; /* Maximum number of cells for pNode */ |
+ |
+ nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; |
+ nCell = NCELL(pNode); |
+ |
+ assert( nCell<=nMaxCell ); |
+ if( nCell<nMaxCell ){ |
+ nodeOverwriteCell(pRtree, pNode, pCell, nCell); |
+ writeInt16(&pNode->zData[2], nCell+1); |
+ pNode->isDirty = 1; |
+ } |
+ |
+ return (nCell==nMaxCell); |
+} |
+ |
+/* |
+** If the node is dirty, write it out to the database. |
+*/ |
+static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ |
+ int rc = SQLITE_OK; |
+ if( pNode->isDirty ){ |
+ sqlite3_stmt *p = pRtree->pWriteNode; |
+ if( pNode->iNode ){ |
+ sqlite3_bind_int64(p, 1, pNode->iNode); |
+ }else{ |
+ sqlite3_bind_null(p, 1); |
+ } |
+ sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC); |
+ sqlite3_step(p); |
+ pNode->isDirty = 0; |
+ rc = sqlite3_reset(p); |
+ if( pNode->iNode==0 && rc==SQLITE_OK ){ |
+ pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); |
+ nodeHashInsert(pRtree, pNode); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Release a reference to a node. If the node is dirty and the reference |
+** count drops to zero, the node data is written to the database. |
+*/ |
+static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ |
+ int rc = SQLITE_OK; |
+ if( pNode ){ |
+ assert( pNode->nRef>0 ); |
+ pNode->nRef--; |
+ if( pNode->nRef==0 ){ |
+ if( pNode->iNode==1 ){ |
+ pRtree->iDepth = -1; |
+ } |
+ if( pNode->pParent ){ |
+ rc = nodeRelease(pRtree, pNode->pParent); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = nodeWrite(pRtree, pNode); |
+ } |
+ nodeHashDelete(pRtree, pNode); |
+ sqlite3_free(pNode); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Return the 64-bit integer value associated with cell iCell of |
+** node pNode. If pNode is a leaf node, this is a rowid. If it is |
+** an internal node, then the 64-bit integer is a child page number. |
+*/ |
+static i64 nodeGetRowid( |
+ Rtree *pRtree, /* The overall R-Tree */ |
+ RtreeNode *pNode, /* The node from which to extract the ID */ |
+ int iCell /* The cell index from which to extract the ID */ |
+){ |
+ assert( iCell<NCELL(pNode) ); |
+ return readInt64(&pNode->zData[4 + pRtree->nBytesPerCell*iCell]); |
+} |
+ |
+/* |
+** Return coordinate iCoord from cell iCell in node pNode. |
+*/ |
+static void nodeGetCoord( |
+ Rtree *pRtree, /* The overall R-Tree */ |
+ RtreeNode *pNode, /* The node from which to extract a coordinate */ |
+ int iCell, /* The index of the cell within the node */ |
+ int iCoord, /* Which coordinate to extract */ |
+ RtreeCoord *pCoord /* OUT: Space to write result to */ |
+){ |
+ readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); |
+} |
+ |
+/* |
+** Deserialize cell iCell of node pNode. Populate the structure pointed |
+** to by pCell with the results. |
+*/ |
+static void nodeGetCell( |
+ Rtree *pRtree, /* The overall R-Tree */ |
+ RtreeNode *pNode, /* The node containing the cell to be read */ |
+ int iCell, /* Index of the cell within the node */ |
+ RtreeCell *pCell /* OUT: Write the cell contents here */ |
+){ |
+ u8 *pData; |
+ RtreeCoord *pCoord; |
+ int ii = 0; |
+ pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); |
+ pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); |
+ pCoord = pCell->aCoord; |
+ do{ |
+ readCoord(pData, &pCoord[ii]); |
+ readCoord(pData+4, &pCoord[ii+1]); |
+ pData += 8; |
+ ii += 2; |
+ }while( ii<pRtree->nDim2 ); |
+} |
+ |
+ |
+/* Forward declaration for the function that does the work of |
+** the virtual table module xCreate() and xConnect() methods. |
+*/ |
+static int rtreeInit( |
+ sqlite3 *, void *, int, const char *const*, sqlite3_vtab **, char **, int |
+); |
+ |
+/* |
+** Rtree virtual table module xCreate method. |
+*/ |
+static int rtreeCreate( |
+ sqlite3 *db, |
+ void *pAux, |
+ int argc, const char *const*argv, |
+ sqlite3_vtab **ppVtab, |
+ char **pzErr |
+){ |
+ return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 1); |
+} |
+ |
+/* |
+** Rtree virtual table module xConnect method. |
+*/ |
+static int rtreeConnect( |
+ sqlite3 *db, |
+ void *pAux, |
+ int argc, const char *const*argv, |
+ sqlite3_vtab **ppVtab, |
+ char **pzErr |
+){ |
+ return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 0); |
+} |
+ |
+/* |
+** Increment the r-tree reference count. |
+*/ |
+static void rtreeReference(Rtree *pRtree){ |
+ pRtree->nBusy++; |
+} |
+ |
+/* |
+** Decrement the r-tree reference count. When the reference count reaches |
+** zero the structure is deleted. |
+*/ |
+static void rtreeRelease(Rtree *pRtree){ |
+ pRtree->nBusy--; |
+ if( pRtree->nBusy==0 ){ |
+ pRtree->inWrTrans = 0; |
+ pRtree->nCursor = 0; |
+ nodeBlobReset(pRtree); |
+ sqlite3_finalize(pRtree->pWriteNode); |
+ sqlite3_finalize(pRtree->pDeleteNode); |
+ sqlite3_finalize(pRtree->pReadRowid); |
+ sqlite3_finalize(pRtree->pWriteRowid); |
+ sqlite3_finalize(pRtree->pDeleteRowid); |
+ sqlite3_finalize(pRtree->pReadParent); |
+ sqlite3_finalize(pRtree->pWriteParent); |
+ sqlite3_finalize(pRtree->pDeleteParent); |
+ sqlite3_free(pRtree); |
+ } |
+} |
+ |
+/* |
+** Rtree virtual table module xDisconnect method. |
+*/ |
+static int rtreeDisconnect(sqlite3_vtab *pVtab){ |
+ rtreeRelease((Rtree *)pVtab); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Rtree virtual table module xDestroy method. |
+*/ |
+static int rtreeDestroy(sqlite3_vtab *pVtab){ |
+ Rtree *pRtree = (Rtree *)pVtab; |
+ int rc; |
+ char *zCreate = sqlite3_mprintf( |
+ "DROP TABLE '%q'.'%q_node';" |
+ "DROP TABLE '%q'.'%q_rowid';" |
+ "DROP TABLE '%q'.'%q_parent';", |
+ pRtree->zDb, pRtree->zName, |
+ pRtree->zDb, pRtree->zName, |
+ pRtree->zDb, pRtree->zName |
+ ); |
+ if( !zCreate ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ nodeBlobReset(pRtree); |
+ rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); |
+ sqlite3_free(zCreate); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rtreeRelease(pRtree); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Rtree virtual table module xOpen method. |
+*/ |
+static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ |
+ int rc = SQLITE_NOMEM; |
+ Rtree *pRtree = (Rtree *)pVTab; |
+ RtreeCursor *pCsr; |
+ |
+ pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); |
+ if( pCsr ){ |
+ memset(pCsr, 0, sizeof(RtreeCursor)); |
+ pCsr->base.pVtab = pVTab; |
+ rc = SQLITE_OK; |
+ pRtree->nCursor++; |
+ } |
+ *ppCursor = (sqlite3_vtab_cursor *)pCsr; |
+ |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Free the RtreeCursor.aConstraint[] array and its contents. |
+*/ |
+static void freeCursorConstraints(RtreeCursor *pCsr){ |
+ if( pCsr->aConstraint ){ |
+ int i; /* Used to iterate through constraint array */ |
+ for(i=0; i<pCsr->nConstraint; i++){ |
+ sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; |
+ if( pInfo ){ |
+ if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); |
+ sqlite3_free(pInfo); |
+ } |
+ } |
+ sqlite3_free(pCsr->aConstraint); |
+ pCsr->aConstraint = 0; |
+ } |
+} |
+ |
+/* |
+** Rtree virtual table module xClose method. |
+*/ |
+static int rtreeClose(sqlite3_vtab_cursor *cur){ |
+ Rtree *pRtree = (Rtree *)(cur->pVtab); |
+ int ii; |
+ RtreeCursor *pCsr = (RtreeCursor *)cur; |
+ assert( pRtree->nCursor>0 ); |
+ freeCursorConstraints(pCsr); |
+ sqlite3_free(pCsr->aPoint); |
+ for(ii=0; ii<RTREE_CACHE_SZ; ii++) nodeRelease(pRtree, pCsr->aNode[ii]); |
+ sqlite3_free(pCsr); |
+ pRtree->nCursor--; |
+ nodeBlobReset(pRtree); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Rtree virtual table module xEof method. |
+** |
+** Return non-zero if the cursor does not currently point to a valid |
+** record (i.e if the scan has finished), or zero otherwise. |
+*/ |
+static int rtreeEof(sqlite3_vtab_cursor *cur){ |
+ RtreeCursor *pCsr = (RtreeCursor *)cur; |
+ return pCsr->atEOF; |
+} |
+ |
+/* |
+** Convert raw bits from the on-disk RTree record into a coordinate value. |
+** The on-disk format is big-endian and needs to be converted for little- |
+** endian platforms. The on-disk record stores integer coordinates if |
+** eInt is true and it stores 32-bit floating point records if eInt is |
+** false. a[] is the four bytes of the on-disk record to be decoded. |
+** Store the results in "r". |
+** |
+** There are five versions of this macro. The last one is generic. The |
+** other four are various architectures-specific optimizations. |
+*/ |
+#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 |
+#define RTREE_DECODE_COORD(eInt, a, r) { \ |
+ RtreeCoord c; /* Coordinate decoded */ \ |
+ c.u = _byteswap_ulong(*(u32*)a); \ |
+ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ |
+} |
+#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) |
+#define RTREE_DECODE_COORD(eInt, a, r) { \ |
+ RtreeCoord c; /* Coordinate decoded */ \ |
+ c.u = __builtin_bswap32(*(u32*)a); \ |
+ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ |
+} |
+#elif SQLITE_BYTEORDER==1234 |
+#define RTREE_DECODE_COORD(eInt, a, r) { \ |
+ RtreeCoord c; /* Coordinate decoded */ \ |
+ memcpy(&c.u,a,4); \ |
+ c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ |
+ ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ |
+ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ |
+} |
+#elif SQLITE_BYTEORDER==4321 |
+#define RTREE_DECODE_COORD(eInt, a, r) { \ |
+ RtreeCoord c; /* Coordinate decoded */ \ |
+ memcpy(&c.u,a,4); \ |
+ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ |
+} |
+#else |
+#define RTREE_DECODE_COORD(eInt, a, r) { \ |
+ RtreeCoord c; /* Coordinate decoded */ \ |
+ c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ |
+ +((u32)a[2]<<8) + a[3]; \ |
+ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ |
+} |
+#endif |
+ |
+/* |
+** Check the RTree node or entry given by pCellData and p against the MATCH |
+** constraint pConstraint. |
+*/ |
+static int rtreeCallbackConstraint( |
+ RtreeConstraint *pConstraint, /* The constraint to test */ |
+ int eInt, /* True if RTree holding integer coordinates */ |
+ u8 *pCellData, /* Raw cell content */ |
+ RtreeSearchPoint *pSearch, /* Container of this cell */ |
+ sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ |
+ int *peWithin /* OUT: visibility of the cell */ |
+){ |
+ sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ |
+ int nCoord = pInfo->nCoord; /* No. of coordinates */ |
+ int rc; /* Callback return code */ |
+ RtreeCoord c; /* Translator union */ |
+ sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ |
+ |
+ assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); |
+ assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); |
+ |
+ if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ |
+ pInfo->iRowid = readInt64(pCellData); |
+ } |
+ pCellData += 8; |
+#ifndef SQLITE_RTREE_INT_ONLY |
+ if( eInt==0 ){ |
+ switch( nCoord ){ |
+ case 10: readCoord(pCellData+36, &c); aCoord[9] = c.f; |
+ readCoord(pCellData+32, &c); aCoord[8] = c.f; |
+ case 8: readCoord(pCellData+28, &c); aCoord[7] = c.f; |
+ readCoord(pCellData+24, &c); aCoord[6] = c.f; |
+ case 6: readCoord(pCellData+20, &c); aCoord[5] = c.f; |
+ readCoord(pCellData+16, &c); aCoord[4] = c.f; |
+ case 4: readCoord(pCellData+12, &c); aCoord[3] = c.f; |
+ readCoord(pCellData+8, &c); aCoord[2] = c.f; |
+ default: readCoord(pCellData+4, &c); aCoord[1] = c.f; |
+ readCoord(pCellData, &c); aCoord[0] = c.f; |
+ } |
+ }else |
+#endif |
+ { |
+ switch( nCoord ){ |
+ case 10: readCoord(pCellData+36, &c); aCoord[9] = c.i; |
+ readCoord(pCellData+32, &c); aCoord[8] = c.i; |
+ case 8: readCoord(pCellData+28, &c); aCoord[7] = c.i; |
+ readCoord(pCellData+24, &c); aCoord[6] = c.i; |
+ case 6: readCoord(pCellData+20, &c); aCoord[5] = c.i; |
+ readCoord(pCellData+16, &c); aCoord[4] = c.i; |
+ case 4: readCoord(pCellData+12, &c); aCoord[3] = c.i; |
+ readCoord(pCellData+8, &c); aCoord[2] = c.i; |
+ default: readCoord(pCellData+4, &c); aCoord[1] = c.i; |
+ readCoord(pCellData, &c); aCoord[0] = c.i; |
+ } |
+ } |
+ if( pConstraint->op==RTREE_MATCH ){ |
+ int eWithin = 0; |
+ rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, |
+ nCoord, aCoord, &eWithin); |
+ if( eWithin==0 ) *peWithin = NOT_WITHIN; |
+ *prScore = RTREE_ZERO; |
+ }else{ |
+ pInfo->aCoord = aCoord; |
+ pInfo->iLevel = pSearch->iLevel - 1; |
+ pInfo->rScore = pInfo->rParentScore = pSearch->rScore; |
+ pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; |
+ rc = pConstraint->u.xQueryFunc(pInfo); |
+ if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; |
+ if( pInfo->rScore<*prScore || *prScore<RTREE_ZERO ){ |
+ *prScore = pInfo->rScore; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Check the internal RTree node given by pCellData against constraint p. |
+** If this constraint cannot be satisfied by any child within the node, |
+** set *peWithin to NOT_WITHIN. |
+*/ |
+static void rtreeNonleafConstraint( |
+ RtreeConstraint *p, /* The constraint to test */ |
+ int eInt, /* True if RTree holds integer coordinates */ |
+ u8 *pCellData, /* Raw cell content as appears on disk */ |
+ int *peWithin /* Adjust downward, as appropriate */ |
+){ |
+ sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ |
+ |
+ /* p->iCoord might point to either a lower or upper bound coordinate |
+ ** in a coordinate pair. But make pCellData point to the lower bound. |
+ */ |
+ pCellData += 8 + 4*(p->iCoord&0xfe); |
+ |
+ assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE |
+ || p->op==RTREE_GT || p->op==RTREE_EQ ); |
+ assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ |
+ switch( p->op ){ |
+ case RTREE_LE: |
+ case RTREE_LT: |
+ case RTREE_EQ: |
+ RTREE_DECODE_COORD(eInt, pCellData, val); |
+ /* val now holds the lower bound of the coordinate pair */ |
+ if( p->u.rValue>=val ) return; |
+ if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ |
+ /* Fall through for the RTREE_EQ case */ |
+ |
+ default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ |
+ pCellData += 4; |
+ RTREE_DECODE_COORD(eInt, pCellData, val); |
+ /* val now holds the upper bound of the coordinate pair */ |
+ if( p->u.rValue<=val ) return; |
+ } |
+ *peWithin = NOT_WITHIN; |
+} |
+ |
+/* |
+** Check the leaf RTree cell given by pCellData against constraint p. |
+** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. |
+** If the constraint is satisfied, leave *peWithin unchanged. |
+** |
+** The constraint is of the form: xN op $val |
+** |
+** The op is given by p->op. The xN is p->iCoord-th coordinate in |
+** pCellData. $val is given by p->u.rValue. |
+*/ |
+static void rtreeLeafConstraint( |
+ RtreeConstraint *p, /* The constraint to test */ |
+ int eInt, /* True if RTree holds integer coordinates */ |
+ u8 *pCellData, /* Raw cell content as appears on disk */ |
+ int *peWithin /* Adjust downward, as appropriate */ |
+){ |
+ RtreeDValue xN; /* Coordinate value converted to a double */ |
+ |
+ assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE |
+ || p->op==RTREE_GT || p->op==RTREE_EQ ); |
+ pCellData += 8 + p->iCoord*4; |
+ assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ |
+ RTREE_DECODE_COORD(eInt, pCellData, xN); |
+ switch( p->op ){ |
+ case RTREE_LE: if( xN <= p->u.rValue ) return; break; |
+ case RTREE_LT: if( xN < p->u.rValue ) return; break; |
+ case RTREE_GE: if( xN >= p->u.rValue ) return; break; |
+ case RTREE_GT: if( xN > p->u.rValue ) return; break; |
+ default: if( xN == p->u.rValue ) return; break; |
+ } |
+ *peWithin = NOT_WITHIN; |
+} |
+ |
+/* |
+** One of the cells in node pNode is guaranteed to have a 64-bit |
+** integer value equal to iRowid. Return the index of this cell. |
+*/ |
+static int nodeRowidIndex( |
+ Rtree *pRtree, |
+ RtreeNode *pNode, |
+ i64 iRowid, |
+ int *piIndex |
+){ |
+ int ii; |
+ int nCell = NCELL(pNode); |
+ assert( nCell<200 ); |
+ for(ii=0; ii<nCell; ii++){ |
+ if( nodeGetRowid(pRtree, pNode, ii)==iRowid ){ |
+ *piIndex = ii; |
+ return SQLITE_OK; |
+ } |
+ } |
+ return SQLITE_CORRUPT_VTAB; |
+} |
+ |
+/* |
+** Return the index of the cell containing a pointer to node pNode |
+** in its parent. If pNode is the root node, return -1. |
+*/ |
+static int nodeParentIndex(Rtree *pRtree, RtreeNode *pNode, int *piIndex){ |
+ RtreeNode *pParent = pNode->pParent; |
+ if( pParent ){ |
+ return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); |
+ } |
+ *piIndex = -1; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Compare two search points. Return negative, zero, or positive if the first |
+** is less than, equal to, or greater than the second. |
+** |
+** The rScore is the primary key. Smaller rScore values come first. |
+** If the rScore is a tie, then use iLevel as the tie breaker with smaller |
+** iLevel values coming first. In this way, if rScore is the same for all |
+** SearchPoints, then iLevel becomes the deciding factor and the result |
+** is a depth-first search, which is the desired default behavior. |
+*/ |
+static int rtreeSearchPointCompare( |
+ const RtreeSearchPoint *pA, |
+ const RtreeSearchPoint *pB |
+){ |
+ if( pA->rScore<pB->rScore ) return -1; |
+ if( pA->rScore>pB->rScore ) return +1; |
+ if( pA->iLevel<pB->iLevel ) return -1; |
+ if( pA->iLevel>pB->iLevel ) return +1; |
+ return 0; |
+} |
+ |
+/* |
+** Interchange two search points in a cursor. |
+*/ |
+static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ |
+ RtreeSearchPoint t = p->aPoint[i]; |
+ assert( i<j ); |
+ p->aPoint[i] = p->aPoint[j]; |
+ p->aPoint[j] = t; |
+ i++; j++; |
+ if( i<RTREE_CACHE_SZ ){ |
+ if( j>=RTREE_CACHE_SZ ){ |
+ nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); |
+ p->aNode[i] = 0; |
+ }else{ |
+ RtreeNode *pTemp = p->aNode[i]; |
+ p->aNode[i] = p->aNode[j]; |
+ p->aNode[j] = pTemp; |
+ } |
+ } |
+} |
+ |
+/* |
+** Return the search point with the lowest current score. |
+*/ |
+static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ |
+ return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; |
+} |
+ |
+/* |
+** Get the RtreeNode for the search point with the lowest score. |
+*/ |
+static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ |
+ sqlite3_int64 id; |
+ int ii = 1 - pCur->bPoint; |
+ assert( ii==0 || ii==1 ); |
+ assert( pCur->bPoint || pCur->nPoint ); |
+ if( pCur->aNode[ii]==0 ){ |
+ assert( pRC!=0 ); |
+ id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; |
+ *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); |
+ } |
+ return pCur->aNode[ii]; |
+} |
+ |
+/* |
+** Push a new element onto the priority queue |
+*/ |
+static RtreeSearchPoint *rtreeEnqueue( |
+ RtreeCursor *pCur, /* The cursor */ |
+ RtreeDValue rScore, /* Score for the new search point */ |
+ u8 iLevel /* Level for the new search point */ |
+){ |
+ int i, j; |
+ RtreeSearchPoint *pNew; |
+ if( pCur->nPoint>=pCur->nPointAlloc ){ |
+ int nNew = pCur->nPointAlloc*2 + 8; |
+ pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); |
+ if( pNew==0 ) return 0; |
+ pCur->aPoint = pNew; |
+ pCur->nPointAlloc = nNew; |
+ } |
+ i = pCur->nPoint++; |
+ pNew = pCur->aPoint + i; |
+ pNew->rScore = rScore; |
+ pNew->iLevel = iLevel; |
+ assert( iLevel<=RTREE_MAX_DEPTH ); |
+ while( i>0 ){ |
+ RtreeSearchPoint *pParent; |
+ j = (i-1)/2; |
+ pParent = pCur->aPoint + j; |
+ if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; |
+ rtreeSearchPointSwap(pCur, j, i); |
+ i = j; |
+ pNew = pParent; |
+ } |
+ return pNew; |
+} |
+ |
+/* |
+** Allocate a new RtreeSearchPoint and return a pointer to it. Return |
+** NULL if malloc fails. |
+*/ |
+static RtreeSearchPoint *rtreeSearchPointNew( |
+ RtreeCursor *pCur, /* The cursor */ |
+ RtreeDValue rScore, /* Score for the new search point */ |
+ u8 iLevel /* Level for the new search point */ |
+){ |
+ RtreeSearchPoint *pNew, *pFirst; |
+ pFirst = rtreeSearchPointFirst(pCur); |
+ pCur->anQueue[iLevel]++; |
+ if( pFirst==0 |
+ || pFirst->rScore>rScore |
+ || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) |
+ ){ |
+ if( pCur->bPoint ){ |
+ int ii; |
+ pNew = rtreeEnqueue(pCur, rScore, iLevel); |
+ if( pNew==0 ) return 0; |
+ ii = (int)(pNew - pCur->aPoint) + 1; |
+ if( ii<RTREE_CACHE_SZ ){ |
+ assert( pCur->aNode[ii]==0 ); |
+ pCur->aNode[ii] = pCur->aNode[0]; |
+ }else{ |
+ nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); |
+ } |
+ pCur->aNode[0] = 0; |
+ *pNew = pCur->sPoint; |
+ } |
+ pCur->sPoint.rScore = rScore; |
+ pCur->sPoint.iLevel = iLevel; |
+ pCur->bPoint = 1; |
+ return &pCur->sPoint; |
+ }else{ |
+ return rtreeEnqueue(pCur, rScore, iLevel); |
+ } |
+} |
+ |
+#if 0 |
+/* Tracing routines for the RtreeSearchPoint queue */ |
+static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ |
+ if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } |
+ printf(" %d.%05lld.%02d %g %d", |
+ p->iLevel, p->id, p->iCell, p->rScore, p->eWithin |
+ ); |
+ idx++; |
+ if( idx<RTREE_CACHE_SZ ){ |
+ printf(" %p\n", pCur->aNode[idx]); |
+ }else{ |
+ printf("\n"); |
+ } |
+} |
+static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ |
+ int ii; |
+ printf("=== %9s ", zPrefix); |
+ if( pCur->bPoint ){ |
+ tracePoint(&pCur->sPoint, -1, pCur); |
+ } |
+ for(ii=0; ii<pCur->nPoint; ii++){ |
+ if( ii>0 || pCur->bPoint ) printf(" "); |
+ tracePoint(&pCur->aPoint[ii], ii, pCur); |
+ } |
+} |
+# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) |
+#else |
+# define RTREE_QUEUE_TRACE(A,B) /* no-op */ |
+#endif |
+ |
+/* Remove the search point with the lowest current score. |
+*/ |
+static void rtreeSearchPointPop(RtreeCursor *p){ |
+ int i, j, k, n; |
+ i = 1 - p->bPoint; |
+ assert( i==0 || i==1 ); |
+ if( p->aNode[i] ){ |
+ nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); |
+ p->aNode[i] = 0; |
+ } |
+ if( p->bPoint ){ |
+ p->anQueue[p->sPoint.iLevel]--; |
+ p->bPoint = 0; |
+ }else if( p->nPoint ){ |
+ p->anQueue[p->aPoint[0].iLevel]--; |
+ n = --p->nPoint; |
+ p->aPoint[0] = p->aPoint[n]; |
+ if( n<RTREE_CACHE_SZ-1 ){ |
+ p->aNode[1] = p->aNode[n+1]; |
+ p->aNode[n+1] = 0; |
+ } |
+ i = 0; |
+ while( (j = i*2+1)<n ){ |
+ k = j+1; |
+ if( k<n && rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[j])<0 ){ |
+ if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ |
+ rtreeSearchPointSwap(p, i, k); |
+ i = k; |
+ }else{ |
+ break; |
+ } |
+ }else{ |
+ if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ |
+ rtreeSearchPointSwap(p, i, j); |
+ i = j; |
+ }else{ |
+ break; |
+ } |
+ } |
+ } |
+ } |
+} |
+ |
+ |
+/* |
+** Continue the search on cursor pCur until the front of the queue |
+** contains an entry suitable for returning as a result-set row, |
+** or until the RtreeSearchPoint queue is empty, indicating that the |
+** query has completed. |
+*/ |
+static int rtreeStepToLeaf(RtreeCursor *pCur){ |
+ RtreeSearchPoint *p; |
+ Rtree *pRtree = RTREE_OF_CURSOR(pCur); |
+ RtreeNode *pNode; |
+ int eWithin; |
+ int rc = SQLITE_OK; |
+ int nCell; |
+ int nConstraint = pCur->nConstraint; |
+ int ii; |
+ int eInt; |
+ RtreeSearchPoint x; |
+ |
+ eInt = pRtree->eCoordType==RTREE_COORD_INT32; |
+ while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ |
+ pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); |
+ if( rc ) return rc; |
+ nCell = NCELL(pNode); |
+ assert( nCell<200 ); |
+ while( p->iCell<nCell ){ |
+ sqlite3_rtree_dbl rScore = (sqlite3_rtree_dbl)-1; |
+ u8 *pCellData = pNode->zData + (4+pRtree->nBytesPerCell*p->iCell); |
+ eWithin = FULLY_WITHIN; |
+ for(ii=0; ii<nConstraint; ii++){ |
+ RtreeConstraint *pConstraint = pCur->aConstraint + ii; |
+ if( pConstraint->op>=RTREE_MATCH ){ |
+ rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, |
+ &rScore, &eWithin); |
+ if( rc ) return rc; |
+ }else if( p->iLevel==1 ){ |
+ rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); |
+ }else{ |
+ rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); |
+ } |
+ if( eWithin==NOT_WITHIN ) break; |
+ } |
+ p->iCell++; |
+ if( eWithin==NOT_WITHIN ) continue; |
+ x.iLevel = p->iLevel - 1; |
+ if( x.iLevel ){ |
+ x.id = readInt64(pCellData); |
+ x.iCell = 0; |
+ }else{ |
+ x.id = p->id; |
+ x.iCell = p->iCell - 1; |
+ } |
+ if( p->iCell>=nCell ){ |
+ RTREE_QUEUE_TRACE(pCur, "POP-S:"); |
+ rtreeSearchPointPop(pCur); |
+ } |
+ if( rScore<RTREE_ZERO ) rScore = RTREE_ZERO; |
+ p = rtreeSearchPointNew(pCur, rScore, x.iLevel); |
+ if( p==0 ) return SQLITE_NOMEM; |
+ p->eWithin = (u8)eWithin; |
+ p->id = x.id; |
+ p->iCell = x.iCell; |
+ RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); |
+ break; |
+ } |
+ if( p->iCell>=nCell ){ |
+ RTREE_QUEUE_TRACE(pCur, "POP-Se:"); |
+ rtreeSearchPointPop(pCur); |
+ } |
+ } |
+ pCur->atEOF = p==0; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Rtree virtual table module xNext method. |
+*/ |
+static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ |
+ RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; |
+ int rc = SQLITE_OK; |
+ |
+ /* Move to the next entry that matches the configured constraints. */ |
+ RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); |
+ rtreeSearchPointPop(pCsr); |
+ rc = rtreeStepToLeaf(pCsr); |
+ return rc; |
+} |
+ |
+/* |
+** Rtree virtual table module xRowid method. |
+*/ |
+static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ |
+ RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; |
+ RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); |
+ int rc = SQLITE_OK; |
+ RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); |
+ if( rc==SQLITE_OK && p ){ |
+ *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Rtree virtual table module xColumn method. |
+*/ |
+static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ |
+ Rtree *pRtree = (Rtree *)cur->pVtab; |
+ RtreeCursor *pCsr = (RtreeCursor *)cur; |
+ RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); |
+ RtreeCoord c; |
+ int rc = SQLITE_OK; |
+ RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); |
+ |
+ if( rc ) return rc; |
+ if( p==0 ) return SQLITE_OK; |
+ if( i==0 ){ |
+ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); |
+ }else{ |
+ nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); |
+#ifndef SQLITE_RTREE_INT_ONLY |
+ if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ |
+ sqlite3_result_double(ctx, c.f); |
+ }else |
+#endif |
+ { |
+ assert( pRtree->eCoordType==RTREE_COORD_INT32 ); |
+ sqlite3_result_int(ctx, c.i); |
+ } |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Use nodeAcquire() to obtain the leaf node containing the record with |
+** rowid iRowid. If successful, set *ppLeaf to point to the node and |
+** return SQLITE_OK. If there is no such record in the table, set |
+** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf |
+** to zero and return an SQLite error code. |
+*/ |
+static int findLeafNode( |
+ Rtree *pRtree, /* RTree to search */ |
+ i64 iRowid, /* The rowid searching for */ |
+ RtreeNode **ppLeaf, /* Write the node here */ |
+ sqlite3_int64 *piNode /* Write the node-id here */ |
+){ |
+ int rc; |
+ *ppLeaf = 0; |
+ sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); |
+ if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ |
+ i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); |
+ if( piNode ) *piNode = iNode; |
+ rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); |
+ sqlite3_reset(pRtree->pReadRowid); |
+ }else{ |
+ rc = sqlite3_reset(pRtree->pReadRowid); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This function is called to configure the RtreeConstraint object passed |
+** as the second argument for a MATCH constraint. The value passed as the |
+** first argument to this function is the right-hand operand to the MATCH |
+** operator. |
+*/ |
+static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ |
+ RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ |
+ sqlite3_rtree_query_info *pInfo; /* Callback information */ |
+ int nBlob; /* Size of the geometry function blob */ |
+ int nExpected; /* Expected size of the BLOB */ |
+ |
+ /* Check that value is actually a blob. */ |
+ if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; |
+ |
+ /* Check that the blob is roughly the right size. */ |
+ nBlob = sqlite3_value_bytes(pValue); |
+ if( nBlob<(int)sizeof(RtreeMatchArg) ){ |
+ return SQLITE_ERROR; |
+ } |
+ |
+ pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); |
+ if( !pInfo ) return SQLITE_NOMEM; |
+ memset(pInfo, 0, sizeof(*pInfo)); |
+ pBlob = (RtreeMatchArg*)&pInfo[1]; |
+ |
+ memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); |
+ nExpected = (int)(sizeof(RtreeMatchArg) + |
+ pBlob->nParam*sizeof(sqlite3_value*) + |
+ (pBlob->nParam-1)*sizeof(RtreeDValue)); |
+ if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ |
+ sqlite3_free(pInfo); |
+ return SQLITE_ERROR; |
+ } |
+ pInfo->pContext = pBlob->cb.pContext; |
+ pInfo->nParam = pBlob->nParam; |
+ pInfo->aParam = pBlob->aParam; |
+ pInfo->apSqlParam = pBlob->apSqlParam; |
+ |
+ if( pBlob->cb.xGeom ){ |
+ pCons->u.xGeom = pBlob->cb.xGeom; |
+ }else{ |
+ pCons->op = RTREE_QUERY; |
+ pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; |
+ } |
+ pCons->pInfo = pInfo; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Rtree virtual table module xFilter method. |
+*/ |
+static int rtreeFilter( |
+ sqlite3_vtab_cursor *pVtabCursor, |
+ int idxNum, const char *idxStr, |
+ int argc, sqlite3_value **argv |
+){ |
+ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; |
+ RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; |
+ RtreeNode *pRoot = 0; |
+ int ii; |
+ int rc = SQLITE_OK; |
+ int iCell = 0; |
+ |
+ rtreeReference(pRtree); |
+ |
+ /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ |
+ freeCursorConstraints(pCsr); |
+ sqlite3_free(pCsr->aPoint); |
+ memset(pCsr, 0, sizeof(RtreeCursor)); |
+ pCsr->base.pVtab = (sqlite3_vtab*)pRtree; |
+ |
+ pCsr->iStrategy = idxNum; |
+ if( idxNum==1 ){ |
+ /* Special case - lookup by rowid. */ |
+ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ |
+ RtreeSearchPoint *p; /* Search point for the leaf */ |
+ i64 iRowid = sqlite3_value_int64(argv[0]); |
+ i64 iNode = 0; |
+ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); |
+ if( rc==SQLITE_OK && pLeaf!=0 ){ |
+ p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); |
+ assert( p!=0 ); /* Always returns pCsr->sPoint */ |
+ pCsr->aNode[0] = pLeaf; |
+ p->id = iNode; |
+ p->eWithin = PARTLY_WITHIN; |
+ rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); |
+ p->iCell = (u8)iCell; |
+ RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); |
+ }else{ |
+ pCsr->atEOF = 1; |
+ } |
+ }else{ |
+ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array |
+ ** with the configured constraints. |
+ */ |
+ rc = nodeAcquire(pRtree, 1, 0, &pRoot); |
+ if( rc==SQLITE_OK && argc>0 ){ |
+ pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); |
+ pCsr->nConstraint = argc; |
+ if( !pCsr->aConstraint ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); |
+ memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); |
+ assert( (idxStr==0 && argc==0) |
+ || (idxStr && (int)strlen(idxStr)==argc*2) ); |
+ for(ii=0; ii<argc; ii++){ |
+ RtreeConstraint *p = &pCsr->aConstraint[ii]; |
+ p->op = idxStr[ii*2]; |
+ p->iCoord = idxStr[ii*2+1]-'0'; |
+ if( p->op>=RTREE_MATCH ){ |
+ /* A MATCH operator. The right-hand-side must be a blob that |
+ ** can be cast into an RtreeMatchArg object. One created using |
+ ** an sqlite3_rtree_geometry_callback() SQL user function. |
+ */ |
+ rc = deserializeGeometry(argv[ii], p); |
+ if( rc!=SQLITE_OK ){ |
+ break; |
+ } |
+ p->pInfo->nCoord = pRtree->nDim2; |
+ p->pInfo->anQueue = pCsr->anQueue; |
+ p->pInfo->mxLevel = pRtree->iDepth + 1; |
+ }else{ |
+#ifdef SQLITE_RTREE_INT_ONLY |
+ p->u.rValue = sqlite3_value_int64(argv[ii]); |
+#else |
+ p->u.rValue = sqlite3_value_double(argv[ii]); |
+#endif |
+ } |
+ } |
+ } |
+ } |
+ if( rc==SQLITE_OK ){ |
+ RtreeSearchPoint *pNew; |
+ pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); |
+ if( pNew==0 ) return SQLITE_NOMEM; |
+ pNew->id = 1; |
+ pNew->iCell = 0; |
+ pNew->eWithin = PARTLY_WITHIN; |
+ assert( pCsr->bPoint==1 ); |
+ pCsr->aNode[0] = pRoot; |
+ pRoot = 0; |
+ RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); |
+ rc = rtreeStepToLeaf(pCsr); |
+ } |
+ } |
+ |
+ nodeRelease(pRtree, pRoot); |
+ rtreeRelease(pRtree); |
+ return rc; |
+} |
+ |
+/* |
+** Rtree virtual table module xBestIndex method. There are three |
+** table scan strategies to choose from (in order from most to |
+** least desirable): |
+** |
+** idxNum idxStr Strategy |
+** ------------------------------------------------ |
+** 1 Unused Direct lookup by rowid. |
+** 2 See below R-tree query or full-table scan. |
+** ------------------------------------------------ |
+** |
+** If strategy 1 is used, then idxStr is not meaningful. If strategy |
+** 2 is used, idxStr is formatted to contain 2 bytes for each |
+** constraint used. The first two bytes of idxStr correspond to |
+** the constraint in sqlite3_index_info.aConstraintUsage[] with |
+** (argvIndex==1) etc. |
+** |
+** The first of each pair of bytes in idxStr identifies the constraint |
+** operator as follows: |
+** |
+** Operator Byte Value |
+** ---------------------- |
+** = 0x41 ('A') |
+** <= 0x42 ('B') |
+** < 0x43 ('C') |
+** >= 0x44 ('D') |
+** > 0x45 ('E') |
+** MATCH 0x46 ('F') |
+** ---------------------- |
+** |
+** The second of each pair of bytes identifies the coordinate column |
+** to which the constraint applies. The leftmost coordinate column |
+** is 'a', the second from the left 'b' etc. |
+*/ |
+static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ |
+ Rtree *pRtree = (Rtree*)tab; |
+ int rc = SQLITE_OK; |
+ int ii; |
+ int bMatch = 0; /* True if there exists a MATCH constraint */ |
+ i64 nRow; /* Estimated rows returned by this scan */ |
+ |
+ int iIdx = 0; |
+ char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; |
+ memset(zIdxStr, 0, sizeof(zIdxStr)); |
+ |
+ /* Check if there exists a MATCH constraint - even an unusable one. If there |
+ ** is, do not consider the lookup-by-rowid plan as using such a plan would |
+ ** require the VDBE to evaluate the MATCH constraint, which is not currently |
+ ** possible. */ |
+ for(ii=0; ii<pIdxInfo->nConstraint; ii++){ |
+ if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){ |
+ bMatch = 1; |
+ } |
+ } |
+ |
+ assert( pIdxInfo->idxStr==0 ); |
+ for(ii=0; ii<pIdxInfo->nConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ |
+ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; |
+ |
+ if( bMatch==0 && p->usable |
+ && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ |
+ ){ |
+ /* We have an equality constraint on the rowid. Use strategy 1. */ |
+ int jj; |
+ for(jj=0; jj<ii; jj++){ |
+ pIdxInfo->aConstraintUsage[jj].argvIndex = 0; |
+ pIdxInfo->aConstraintUsage[jj].omit = 0; |
+ } |
+ pIdxInfo->idxNum = 1; |
+ pIdxInfo->aConstraintUsage[ii].argvIndex = 1; |
+ pIdxInfo->aConstraintUsage[jj].omit = 1; |
+ |
+ /* This strategy involves a two rowid lookups on an B-Tree structures |
+ ** and then a linear search of an R-Tree node. This should be |
+ ** considered almost as quick as a direct rowid lookup (for which |
+ ** sqlite uses an internal cost of 0.0). It is expected to return |
+ ** a single row. |
+ */ |
+ pIdxInfo->estimatedCost = 30.0; |
+ pIdxInfo->estimatedRows = 1; |
+ return SQLITE_OK; |
+ } |
+ |
+ if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ |
+ u8 op; |
+ switch( p->op ){ |
+ case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; |
+ case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; |
+ case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; |
+ case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; |
+ case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; |
+ default: |
+ assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); |
+ op = RTREE_MATCH; |
+ break; |
+ } |
+ zIdxStr[iIdx++] = op; |
+ zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); |
+ pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); |
+ pIdxInfo->aConstraintUsage[ii].omit = 1; |
+ } |
+ } |
+ |
+ pIdxInfo->idxNum = 2; |
+ pIdxInfo->needToFreeIdxStr = 1; |
+ if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ |
+ return SQLITE_NOMEM; |
+ } |
+ |
+ nRow = pRtree->nRowEst >> (iIdx/2); |
+ pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; |
+ pIdxInfo->estimatedRows = nRow; |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Return the N-dimensional volumn of the cell stored in *p. |
+*/ |
+static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ |
+ RtreeDValue area = (RtreeDValue)1; |
+ assert( pRtree->nDim>=1 && pRtree->nDim<=5 ); |
+#ifndef SQLITE_RTREE_INT_ONLY |
+ if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ |
+ switch( pRtree->nDim ){ |
+ case 5: area = p->aCoord[9].f - p->aCoord[8].f; |
+ case 4: area *= p->aCoord[7].f - p->aCoord[6].f; |
+ case 3: area *= p->aCoord[5].f - p->aCoord[4].f; |
+ case 2: area *= p->aCoord[3].f - p->aCoord[2].f; |
+ default: area *= p->aCoord[1].f - p->aCoord[0].f; |
+ } |
+ }else |
+#endif |
+ { |
+ switch( pRtree->nDim ){ |
+ case 5: area = p->aCoord[9].i - p->aCoord[8].i; |
+ case 4: area *= p->aCoord[7].i - p->aCoord[6].i; |
+ case 3: area *= p->aCoord[5].i - p->aCoord[4].i; |
+ case 2: area *= p->aCoord[3].i - p->aCoord[2].i; |
+ default: area *= p->aCoord[1].i - p->aCoord[0].i; |
+ } |
+ } |
+ return area; |
+} |
+ |
+/* |
+** Return the margin length of cell p. The margin length is the sum |
+** of the objects size in each dimension. |
+*/ |
+static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ |
+ RtreeDValue margin = 0; |
+ int ii = pRtree->nDim2 - 2; |
+ do{ |
+ margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); |
+ ii -= 2; |
+ }while( ii>=0 ); |
+ return margin; |
+} |
+ |
+/* |
+** Store the union of cells p1 and p2 in p1. |
+*/ |
+static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ |
+ int ii = 0; |
+ if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ |
+ do{ |
+ p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); |
+ p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); |
+ ii += 2; |
+ }while( ii<pRtree->nDim2 ); |
+ }else{ |
+ do{ |
+ p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); |
+ p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); |
+ ii += 2; |
+ }while( ii<pRtree->nDim2 ); |
+ } |
+} |
+ |
+/* |
+** Return true if the area covered by p2 is a subset of the area covered |
+** by p1. False otherwise. |
+*/ |
+static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ |
+ int ii; |
+ int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); |
+ for(ii=0; ii<pRtree->nDim2; ii+=2){ |
+ RtreeCoord *a1 = &p1->aCoord[ii]; |
+ RtreeCoord *a2 = &p2->aCoord[ii]; |
+ if( (!isInt && (a2[0].f<a1[0].f || a2[1].f>a1[1].f)) |
+ || ( isInt && (a2[0].i<a1[0].i || a2[1].i>a1[1].i)) |
+ ){ |
+ return 0; |
+ } |
+ } |
+ return 1; |
+} |
+ |
+/* |
+** Return the amount cell p would grow by if it were unioned with pCell. |
+*/ |
+static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ |
+ RtreeDValue area; |
+ RtreeCell cell; |
+ memcpy(&cell, p, sizeof(RtreeCell)); |
+ area = cellArea(pRtree, &cell); |
+ cellUnion(pRtree, &cell, pCell); |
+ return (cellArea(pRtree, &cell)-area); |
+} |
+ |
+static RtreeDValue cellOverlap( |
+ Rtree *pRtree, |
+ RtreeCell *p, |
+ RtreeCell *aCell, |
+ int nCell |
+){ |
+ int ii; |
+ RtreeDValue overlap = RTREE_ZERO; |
+ for(ii=0; ii<nCell; ii++){ |
+ int jj; |
+ RtreeDValue o = (RtreeDValue)1; |
+ for(jj=0; jj<pRtree->nDim2; jj+=2){ |
+ RtreeDValue x1, x2; |
+ x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); |
+ x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); |
+ if( x2<x1 ){ |
+ o = (RtreeDValue)0; |
+ break; |
+ }else{ |
+ o = o * (x2-x1); |
+ } |
+ } |
+ overlap += o; |
+ } |
+ return overlap; |
+} |
+ |
+ |
+/* |
+** This function implements the ChooseLeaf algorithm from Gutman[84]. |
+** ChooseSubTree in r*tree terminology. |
+*/ |
+static int ChooseLeaf( |
+ Rtree *pRtree, /* Rtree table */ |
+ RtreeCell *pCell, /* Cell to insert into rtree */ |
+ int iHeight, /* Height of sub-tree rooted at pCell */ |
+ RtreeNode **ppLeaf /* OUT: Selected leaf page */ |
+){ |
+ int rc; |
+ int ii; |
+ RtreeNode *pNode; |
+ rc = nodeAcquire(pRtree, 1, 0, &pNode); |
+ |
+ for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ |
+ int iCell; |
+ sqlite3_int64 iBest = 0; |
+ |
+ RtreeDValue fMinGrowth = RTREE_ZERO; |
+ RtreeDValue fMinArea = RTREE_ZERO; |
+ |
+ int nCell = NCELL(pNode); |
+ RtreeCell cell; |
+ RtreeNode *pChild; |
+ |
+ RtreeCell *aCell = 0; |
+ |
+ /* Select the child node which will be enlarged the least if pCell |
+ ** is inserted into it. Resolve ties by choosing the entry with |
+ ** the smallest area. |
+ */ |
+ for(iCell=0; iCell<nCell; iCell++){ |
+ int bBest = 0; |
+ RtreeDValue growth; |
+ RtreeDValue area; |
+ nodeGetCell(pRtree, pNode, iCell, &cell); |
+ growth = cellGrowth(pRtree, &cell, pCell); |
+ area = cellArea(pRtree, &cell); |
+ if( iCell==0||growth<fMinGrowth||(growth==fMinGrowth && area<fMinArea) ){ |
+ bBest = 1; |
+ } |
+ if( bBest ){ |
+ fMinGrowth = growth; |
+ fMinArea = area; |
+ iBest = cell.iRowid; |
+ } |
+ } |
+ |
+ sqlite3_free(aCell); |
+ rc = nodeAcquire(pRtree, iBest, pNode, &pChild); |
+ nodeRelease(pRtree, pNode); |
+ pNode = pChild; |
+ } |
+ |
+ *ppLeaf = pNode; |
+ return rc; |
+} |
+ |
+/* |
+** A cell with the same content as pCell has just been inserted into |
+** the node pNode. This function updates the bounding box cells in |
+** all ancestor elements. |
+*/ |
+static int AdjustTree( |
+ Rtree *pRtree, /* Rtree table */ |
+ RtreeNode *pNode, /* Adjust ancestry of this node. */ |
+ RtreeCell *pCell /* This cell was just inserted */ |
+){ |
+ RtreeNode *p = pNode; |
+ while( p->pParent ){ |
+ RtreeNode *pParent = p->pParent; |
+ RtreeCell cell; |
+ int iCell; |
+ |
+ if( nodeParentIndex(pRtree, p, &iCell) ){ |
+ return SQLITE_CORRUPT_VTAB; |
+ } |
+ |
+ nodeGetCell(pRtree, pParent, iCell, &cell); |
+ if( !cellContains(pRtree, &cell, pCell) ){ |
+ cellUnion(pRtree, &cell, pCell); |
+ nodeOverwriteCell(pRtree, pParent, &cell, iCell); |
+ } |
+ |
+ p = pParent; |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Write mapping (iRowid->iNode) to the <rtree>_rowid table. |
+*/ |
+static int rowidWrite(Rtree *pRtree, sqlite3_int64 iRowid, sqlite3_int64 iNode){ |
+ sqlite3_bind_int64(pRtree->pWriteRowid, 1, iRowid); |
+ sqlite3_bind_int64(pRtree->pWriteRowid, 2, iNode); |
+ sqlite3_step(pRtree->pWriteRowid); |
+ return sqlite3_reset(pRtree->pWriteRowid); |
+} |
+ |
+/* |
+** Write mapping (iNode->iPar) to the <rtree>_parent table. |
+*/ |
+static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ |
+ sqlite3_bind_int64(pRtree->pWriteParent, 1, iNode); |
+ sqlite3_bind_int64(pRtree->pWriteParent, 2, iPar); |
+ sqlite3_step(pRtree->pWriteParent); |
+ return sqlite3_reset(pRtree->pWriteParent); |
+} |
+ |
+static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); |
+ |
+ |
+/* |
+** Arguments aIdx, aDistance and aSpare all point to arrays of size |
+** nIdx. The aIdx array contains the set of integers from 0 to |
+** (nIdx-1) in no particular order. This function sorts the values |
+** in aIdx according to the indexed values in aDistance. For |
+** example, assuming the inputs: |
+** |
+** aIdx = { 0, 1, 2, 3 } |
+** aDistance = { 5.0, 2.0, 7.0, 6.0 } |
+** |
+** this function sets the aIdx array to contain: |
+** |
+** aIdx = { 0, 1, 2, 3 } |
+** |
+** The aSpare array is used as temporary working space by the |
+** sorting algorithm. |
+*/ |
+static void SortByDistance( |
+ int *aIdx, |
+ int nIdx, |
+ RtreeDValue *aDistance, |
+ int *aSpare |
+){ |
+ if( nIdx>1 ){ |
+ int iLeft = 0; |
+ int iRight = 0; |
+ |
+ int nLeft = nIdx/2; |
+ int nRight = nIdx-nLeft; |
+ int *aLeft = aIdx; |
+ int *aRight = &aIdx[nLeft]; |
+ |
+ SortByDistance(aLeft, nLeft, aDistance, aSpare); |
+ SortByDistance(aRight, nRight, aDistance, aSpare); |
+ |
+ memcpy(aSpare, aLeft, sizeof(int)*nLeft); |
+ aLeft = aSpare; |
+ |
+ while( iLeft<nLeft || iRight<nRight ){ |
+ if( iLeft==nLeft ){ |
+ aIdx[iLeft+iRight] = aRight[iRight]; |
+ iRight++; |
+ }else if( iRight==nRight ){ |
+ aIdx[iLeft+iRight] = aLeft[iLeft]; |
+ iLeft++; |
+ }else{ |
+ RtreeDValue fLeft = aDistance[aLeft[iLeft]]; |
+ RtreeDValue fRight = aDistance[aRight[iRight]]; |
+ if( fLeft<fRight ){ |
+ aIdx[iLeft+iRight] = aLeft[iLeft]; |
+ iLeft++; |
+ }else{ |
+ aIdx[iLeft+iRight] = aRight[iRight]; |
+ iRight++; |
+ } |
+ } |
+ } |
+ |
+#if 0 |
+ /* Check that the sort worked */ |
+ { |
+ int jj; |
+ for(jj=1; jj<nIdx; jj++){ |
+ RtreeDValue left = aDistance[aIdx[jj-1]]; |
+ RtreeDValue right = aDistance[aIdx[jj]]; |
+ assert( left<=right ); |
+ } |
+ } |
+#endif |
+ } |
+} |
+ |
+/* |
+** Arguments aIdx, aCell and aSpare all point to arrays of size |
+** nIdx. The aIdx array contains the set of integers from 0 to |
+** (nIdx-1) in no particular order. This function sorts the values |
+** in aIdx according to dimension iDim of the cells in aCell. The |
+** minimum value of dimension iDim is considered first, the |
+** maximum used to break ties. |
+** |
+** The aSpare array is used as temporary working space by the |
+** sorting algorithm. |
+*/ |
+static void SortByDimension( |
+ Rtree *pRtree, |
+ int *aIdx, |
+ int nIdx, |
+ int iDim, |
+ RtreeCell *aCell, |
+ int *aSpare |
+){ |
+ if( nIdx>1 ){ |
+ |
+ int iLeft = 0; |
+ int iRight = 0; |
+ |
+ int nLeft = nIdx/2; |
+ int nRight = nIdx-nLeft; |
+ int *aLeft = aIdx; |
+ int *aRight = &aIdx[nLeft]; |
+ |
+ SortByDimension(pRtree, aLeft, nLeft, iDim, aCell, aSpare); |
+ SortByDimension(pRtree, aRight, nRight, iDim, aCell, aSpare); |
+ |
+ memcpy(aSpare, aLeft, sizeof(int)*nLeft); |
+ aLeft = aSpare; |
+ while( iLeft<nLeft || iRight<nRight ){ |
+ RtreeDValue xleft1 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2]); |
+ RtreeDValue xleft2 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2+1]); |
+ RtreeDValue xright1 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2]); |
+ RtreeDValue xright2 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2+1]); |
+ if( (iLeft!=nLeft) && ((iRight==nRight) |
+ || (xleft1<xright1) |
+ || (xleft1==xright1 && xleft2<xright2) |
+ )){ |
+ aIdx[iLeft+iRight] = aLeft[iLeft]; |
+ iLeft++; |
+ }else{ |
+ aIdx[iLeft+iRight] = aRight[iRight]; |
+ iRight++; |
+ } |
+ } |
+ |
+#if 0 |
+ /* Check that the sort worked */ |
+ { |
+ int jj; |
+ for(jj=1; jj<nIdx; jj++){ |
+ RtreeDValue xleft1 = aCell[aIdx[jj-1]].aCoord[iDim*2]; |
+ RtreeDValue xleft2 = aCell[aIdx[jj-1]].aCoord[iDim*2+1]; |
+ RtreeDValue xright1 = aCell[aIdx[jj]].aCoord[iDim*2]; |
+ RtreeDValue xright2 = aCell[aIdx[jj]].aCoord[iDim*2+1]; |
+ assert( xleft1<=xright1 && (xleft1<xright1 || xleft2<=xright2) ); |
+ } |
+ } |
+#endif |
+ } |
+} |
+ |
+/* |
+** Implementation of the R*-tree variant of SplitNode from Beckman[1990]. |
+*/ |
+static int splitNodeStartree( |
+ Rtree *pRtree, |
+ RtreeCell *aCell, |
+ int nCell, |
+ RtreeNode *pLeft, |
+ RtreeNode *pRight, |
+ RtreeCell *pBboxLeft, |
+ RtreeCell *pBboxRight |
+){ |
+ int **aaSorted; |
+ int *aSpare; |
+ int ii; |
+ |
+ int iBestDim = 0; |
+ int iBestSplit = 0; |
+ RtreeDValue fBestMargin = RTREE_ZERO; |
+ |
+ int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); |
+ |
+ aaSorted = (int **)sqlite3_malloc(nByte); |
+ if( !aaSorted ){ |
+ return SQLITE_NOMEM; |
+ } |
+ |
+ aSpare = &((int *)&aaSorted[pRtree->nDim])[pRtree->nDim*nCell]; |
+ memset(aaSorted, 0, nByte); |
+ for(ii=0; ii<pRtree->nDim; ii++){ |
+ int jj; |
+ aaSorted[ii] = &((int *)&aaSorted[pRtree->nDim])[ii*nCell]; |
+ for(jj=0; jj<nCell; jj++){ |
+ aaSorted[ii][jj] = jj; |
+ } |
+ SortByDimension(pRtree, aaSorted[ii], nCell, ii, aCell, aSpare); |
+ } |
+ |
+ for(ii=0; ii<pRtree->nDim; ii++){ |
+ RtreeDValue margin = RTREE_ZERO; |
+ RtreeDValue fBestOverlap = RTREE_ZERO; |
+ RtreeDValue fBestArea = RTREE_ZERO; |
+ int iBestLeft = 0; |
+ int nLeft; |
+ |
+ for( |
+ nLeft=RTREE_MINCELLS(pRtree); |
+ nLeft<=(nCell-RTREE_MINCELLS(pRtree)); |
+ nLeft++ |
+ ){ |
+ RtreeCell left; |
+ RtreeCell right; |
+ int kk; |
+ RtreeDValue overlap; |
+ RtreeDValue area; |
+ |
+ memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); |
+ memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); |
+ for(kk=1; kk<(nCell-1); kk++){ |
+ if( kk<nLeft ){ |
+ cellUnion(pRtree, &left, &aCell[aaSorted[ii][kk]]); |
+ }else{ |
+ cellUnion(pRtree, &right, &aCell[aaSorted[ii][kk]]); |
+ } |
+ } |
+ margin += cellMargin(pRtree, &left); |
+ margin += cellMargin(pRtree, &right); |
+ overlap = cellOverlap(pRtree, &left, &right, 1); |
+ area = cellArea(pRtree, &left) + cellArea(pRtree, &right); |
+ if( (nLeft==RTREE_MINCELLS(pRtree)) |
+ || (overlap<fBestOverlap) |
+ || (overlap==fBestOverlap && area<fBestArea) |
+ ){ |
+ iBestLeft = nLeft; |
+ fBestOverlap = overlap; |
+ fBestArea = area; |
+ } |
+ } |
+ |
+ if( ii==0 || margin<fBestMargin ){ |
+ iBestDim = ii; |
+ fBestMargin = margin; |
+ iBestSplit = iBestLeft; |
+ } |
+ } |
+ |
+ memcpy(pBboxLeft, &aCell[aaSorted[iBestDim][0]], sizeof(RtreeCell)); |
+ memcpy(pBboxRight, &aCell[aaSorted[iBestDim][iBestSplit]], sizeof(RtreeCell)); |
+ for(ii=0; ii<nCell; ii++){ |
+ RtreeNode *pTarget = (ii<iBestSplit)?pLeft:pRight; |
+ RtreeCell *pBbox = (ii<iBestSplit)?pBboxLeft:pBboxRight; |
+ RtreeCell *pCell = &aCell[aaSorted[iBestDim][ii]]; |
+ nodeInsertCell(pRtree, pTarget, pCell); |
+ cellUnion(pRtree, pBbox, pCell); |
+ } |
+ |
+ sqlite3_free(aaSorted); |
+ return SQLITE_OK; |
+} |
+ |
+ |
+static int updateMapping( |
+ Rtree *pRtree, |
+ i64 iRowid, |
+ RtreeNode *pNode, |
+ int iHeight |
+){ |
+ int (*xSetMapping)(Rtree *, sqlite3_int64, sqlite3_int64); |
+ xSetMapping = ((iHeight==0)?rowidWrite:parentWrite); |
+ if( iHeight>0 ){ |
+ RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); |
+ if( pChild ){ |
+ nodeRelease(pRtree, pChild->pParent); |
+ nodeReference(pNode); |
+ pChild->pParent = pNode; |
+ } |
+ } |
+ return xSetMapping(pRtree, iRowid, pNode->iNode); |
+} |
+ |
+static int SplitNode( |
+ Rtree *pRtree, |
+ RtreeNode *pNode, |
+ RtreeCell *pCell, |
+ int iHeight |
+){ |
+ int i; |
+ int newCellIsRight = 0; |
+ |
+ int rc = SQLITE_OK; |
+ int nCell = NCELL(pNode); |
+ RtreeCell *aCell; |
+ int *aiUsed; |
+ |
+ RtreeNode *pLeft = 0; |
+ RtreeNode *pRight = 0; |
+ |
+ RtreeCell leftbbox; |
+ RtreeCell rightbbox; |
+ |
+ /* Allocate an array and populate it with a copy of pCell and |
+ ** all cells from node pLeft. Then zero the original node. |
+ */ |
+ aCell = sqlite3_malloc((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); |
+ if( !aCell ){ |
+ rc = SQLITE_NOMEM; |
+ goto splitnode_out; |
+ } |
+ aiUsed = (int *)&aCell[nCell+1]; |
+ memset(aiUsed, 0, sizeof(int)*(nCell+1)); |
+ for(i=0; i<nCell; i++){ |
+ nodeGetCell(pRtree, pNode, i, &aCell[i]); |
+ } |
+ nodeZero(pRtree, pNode); |
+ memcpy(&aCell[nCell], pCell, sizeof(RtreeCell)); |
+ nCell++; |
+ |
+ if( pNode->iNode==1 ){ |
+ pRight = nodeNew(pRtree, pNode); |
+ pLeft = nodeNew(pRtree, pNode); |
+ pRtree->iDepth++; |
+ pNode->isDirty = 1; |
+ writeInt16(pNode->zData, pRtree->iDepth); |
+ }else{ |
+ pLeft = pNode; |
+ pRight = nodeNew(pRtree, pLeft->pParent); |
+ nodeReference(pLeft); |
+ } |
+ |
+ if( !pLeft || !pRight ){ |
+ rc = SQLITE_NOMEM; |
+ goto splitnode_out; |
+ } |
+ |
+ memset(pLeft->zData, 0, pRtree->iNodeSize); |
+ memset(pRight->zData, 0, pRtree->iNodeSize); |
+ |
+ rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, |
+ &leftbbox, &rightbbox); |
+ if( rc!=SQLITE_OK ){ |
+ goto splitnode_out; |
+ } |
+ |
+ /* Ensure both child nodes have node numbers assigned to them by calling |
+ ** nodeWrite(). Node pRight always needs a node number, as it was created |
+ ** by nodeNew() above. But node pLeft sometimes already has a node number. |
+ ** In this case avoid the all to nodeWrite(). |
+ */ |
+ if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) |
+ || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) |
+ ){ |
+ goto splitnode_out; |
+ } |
+ |
+ rightbbox.iRowid = pRight->iNode; |
+ leftbbox.iRowid = pLeft->iNode; |
+ |
+ if( pNode->iNode==1 ){ |
+ rc = rtreeInsertCell(pRtree, pLeft->pParent, &leftbbox, iHeight+1); |
+ if( rc!=SQLITE_OK ){ |
+ goto splitnode_out; |
+ } |
+ }else{ |
+ RtreeNode *pParent = pLeft->pParent; |
+ int iCell; |
+ rc = nodeParentIndex(pRtree, pLeft, &iCell); |
+ if( rc==SQLITE_OK ){ |
+ nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); |
+ rc = AdjustTree(pRtree, pParent, &leftbbox); |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ goto splitnode_out; |
+ } |
+ } |
+ if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ |
+ goto splitnode_out; |
+ } |
+ |
+ for(i=0; i<NCELL(pRight); i++){ |
+ i64 iRowid = nodeGetRowid(pRtree, pRight, i); |
+ rc = updateMapping(pRtree, iRowid, pRight, iHeight); |
+ if( iRowid==pCell->iRowid ){ |
+ newCellIsRight = 1; |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ goto splitnode_out; |
+ } |
+ } |
+ if( pNode->iNode==1 ){ |
+ for(i=0; i<NCELL(pLeft); i++){ |
+ i64 iRowid = nodeGetRowid(pRtree, pLeft, i); |
+ rc = updateMapping(pRtree, iRowid, pLeft, iHeight); |
+ if( rc!=SQLITE_OK ){ |
+ goto splitnode_out; |
+ } |
+ } |
+ }else if( newCellIsRight==0 ){ |
+ rc = updateMapping(pRtree, pCell->iRowid, pLeft, iHeight); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = nodeRelease(pRtree, pRight); |
+ pRight = 0; |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = nodeRelease(pRtree, pLeft); |
+ pLeft = 0; |
+ } |
+ |
+splitnode_out: |
+ nodeRelease(pRtree, pRight); |
+ nodeRelease(pRtree, pLeft); |
+ sqlite3_free(aCell); |
+ return rc; |
+} |
+ |
+/* |
+** If node pLeaf is not the root of the r-tree and its pParent pointer is |
+** still NULL, load all ancestor nodes of pLeaf into memory and populate |
+** the pLeaf->pParent chain all the way up to the root node. |
+** |
+** This operation is required when a row is deleted (or updated - an update |
+** is implemented as a delete followed by an insert). SQLite provides the |
+** rowid of the row to delete, which can be used to find the leaf on which |
+** the entry resides (argument pLeaf). Once the leaf is located, this |
+** function is called to determine its ancestry. |
+*/ |
+static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ |
+ int rc = SQLITE_OK; |
+ RtreeNode *pChild = pLeaf; |
+ while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ |
+ int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ |
+ sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); |
+ rc = sqlite3_step(pRtree->pReadParent); |
+ if( rc==SQLITE_ROW ){ |
+ RtreeNode *pTest; /* Used to test for reference loops */ |
+ i64 iNode; /* Node number of parent node */ |
+ |
+ /* Before setting pChild->pParent, test that we are not creating a |
+ ** loop of references (as we would if, say, pChild==pParent). We don't |
+ ** want to do this as it leads to a memory leak when trying to delete |
+ ** the referenced counted node structures. |
+ */ |
+ iNode = sqlite3_column_int64(pRtree->pReadParent, 0); |
+ for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); |
+ if( !pTest ){ |
+ rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); |
+ } |
+ } |
+ rc = sqlite3_reset(pRtree->pReadParent); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; |
+ pChild = pChild->pParent; |
+ } |
+ return rc; |
+} |
+ |
+static int deleteCell(Rtree *, RtreeNode *, int, int); |
+ |
+static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ |
+ int rc; |
+ int rc2; |
+ RtreeNode *pParent = 0; |
+ int iCell; |
+ |
+ assert( pNode->nRef==1 ); |
+ |
+ /* Remove the entry in the parent cell. */ |
+ rc = nodeParentIndex(pRtree, pNode, &iCell); |
+ if( rc==SQLITE_OK ){ |
+ pParent = pNode->pParent; |
+ pNode->pParent = 0; |
+ rc = deleteCell(pRtree, pParent, iCell, iHeight+1); |
+ } |
+ rc2 = nodeRelease(pRtree, pParent); |
+ if( rc==SQLITE_OK ){ |
+ rc = rc2; |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ |
+ /* Remove the xxx_node entry. */ |
+ sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); |
+ sqlite3_step(pRtree->pDeleteNode); |
+ if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteNode)) ){ |
+ return rc; |
+ } |
+ |
+ /* Remove the xxx_parent entry. */ |
+ sqlite3_bind_int64(pRtree->pDeleteParent, 1, pNode->iNode); |
+ sqlite3_step(pRtree->pDeleteParent); |
+ if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ |
+ return rc; |
+ } |
+ |
+ /* Remove the node from the in-memory hash table and link it into |
+ ** the Rtree.pDeleted list. Its contents will be re-inserted later on. |
+ */ |
+ nodeHashDelete(pRtree, pNode); |
+ pNode->iNode = iHeight; |
+ pNode->pNext = pRtree->pDeleted; |
+ pNode->nRef++; |
+ pRtree->pDeleted = pNode; |
+ |
+ return SQLITE_OK; |
+} |
+ |
+static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ |
+ RtreeNode *pParent = pNode->pParent; |
+ int rc = SQLITE_OK; |
+ if( pParent ){ |
+ int ii; |
+ int nCell = NCELL(pNode); |
+ RtreeCell box; /* Bounding box for pNode */ |
+ nodeGetCell(pRtree, pNode, 0, &box); |
+ for(ii=1; ii<nCell; ii++){ |
+ RtreeCell cell; |
+ nodeGetCell(pRtree, pNode, ii, &cell); |
+ cellUnion(pRtree, &box, &cell); |
+ } |
+ box.iRowid = pNode->iNode; |
+ rc = nodeParentIndex(pRtree, pNode, &ii); |
+ if( rc==SQLITE_OK ){ |
+ nodeOverwriteCell(pRtree, pParent, &box, ii); |
+ rc = fixBoundingBox(pRtree, pParent); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Delete the cell at index iCell of node pNode. After removing the |
+** cell, adjust the r-tree data structure if required. |
+*/ |
+static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ |
+ RtreeNode *pParent; |
+ int rc; |
+ |
+ if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ |
+ return rc; |
+ } |
+ |
+ /* Remove the cell from the node. This call just moves bytes around |
+ ** the in-memory node image, so it cannot fail. |
+ */ |
+ nodeDeleteCell(pRtree, pNode, iCell); |
+ |
+ /* If the node is not the tree root and now has less than the minimum |
+ ** number of cells, remove it from the tree. Otherwise, update the |
+ ** cell in the parent node so that it tightly contains the updated |
+ ** node. |
+ */ |
+ pParent = pNode->pParent; |
+ assert( pParent || pNode->iNode==1 ); |
+ if( pParent ){ |
+ if( NCELL(pNode)<RTREE_MINCELLS(pRtree) ){ |
+ rc = removeNode(pRtree, pNode, iHeight); |
+ }else{ |
+ rc = fixBoundingBox(pRtree, pNode); |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+static int Reinsert( |
+ Rtree *pRtree, |
+ RtreeNode *pNode, |
+ RtreeCell *pCell, |
+ int iHeight |
+){ |
+ int *aOrder; |
+ int *aSpare; |
+ RtreeCell *aCell; |
+ RtreeDValue *aDistance; |
+ int nCell; |
+ RtreeDValue aCenterCoord[RTREE_MAX_DIMENSIONS]; |
+ int iDim; |
+ int ii; |
+ int rc = SQLITE_OK; |
+ int n; |
+ |
+ memset(aCenterCoord, 0, sizeof(RtreeDValue)*RTREE_MAX_DIMENSIONS); |
+ |
+ nCell = NCELL(pNode)+1; |
+ n = (nCell+1)&(~1); |
+ |
+ /* Allocate the buffers used by this operation. The allocation is |
+ ** relinquished before this function returns. |
+ */ |
+ aCell = (RtreeCell *)sqlite3_malloc(n * ( |
+ sizeof(RtreeCell) + /* aCell array */ |
+ sizeof(int) + /* aOrder array */ |
+ sizeof(int) + /* aSpare array */ |
+ sizeof(RtreeDValue) /* aDistance array */ |
+ )); |
+ if( !aCell ){ |
+ return SQLITE_NOMEM; |
+ } |
+ aOrder = (int *)&aCell[n]; |
+ aSpare = (int *)&aOrder[n]; |
+ aDistance = (RtreeDValue *)&aSpare[n]; |
+ |
+ for(ii=0; ii<nCell; ii++){ |
+ if( ii==(nCell-1) ){ |
+ memcpy(&aCell[ii], pCell, sizeof(RtreeCell)); |
+ }else{ |
+ nodeGetCell(pRtree, pNode, ii, &aCell[ii]); |
+ } |
+ aOrder[ii] = ii; |
+ for(iDim=0; iDim<pRtree->nDim; iDim++){ |
+ aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); |
+ aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); |
+ } |
+ } |
+ for(iDim=0; iDim<pRtree->nDim; iDim++){ |
+ aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); |
+ } |
+ |
+ for(ii=0; ii<nCell; ii++){ |
+ aDistance[ii] = RTREE_ZERO; |
+ for(iDim=0; iDim<pRtree->nDim; iDim++){ |
+ RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - |
+ DCOORD(aCell[ii].aCoord[iDim*2])); |
+ aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); |
+ } |
+ } |
+ |
+ SortByDistance(aOrder, nCell, aDistance, aSpare); |
+ nodeZero(pRtree, pNode); |
+ |
+ for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ |
+ RtreeCell *p = &aCell[aOrder[ii]]; |
+ nodeInsertCell(pRtree, pNode, p); |
+ if( p->iRowid==pCell->iRowid ){ |
+ if( iHeight==0 ){ |
+ rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); |
+ }else{ |
+ rc = parentWrite(pRtree, p->iRowid, pNode->iNode); |
+ } |
+ } |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = fixBoundingBox(pRtree, pNode); |
+ } |
+ for(; rc==SQLITE_OK && ii<nCell; ii++){ |
+ /* Find a node to store this cell in. pNode->iNode currently contains |
+ ** the height of the sub-tree headed by the cell. |
+ */ |
+ RtreeNode *pInsert; |
+ RtreeCell *p = &aCell[aOrder[ii]]; |
+ rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); |
+ rc2 = nodeRelease(pRtree, pInsert); |
+ if( rc==SQLITE_OK ){ |
+ rc = rc2; |
+ } |
+ } |
+ } |
+ |
+ sqlite3_free(aCell); |
+ return rc; |
+} |
+ |
+/* |
+** Insert cell pCell into node pNode. Node pNode is the head of a |
+** subtree iHeight high (leaf nodes have iHeight==0). |
+*/ |
+static int rtreeInsertCell( |
+ Rtree *pRtree, |
+ RtreeNode *pNode, |
+ RtreeCell *pCell, |
+ int iHeight |
+){ |
+ int rc = SQLITE_OK; |
+ if( iHeight>0 ){ |
+ RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid); |
+ if( pChild ){ |
+ nodeRelease(pRtree, pChild->pParent); |
+ nodeReference(pNode); |
+ pChild->pParent = pNode; |
+ } |
+ } |
+ if( nodeInsertCell(pRtree, pNode, pCell) ){ |
+ if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ |
+ rc = SplitNode(pRtree, pNode, pCell, iHeight); |
+ }else{ |
+ pRtree->iReinsertHeight = iHeight; |
+ rc = Reinsert(pRtree, pNode, pCell, iHeight); |
+ } |
+ }else{ |
+ rc = AdjustTree(pRtree, pNode, pCell); |
+ if( rc==SQLITE_OK ){ |
+ if( iHeight==0 ){ |
+ rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); |
+ }else{ |
+ rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); |
+ } |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ |
+ int ii; |
+ int rc = SQLITE_OK; |
+ int nCell = NCELL(pNode); |
+ |
+ for(ii=0; rc==SQLITE_OK && ii<nCell; ii++){ |
+ RtreeNode *pInsert; |
+ RtreeCell cell; |
+ nodeGetCell(pRtree, pNode, ii, &cell); |
+ |
+ /* Find a node to store this cell in. pNode->iNode currently contains |
+ ** the height of the sub-tree headed by the cell. |
+ */ |
+ rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); |
+ rc2 = nodeRelease(pRtree, pInsert); |
+ if( rc==SQLITE_OK ){ |
+ rc = rc2; |
+ } |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Select a currently unused rowid for a new r-tree record. |
+*/ |
+static int newRowid(Rtree *pRtree, i64 *piRowid){ |
+ int rc; |
+ sqlite3_bind_null(pRtree->pWriteRowid, 1); |
+ sqlite3_bind_null(pRtree->pWriteRowid, 2); |
+ sqlite3_step(pRtree->pWriteRowid); |
+ rc = sqlite3_reset(pRtree->pWriteRowid); |
+ *piRowid = sqlite3_last_insert_rowid(pRtree->db); |
+ return rc; |
+} |
+ |
+/* |
+** Remove the entry with rowid=iDelete from the r-tree structure. |
+*/ |
+static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ |
+ int rc; /* Return code */ |
+ RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ |
+ int iCell; /* Index of iDelete cell in pLeaf */ |
+ RtreeNode *pRoot; /* Root node of rtree structure */ |
+ |
+ |
+ /* Obtain a reference to the root node to initialize Rtree.iDepth */ |
+ rc = nodeAcquire(pRtree, 1, 0, &pRoot); |
+ |
+ /* Obtain a reference to the leaf node that contains the entry |
+ ** about to be deleted. |
+ */ |
+ if( rc==SQLITE_OK ){ |
+ rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); |
+ } |
+ |
+ /* Delete the cell in question from the leaf node. */ |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); |
+ if( rc==SQLITE_OK ){ |
+ rc = deleteCell(pRtree, pLeaf, iCell, 0); |
+ } |
+ rc2 = nodeRelease(pRtree, pLeaf); |
+ if( rc==SQLITE_OK ){ |
+ rc = rc2; |
+ } |
+ } |
+ |
+ /* Delete the corresponding entry in the <rtree>_rowid table. */ |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); |
+ sqlite3_step(pRtree->pDeleteRowid); |
+ rc = sqlite3_reset(pRtree->pDeleteRowid); |
+ } |
+ |
+ /* Check if the root node now has exactly one child. If so, remove |
+ ** it, schedule the contents of the child for reinsertion and |
+ ** reduce the tree height by one. |
+ ** |
+ ** This is equivalent to copying the contents of the child into |
+ ** the root node (the operation that Gutman's paper says to perform |
+ ** in this scenario). |
+ */ |
+ if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ |
+ int rc2; |
+ RtreeNode *pChild; |
+ i64 iChild = nodeGetRowid(pRtree, pRoot, 0); |
+ rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); |
+ if( rc==SQLITE_OK ){ |
+ rc = removeNode(pRtree, pChild, pRtree->iDepth-1); |
+ } |
+ rc2 = nodeRelease(pRtree, pChild); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ if( rc==SQLITE_OK ){ |
+ pRtree->iDepth--; |
+ writeInt16(pRoot->zData, pRtree->iDepth); |
+ pRoot->isDirty = 1; |
+ } |
+ } |
+ |
+ /* Re-insert the contents of any underfull nodes removed from the tree. */ |
+ for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ |
+ if( rc==SQLITE_OK ){ |
+ rc = reinsertNodeContent(pRtree, pLeaf); |
+ } |
+ pRtree->pDeleted = pLeaf->pNext; |
+ sqlite3_free(pLeaf); |
+ } |
+ |
+ /* Release the reference to the root node. */ |
+ if( rc==SQLITE_OK ){ |
+ rc = nodeRelease(pRtree, pRoot); |
+ }else{ |
+ nodeRelease(pRtree, pRoot); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Rounding constants for float->double conversion. |
+*/ |
+#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ |
+#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ |
+ |
+#if !defined(SQLITE_RTREE_INT_ONLY) |
+/* |
+** Convert an sqlite3_value into an RtreeValue (presumably a float) |
+** while taking care to round toward negative or positive, respectively. |
+*/ |
+static RtreeValue rtreeValueDown(sqlite3_value *v){ |
+ double d = sqlite3_value_double(v); |
+ float f = (float)d; |
+ if( f>d ){ |
+ f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); |
+ } |
+ return f; |
+} |
+static RtreeValue rtreeValueUp(sqlite3_value *v){ |
+ double d = sqlite3_value_double(v); |
+ float f = (float)d; |
+ if( f<d ){ |
+ f = (float)(d*(d<0 ? RNDTOWARDS : RNDAWAY)); |
+ } |
+ return f; |
+} |
+#endif /* !defined(SQLITE_RTREE_INT_ONLY) */ |
+ |
+/* |
+** A constraint has failed while inserting a row into an rtree table. |
+** Assuming no OOM error occurs, this function sets the error message |
+** (at pRtree->base.zErrMsg) to an appropriate value and returns |
+** SQLITE_CONSTRAINT. |
+** |
+** Parameter iCol is the index of the leftmost column involved in the |
+** constraint failure. If it is 0, then the constraint that failed is |
+** the unique constraint on the id column. Otherwise, it is the rtree |
+** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. |
+** |
+** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. |
+*/ |
+static int rtreeConstraintError(Rtree *pRtree, int iCol){ |
+ sqlite3_stmt *pStmt = 0; |
+ char *zSql; |
+ int rc; |
+ |
+ assert( iCol==0 || iCol%2 ); |
+ zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); |
+ if( zSql ){ |
+ rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ sqlite3_free(zSql); |
+ |
+ if( rc==SQLITE_OK ){ |
+ if( iCol==0 ){ |
+ const char *zCol = sqlite3_column_name(pStmt, 0); |
+ pRtree->base.zErrMsg = sqlite3_mprintf( |
+ "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol |
+ ); |
+ }else{ |
+ const char *zCol1 = sqlite3_column_name(pStmt, iCol); |
+ const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); |
+ pRtree->base.zErrMsg = sqlite3_mprintf( |
+ "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 |
+ ); |
+ } |
+ } |
+ |
+ sqlite3_finalize(pStmt); |
+ return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); |
+} |
+ |
+ |
+ |
+/* |
+** The xUpdate method for rtree module virtual tables. |
+*/ |
+static int rtreeUpdate( |
+ sqlite3_vtab *pVtab, |
+ int nData, |
+ sqlite3_value **azData, |
+ sqlite_int64 *pRowid |
+){ |
+ Rtree *pRtree = (Rtree *)pVtab; |
+ int rc = SQLITE_OK; |
+ RtreeCell cell; /* New cell to insert if nData>1 */ |
+ int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ |
+ |
+ rtreeReference(pRtree); |
+ assert(nData>=1); |
+ |
+ cell.iRowid = 0; /* Used only to suppress a compiler warning */ |
+ |
+ /* Constraint handling. A write operation on an r-tree table may return |
+ ** SQLITE_CONSTRAINT for two reasons: |
+ ** |
+ ** 1. A duplicate rowid value, or |
+ ** 2. The supplied data violates the "x2>=x1" constraint. |
+ ** |
+ ** In the first case, if the conflict-handling mode is REPLACE, then |
+ ** the conflicting row can be removed before proceeding. In the second |
+ ** case, SQLITE_CONSTRAINT must be returned regardless of the |
+ ** conflict-handling mode specified by the user. |
+ */ |
+ if( nData>1 ){ |
+ int ii; |
+ |
+ /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. |
+ ** |
+ ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared |
+ ** with "column" that are interpreted as table constraints. |
+ ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); |
+ ** This problem was discovered after years of use, so we silently ignore |
+ ** these kinds of misdeclared tables to avoid breaking any legacy. |
+ */ |
+ assert( nData<=(pRtree->nDim2 + 3) ); |
+ |
+#ifndef SQLITE_RTREE_INT_ONLY |
+ if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ |
+ for(ii=0; ii<nData-4; ii+=2){ |
+ cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]); |
+ cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]); |
+ if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){ |
+ rc = rtreeConstraintError(pRtree, ii+1); |
+ goto constraint; |
+ } |
+ } |
+ }else |
+#endif |
+ { |
+ for(ii=0; ii<nData-4; ii+=2){ |
+ cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]); |
+ cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]); |
+ if( cell.aCoord[ii].i>cell.aCoord[ii+1].i ){ |
+ rc = rtreeConstraintError(pRtree, ii+1); |
+ goto constraint; |
+ } |
+ } |
+ } |
+ |
+ /* If a rowid value was supplied, check if it is already present in |
+ ** the table. If so, the constraint has failed. */ |
+ if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ |
+ cell.iRowid = sqlite3_value_int64(azData[2]); |
+ if( sqlite3_value_type(azData[0])==SQLITE_NULL |
+ || sqlite3_value_int64(azData[0])!=cell.iRowid |
+ ){ |
+ int steprc; |
+ sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); |
+ steprc = sqlite3_step(pRtree->pReadRowid); |
+ rc = sqlite3_reset(pRtree->pReadRowid); |
+ if( SQLITE_ROW==steprc ){ |
+ if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ |
+ rc = rtreeDeleteRowid(pRtree, cell.iRowid); |
+ }else{ |
+ rc = rtreeConstraintError(pRtree, 0); |
+ goto constraint; |
+ } |
+ } |
+ } |
+ bHaveRowid = 1; |
+ } |
+ } |
+ |
+ /* If azData[0] is not an SQL NULL value, it is the rowid of a |
+ ** record to delete from the r-tree table. The following block does |
+ ** just that. |
+ */ |
+ if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ |
+ rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); |
+ } |
+ |
+ /* If the azData[] array contains more than one element, elements |
+ ** (azData[2]..azData[argc-1]) contain a new record to insert into |
+ ** the r-tree structure. |
+ */ |
+ if( rc==SQLITE_OK && nData>1 ){ |
+ /* Insert the new record into the r-tree */ |
+ RtreeNode *pLeaf = 0; |
+ |
+ /* Figure out the rowid of the new row. */ |
+ if( bHaveRowid==0 ){ |
+ rc = newRowid(pRtree, &cell.iRowid); |
+ } |
+ *pRowid = cell.iRowid; |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ pRtree->iReinsertHeight = -1; |
+ rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); |
+ rc2 = nodeRelease(pRtree, pLeaf); |
+ if( rc==SQLITE_OK ){ |
+ rc = rc2; |
+ } |
+ } |
+ } |
+ |
+constraint: |
+ rtreeRelease(pRtree); |
+ return rc; |
+} |
+ |
+/* |
+** Called when a transaction starts. |
+*/ |
+static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ |
+ Rtree *pRtree = (Rtree *)pVtab; |
+ assert( pRtree->inWrTrans==0 ); |
+ pRtree->inWrTrans++; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Called when a transaction completes (either by COMMIT or ROLLBACK). |
+** The sqlite3_blob object should be released at this point. |
+*/ |
+static int rtreeEndTransaction(sqlite3_vtab *pVtab){ |
+ Rtree *pRtree = (Rtree *)pVtab; |
+ pRtree->inWrTrans = 0; |
+ nodeBlobReset(pRtree); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The xRename method for rtree module virtual tables. |
+*/ |
+static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ |
+ Rtree *pRtree = (Rtree *)pVtab; |
+ int rc = SQLITE_NOMEM; |
+ char *zSql = sqlite3_mprintf( |
+ "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" |
+ "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" |
+ "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" |
+ , pRtree->zDb, pRtree->zName, zNewName |
+ , pRtree->zDb, pRtree->zName, zNewName |
+ , pRtree->zDb, pRtree->zName, zNewName |
+ ); |
+ if( zSql ){ |
+ rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); |
+ sqlite3_free(zSql); |
+ } |
+ return rc; |
+} |
+ |
+ |
+/* |
+** This function populates the pRtree->nRowEst variable with an estimate |
+** of the number of rows in the virtual table. If possible, this is based |
+** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. |
+*/ |
+static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ |
+ const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; |
+ char *zSql; |
+ sqlite3_stmt *p; |
+ int rc; |
+ i64 nRow = 0; |
+ |
+ rc = sqlite3_table_column_metadata( |
+ db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 |
+ ); |
+ if( rc!=SQLITE_OK ){ |
+ pRtree->nRowEst = RTREE_DEFAULT_ROWEST; |
+ return rc==SQLITE_ERROR ? SQLITE_OK : rc; |
+ } |
+ zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); |
+ if( zSql==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); |
+ if( rc==SQLITE_OK ){ |
+ if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); |
+ rc = sqlite3_finalize(p); |
+ }else if( rc!=SQLITE_NOMEM ){ |
+ rc = SQLITE_OK; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ if( nRow==0 ){ |
+ pRtree->nRowEst = RTREE_DEFAULT_ROWEST; |
+ }else{ |
+ pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); |
+ } |
+ } |
+ sqlite3_free(zSql); |
+ } |
+ |
+ return rc; |
+} |
+ |
+static sqlite3_module rtreeModule = { |
+ 0, /* iVersion */ |
+ rtreeCreate, /* xCreate - create a table */ |
+ rtreeConnect, /* xConnect - connect to an existing table */ |
+ rtreeBestIndex, /* xBestIndex - Determine search strategy */ |
+ rtreeDisconnect, /* xDisconnect - Disconnect from a table */ |
+ rtreeDestroy, /* xDestroy - Drop a table */ |
+ rtreeOpen, /* xOpen - open a cursor */ |
+ rtreeClose, /* xClose - close a cursor */ |
+ rtreeFilter, /* xFilter - configure scan constraints */ |
+ rtreeNext, /* xNext - advance a cursor */ |
+ rtreeEof, /* xEof */ |
+ rtreeColumn, /* xColumn - read data */ |
+ rtreeRowid, /* xRowid - read data */ |
+ rtreeUpdate, /* xUpdate - write data */ |
+ rtreeBeginTransaction, /* xBegin - begin transaction */ |
+ rtreeEndTransaction, /* xSync - sync transaction */ |
+ rtreeEndTransaction, /* xCommit - commit transaction */ |
+ rtreeEndTransaction, /* xRollback - rollback transaction */ |
+ 0, /* xFindFunction - function overloading */ |
+ rtreeRename, /* xRename - rename the table */ |
+ 0, /* xSavepoint */ |
+ 0, /* xRelease */ |
+ 0, /* xRollbackTo */ |
+}; |
+ |
+static int rtreeSqlInit( |
+ Rtree *pRtree, |
+ sqlite3 *db, |
+ const char *zDb, |
+ const char *zPrefix, |
+ int isCreate |
+){ |
+ int rc = SQLITE_OK; |
+ |
+ #define N_STATEMENT 8 |
+ static const char *azSql[N_STATEMENT] = { |
+ /* Write the xxx_node table */ |
+ "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", |
+ "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", |
+ |
+ /* Read and write the xxx_rowid table */ |
+ "SELECT nodeno FROM '%q'.'%q_rowid' WHERE rowid = :1", |
+ "INSERT OR REPLACE INTO '%q'.'%q_rowid' VALUES(:1, :2)", |
+ "DELETE FROM '%q'.'%q_rowid' WHERE rowid = :1", |
+ |
+ /* Read and write the xxx_parent table */ |
+ "SELECT parentnode FROM '%q'.'%q_parent' WHERE nodeno = :1", |
+ "INSERT OR REPLACE INTO '%q'.'%q_parent' VALUES(:1, :2)", |
+ "DELETE FROM '%q'.'%q_parent' WHERE nodeno = :1" |
+ }; |
+ sqlite3_stmt **appStmt[N_STATEMENT]; |
+ int i; |
+ |
+ pRtree->db = db; |
+ |
+ if( isCreate ){ |
+ char *zCreate = sqlite3_mprintf( |
+"CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" |
+"CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" |
+"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," |
+ " parentnode INTEGER);" |
+"INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", |
+ zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize |
+ ); |
+ if( !zCreate ){ |
+ return SQLITE_NOMEM; |
+ } |
+ rc = sqlite3_exec(db, zCreate, 0, 0, 0); |
+ sqlite3_free(zCreate); |
+ if( rc!=SQLITE_OK ){ |
+ return rc; |
+ } |
+ } |
+ |
+ appStmt[0] = &pRtree->pWriteNode; |
+ appStmt[1] = &pRtree->pDeleteNode; |
+ appStmt[2] = &pRtree->pReadRowid; |
+ appStmt[3] = &pRtree->pWriteRowid; |
+ appStmt[4] = &pRtree->pDeleteRowid; |
+ appStmt[5] = &pRtree->pReadParent; |
+ appStmt[6] = &pRtree->pWriteParent; |
+ appStmt[7] = &pRtree->pDeleteParent; |
+ |
+ rc = rtreeQueryStat1(db, pRtree); |
+ for(i=0; i<N_STATEMENT && rc==SQLITE_OK; i++){ |
+ char *zSql = sqlite3_mprintf(azSql[i], zDb, zPrefix); |
+ if( zSql ){ |
+ rc = sqlite3_prepare_v2(db, zSql, -1, appStmt[i], 0); |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ sqlite3_free(zSql); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** The second argument to this function contains the text of an SQL statement |
+** that returns a single integer value. The statement is compiled and executed |
+** using database connection db. If successful, the integer value returned |
+** is written to *piVal and SQLITE_OK returned. Otherwise, an SQLite error |
+** code is returned and the value of *piVal after returning is not defined. |
+*/ |
+static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ |
+ int rc = SQLITE_NOMEM; |
+ if( zSql ){ |
+ sqlite3_stmt *pStmt = 0; |
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); |
+ if( rc==SQLITE_OK ){ |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ *piVal = sqlite3_column_int(pStmt, 0); |
+ } |
+ rc = sqlite3_finalize(pStmt); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This function is called from within the xConnect() or xCreate() method to |
+** determine the node-size used by the rtree table being created or connected |
+** to. If successful, pRtree->iNodeSize is populated and SQLITE_OK returned. |
+** Otherwise, an SQLite error code is returned. |
+** |
+** If this function is being called as part of an xConnect(), then the rtree |
+** table already exists. In this case the node-size is determined by inspecting |
+** the root node of the tree. |
+** |
+** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. |
+** This ensures that each node is stored on a single database page. If the |
+** database page-size is so large that more than RTREE_MAXCELLS entries |
+** would fit in a single node, use a smaller node-size. |
+*/ |
+static int getNodeSize( |
+ sqlite3 *db, /* Database handle */ |
+ Rtree *pRtree, /* Rtree handle */ |
+ int isCreate, /* True for xCreate, false for xConnect */ |
+ char **pzErr /* OUT: Error message, if any */ |
+){ |
+ int rc; |
+ char *zSql; |
+ if( isCreate ){ |
+ int iPageSize = 0; |
+ zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); |
+ rc = getIntFromStmt(db, zSql, &iPageSize); |
+ if( rc==SQLITE_OK ){ |
+ pRtree->iNodeSize = iPageSize-64; |
+ if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)<pRtree->iNodeSize ){ |
+ pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; |
+ } |
+ }else{ |
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); |
+ } |
+ }else{ |
+ zSql = sqlite3_mprintf( |
+ "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", |
+ pRtree->zDb, pRtree->zName |
+ ); |
+ rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); |
+ if( rc!=SQLITE_OK ){ |
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); |
+ } |
+ } |
+ |
+ sqlite3_free(zSql); |
+ return rc; |
+} |
+ |
+/* |
+** This function is the implementation of both the xConnect and xCreate |
+** methods of the r-tree virtual table. |
+** |
+** argv[0] -> module name |
+** argv[1] -> database name |
+** argv[2] -> table name |
+** argv[...] -> column names... |
+*/ |
+static int rtreeInit( |
+ sqlite3 *db, /* Database connection */ |
+ void *pAux, /* One of the RTREE_COORD_* constants */ |
+ int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ |
+ sqlite3_vtab **ppVtab, /* OUT: New virtual table */ |
+ char **pzErr, /* OUT: Error message, if any */ |
+ int isCreate /* True for xCreate, false for xConnect */ |
+){ |
+ int rc = SQLITE_OK; |
+ Rtree *pRtree; |
+ int nDb; /* Length of string argv[1] */ |
+ int nName; /* Length of string argv[2] */ |
+ int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); |
+ |
+ const char *aErrMsg[] = { |
+ 0, /* 0 */ |
+ "Wrong number of columns for an rtree table", /* 1 */ |
+ "Too few columns for an rtree table", /* 2 */ |
+ "Too many columns for an rtree table" /* 3 */ |
+ }; |
+ |
+ int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2; |
+ if( aErrMsg[iErr] ){ |
+ *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); |
+ return SQLITE_ERROR; |
+ } |
+ |
+ sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); |
+ |
+ /* Allocate the sqlite3_vtab structure */ |
+ nDb = (int)strlen(argv[1]); |
+ nName = (int)strlen(argv[2]); |
+ pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); |
+ if( !pRtree ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); |
+ pRtree->nBusy = 1; |
+ pRtree->base.pModule = &rtreeModule; |
+ pRtree->zDb = (char *)&pRtree[1]; |
+ pRtree->zName = &pRtree->zDb[nDb+1]; |
+ pRtree->nDim = (u8)((argc-4)/2); |
+ pRtree->nDim2 = pRtree->nDim*2; |
+ pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; |
+ pRtree->eCoordType = (u8)eCoordType; |
+ memcpy(pRtree->zDb, argv[1], nDb); |
+ memcpy(pRtree->zName, argv[2], nName); |
+ |
+ /* Figure out the node size to use. */ |
+ rc = getNodeSize(db, pRtree, isCreate, pzErr); |
+ |
+ /* Create/Connect to the underlying relational database schema. If |
+ ** that is successful, call sqlite3_declare_vtab() to configure |
+ ** the r-tree table schema. |
+ */ |
+ if( rc==SQLITE_OK ){ |
+ if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){ |
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); |
+ }else{ |
+ char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]); |
+ char *zTmp; |
+ int ii; |
+ for(ii=4; zSql && ii<argc; ii++){ |
+ zTmp = zSql; |
+ zSql = sqlite3_mprintf("%s, %s", zTmp, argv[ii]); |
+ sqlite3_free(zTmp); |
+ } |
+ if( zSql ){ |
+ zTmp = zSql; |
+ zSql = sqlite3_mprintf("%s);", zTmp); |
+ sqlite3_free(zTmp); |
+ } |
+ if( !zSql ){ |
+ rc = SQLITE_NOMEM; |
+ }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ |
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); |
+ } |
+ sqlite3_free(zSql); |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ *ppVtab = (sqlite3_vtab *)pRtree; |
+ }else{ |
+ assert( *ppVtab==0 ); |
+ assert( pRtree->nBusy==1 ); |
+ rtreeRelease(pRtree); |
+ } |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Implementation of a scalar function that decodes r-tree nodes to |
+** human readable strings. This can be used for debugging and analysis. |
+** |
+** The scalar function takes two arguments: (1) the number of dimensions |
+** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing |
+** an r-tree node. For a two-dimensional r-tree structure called "rt", to |
+** deserialize all nodes, a statement like: |
+** |
+** SELECT rtreenode(2, data) FROM rt_node; |
+** |
+** The human readable string takes the form of a Tcl list with one |
+** entry for each cell in the r-tree node. Each entry is itself a |
+** list, containing the 8-byte rowid/pageno followed by the |
+** <num-dimension>*2 coordinates. |
+*/ |
+static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ |
+ char *zText = 0; |
+ RtreeNode node; |
+ Rtree tree; |
+ int ii; |
+ |
+ UNUSED_PARAMETER(nArg); |
+ memset(&node, 0, sizeof(RtreeNode)); |
+ memset(&tree, 0, sizeof(Rtree)); |
+ tree.nDim = (u8)sqlite3_value_int(apArg[0]); |
+ tree.nDim2 = tree.nDim*2; |
+ tree.nBytesPerCell = 8 + 8 * tree.nDim; |
+ node.zData = (u8 *)sqlite3_value_blob(apArg[1]); |
+ |
+ for(ii=0; ii<NCELL(&node); ii++){ |
+ char zCell[512]; |
+ int nCell = 0; |
+ RtreeCell cell; |
+ int jj; |
+ |
+ nodeGetCell(&tree, &node, ii, &cell); |
+ sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); |
+ nCell = (int)strlen(zCell); |
+ for(jj=0; jj<tree.nDim2; jj++){ |
+#ifndef SQLITE_RTREE_INT_ONLY |
+ sqlite3_snprintf(512-nCell,&zCell[nCell], " %g", |
+ (double)cell.aCoord[jj].f); |
+#else |
+ sqlite3_snprintf(512-nCell,&zCell[nCell], " %d", |
+ cell.aCoord[jj].i); |
+#endif |
+ nCell = (int)strlen(zCell); |
+ } |
+ |
+ if( zText ){ |
+ char *zTextNew = sqlite3_mprintf("%s {%s}", zText, zCell); |
+ sqlite3_free(zText); |
+ zText = zTextNew; |
+ }else{ |
+ zText = sqlite3_mprintf("{%s}", zCell); |
+ } |
+ } |
+ |
+ sqlite3_result_text(ctx, zText, -1, sqlite3_free); |
+} |
+ |
+/* This routine implements an SQL function that returns the "depth" parameter |
+** from the front of a blob that is an r-tree node. For example: |
+** |
+** SELECT rtreedepth(data) FROM rt_node WHERE nodeno=1; |
+** |
+** The depth value is 0 for all nodes other than the root node, and the root |
+** node always has nodeno=1, so the example above is the primary use for this |
+** routine. This routine is intended for testing and analysis only. |
+*/ |
+static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ |
+ UNUSED_PARAMETER(nArg); |
+ if( sqlite3_value_type(apArg[0])!=SQLITE_BLOB |
+ || sqlite3_value_bytes(apArg[0])<2 |
+ ){ |
+ sqlite3_result_error(ctx, "Invalid argument to rtreedepth()", -1); |
+ }else{ |
+ u8 *zBlob = (u8 *)sqlite3_value_blob(apArg[0]); |
+ sqlite3_result_int(ctx, readInt16(zBlob)); |
+ } |
+} |
+ |
+/* |
+** Register the r-tree module with database handle db. This creates the |
+** virtual table module "rtree" and the debugging/analysis scalar |
+** function "rtreenode". |
+*/ |
+SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ |
+ const int utf8 = SQLITE_UTF8; |
+ int rc; |
+ |
+ rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+#ifdef SQLITE_RTREE_INT_ONLY |
+ void *c = (void *)RTREE_COORD_INT32; |
+#else |
+ void *c = (void *)RTREE_COORD_REAL32; |
+#endif |
+ rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ void *c = (void *)RTREE_COORD_INT32; |
+ rc = sqlite3_create_module_v2(db, "rtree_i32", &rtreeModule, c, 0); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This routine deletes the RtreeGeomCallback object that was attached |
+** one of the SQL functions create by sqlite3_rtree_geometry_callback() |
+** or sqlite3_rtree_query_callback(). In other words, this routine is the |
+** destructor for an RtreeGeomCallback objecct. This routine is called when |
+** the corresponding SQL function is deleted. |
+*/ |
+static void rtreeFreeCallback(void *p){ |
+ RtreeGeomCallback *pInfo = (RtreeGeomCallback*)p; |
+ if( pInfo->xDestructor ) pInfo->xDestructor(pInfo->pContext); |
+ sqlite3_free(p); |
+} |
+ |
+/* |
+** This routine frees the BLOB that is returned by geomCallback(). |
+*/ |
+static void rtreeMatchArgFree(void *pArg){ |
+ int i; |
+ RtreeMatchArg *p = (RtreeMatchArg*)pArg; |
+ for(i=0; i<p->nParam; i++){ |
+ sqlite3_value_free(p->apSqlParam[i]); |
+ } |
+ sqlite3_free(p); |
+} |
+ |
+/* |
+** Each call to sqlite3_rtree_geometry_callback() or |
+** sqlite3_rtree_query_callback() creates an ordinary SQLite |
+** scalar function that is implemented by this routine. |
+** |
+** All this function does is construct an RtreeMatchArg object that |
+** contains the geometry-checking callback routines and a list of |
+** parameters to this function, then return that RtreeMatchArg object |
+** as a BLOB. |
+** |
+** The R-Tree MATCH operator will read the returned BLOB, deserialize |
+** the RtreeMatchArg object, and use the RtreeMatchArg object to figure |
+** out which elements of the R-Tree should be returned by the query. |
+*/ |
+static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ |
+ RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); |
+ RtreeMatchArg *pBlob; |
+ int nBlob; |
+ int memErr = 0; |
+ |
+ nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) |
+ + nArg*sizeof(sqlite3_value*); |
+ pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); |
+ if( !pBlob ){ |
+ sqlite3_result_error_nomem(ctx); |
+ }else{ |
+ int i; |
+ pBlob->magic = RTREE_GEOMETRY_MAGIC; |
+ pBlob->cb = pGeomCtx[0]; |
+ pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; |
+ pBlob->nParam = nArg; |
+ for(i=0; i<nArg; i++){ |
+ pBlob->apSqlParam[i] = sqlite3_value_dup(aArg[i]); |
+ if( pBlob->apSqlParam[i]==0 ) memErr = 1; |
+#ifdef SQLITE_RTREE_INT_ONLY |
+ pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); |
+#else |
+ pBlob->aParam[i] = sqlite3_value_double(aArg[i]); |
+#endif |
+ } |
+ if( memErr ){ |
+ sqlite3_result_error_nomem(ctx); |
+ rtreeMatchArgFree(pBlob); |
+ }else{ |
+ sqlite3_result_blob(ctx, pBlob, nBlob, rtreeMatchArgFree); |
+ } |
+ } |
+} |
+ |
+/* |
+** Register a new geometry function for use with the r-tree MATCH operator. |
+*/ |
+SQLITE_API int sqlite3_rtree_geometry_callback( |
+ sqlite3 *db, /* Register SQL function on this connection */ |
+ const char *zGeom, /* Name of the new SQL function */ |
+ int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ |
+ void *pContext /* Extra data associated with the callback */ |
+){ |
+ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ |
+ |
+ /* Allocate and populate the context object. */ |
+ pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); |
+ if( !pGeomCtx ) return SQLITE_NOMEM; |
+ pGeomCtx->xGeom = xGeom; |
+ pGeomCtx->xQueryFunc = 0; |
+ pGeomCtx->xDestructor = 0; |
+ pGeomCtx->pContext = pContext; |
+ return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, |
+ (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback |
+ ); |
+} |
+ |
+/* |
+** Register a new 2nd-generation geometry function for use with the |
+** r-tree MATCH operator. |
+*/ |
+SQLITE_API int sqlite3_rtree_query_callback( |
+ sqlite3 *db, /* Register SQL function on this connection */ |
+ const char *zQueryFunc, /* Name of new SQL function */ |
+ int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ |
+ void *pContext, /* Extra data passed into the callback */ |
+ void (*xDestructor)(void*) /* Destructor for the extra data */ |
+){ |
+ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ |
+ |
+ /* Allocate and populate the context object. */ |
+ pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); |
+ if( !pGeomCtx ) return SQLITE_NOMEM; |
+ pGeomCtx->xGeom = 0; |
+ pGeomCtx->xQueryFunc = xQueryFunc; |
+ pGeomCtx->xDestructor = xDestructor; |
+ pGeomCtx->pContext = pContext; |
+ return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, |
+ (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback |
+ ); |
+} |
+ |
+#if !SQLITE_CORE |
+#ifdef _WIN32 |
+__declspec(dllexport) |
+#endif |
+SQLITE_API int sqlite3_rtree_init( |
+ sqlite3 *db, |
+ char **pzErrMsg, |
+ const sqlite3_api_routines *pApi |
+){ |
+ SQLITE_EXTENSION_INIT2(pApi) |
+ return sqlite3RtreeInit(db); |
+} |
+#endif |
+ |
+#endif |
+ |
+/************** End of rtree.c ***********************************************/ |
+/************** Begin file icu.c *********************************************/ |
+/* |
+** 2007 May 6 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+************************************************************************* |
+** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ |
+** |
+** This file implements an integration between the ICU library |
+** ("International Components for Unicode", an open-source library |
+** for handling unicode data) and SQLite. The integration uses |
+** ICU to provide the following to SQLite: |
+** |
+** * An implementation of the SQL regexp() function (and hence REGEXP |
+** operator) using the ICU uregex_XX() APIs. |
+** |
+** * Implementations of the SQL scalar upper() and lower() functions |
+** for case mapping. |
+** |
+** * Integration of ICU and SQLite collation sequences. |
+** |
+** * An implementation of the LIKE operator that uses ICU to |
+** provide case-independent matching. |
+*/ |
+ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) |
+ |
+/* Include ICU headers */ |
+#include <unicode/utypes.h> |
+#include <unicode/uregex.h> |
+#include <unicode/ustring.h> |
+#include <unicode/ucol.h> |
+ |
+/* #include <assert.h> */ |
+ |
+#ifndef SQLITE_CORE |
+/* #include "sqlite3ext.h" */ |
+ SQLITE_EXTENSION_INIT1 |
+#else |
+/* #include "sqlite3.h" */ |
+#endif |
+ |
+/* |
+** Maximum length (in bytes) of the pattern in a LIKE or GLOB |
+** operator. |
+*/ |
+#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH |
+# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 |
+#endif |
+ |
+/* |
+** Version of sqlite3_free() that is always a function, never a macro. |
+*/ |
+static void xFree(void *p){ |
+ sqlite3_free(p); |
+} |
+ |
+/* |
+** This lookup table is used to help decode the first byte of |
+** a multi-byte UTF8 character. It is copied here from SQLite source |
+** code file utf8.c. |
+*/ |
+static const unsigned char icuUtf8Trans1[] = { |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, |
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, |
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, |
+}; |
+ |
+#define SQLITE_ICU_READ_UTF8(zIn, c) \ |
+ c = *(zIn++); \ |
+ if( c>=0xc0 ){ \ |
+ c = icuUtf8Trans1[c-0xc0]; \ |
+ while( (*zIn & 0xc0)==0x80 ){ \ |
+ c = (c<<6) + (0x3f & *(zIn++)); \ |
+ } \ |
+ } |
+ |
+#define SQLITE_ICU_SKIP_UTF8(zIn) \ |
+ assert( *zIn ); \ |
+ if( *(zIn++)>=0xc0 ){ \ |
+ while( (*zIn & 0xc0)==0x80 ){zIn++;} \ |
+ } |
+ |
+ |
+/* |
+** Compare two UTF-8 strings for equality where the first string is |
+** a "LIKE" expression. Return true (1) if they are the same and |
+** false (0) if they are different. |
+*/ |
+static int icuLikeCompare( |
+ const uint8_t *zPattern, /* LIKE pattern */ |
+ const uint8_t *zString, /* The UTF-8 string to compare against */ |
+ const UChar32 uEsc /* The escape character */ |
+){ |
+ static const int MATCH_ONE = (UChar32)'_'; |
+ static const int MATCH_ALL = (UChar32)'%'; |
+ |
+ int prevEscape = 0; /* True if the previous character was uEsc */ |
+ |
+ while( 1 ){ |
+ |
+ /* Read (and consume) the next character from the input pattern. */ |
+ UChar32 uPattern; |
+ SQLITE_ICU_READ_UTF8(zPattern, uPattern); |
+ if( uPattern==0 ) break; |
+ |
+ /* There are now 4 possibilities: |
+ ** |
+ ** 1. uPattern is an unescaped match-all character "%", |
+ ** 2. uPattern is an unescaped match-one character "_", |
+ ** 3. uPattern is an unescaped escape character, or |
+ ** 4. uPattern is to be handled as an ordinary character |
+ */ |
+ if( !prevEscape && uPattern==MATCH_ALL ){ |
+ /* Case 1. */ |
+ uint8_t c; |
+ |
+ /* Skip any MATCH_ALL or MATCH_ONE characters that follow a |
+ ** MATCH_ALL. For each MATCH_ONE, skip one character in the |
+ ** test string. |
+ */ |
+ while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ |
+ if( c==MATCH_ONE ){ |
+ if( *zString==0 ) return 0; |
+ SQLITE_ICU_SKIP_UTF8(zString); |
+ } |
+ zPattern++; |
+ } |
+ |
+ if( *zPattern==0 ) return 1; |
+ |
+ while( *zString ){ |
+ if( icuLikeCompare(zPattern, zString, uEsc) ){ |
+ return 1; |
+ } |
+ SQLITE_ICU_SKIP_UTF8(zString); |
+ } |
+ return 0; |
+ |
+ }else if( !prevEscape && uPattern==MATCH_ONE ){ |
+ /* Case 2. */ |
+ if( *zString==0 ) return 0; |
+ SQLITE_ICU_SKIP_UTF8(zString); |
+ |
+ }else if( !prevEscape && uPattern==uEsc){ |
+ /* Case 3. */ |
+ prevEscape = 1; |
+ |
+ }else{ |
+ /* Case 4. */ |
+ UChar32 uString; |
+ SQLITE_ICU_READ_UTF8(zString, uString); |
+ uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); |
+ uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); |
+ if( uString!=uPattern ){ |
+ return 0; |
+ } |
+ prevEscape = 0; |
+ } |
+ } |
+ |
+ return *zString==0; |
+} |
+ |
+/* |
+** Implementation of the like() SQL function. This function implements |
+** the build-in LIKE operator. The first argument to the function is the |
+** pattern and the second argument is the string. So, the SQL statements: |
+** |
+** A LIKE B |
+** |
+** is implemented as like(B, A). If there is an escape character E, |
+** |
+** A LIKE B ESCAPE E |
+** |
+** is mapped to like(B, A, E). |
+*/ |
+static void icuLikeFunc( |
+ sqlite3_context *context, |
+ int argc, |
+ sqlite3_value **argv |
+){ |
+ const unsigned char *zA = sqlite3_value_text(argv[0]); |
+ const unsigned char *zB = sqlite3_value_text(argv[1]); |
+ UChar32 uEsc = 0; |
+ |
+ /* Limit the length of the LIKE or GLOB pattern to avoid problems |
+ ** of deep recursion and N*N behavior in patternCompare(). |
+ */ |
+ if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ |
+ sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); |
+ return; |
+ } |
+ |
+ |
+ if( argc==3 ){ |
+ /* The escape character string must consist of a single UTF-8 character. |
+ ** Otherwise, return an error. |
+ */ |
+ int nE= sqlite3_value_bytes(argv[2]); |
+ const unsigned char *zE = sqlite3_value_text(argv[2]); |
+ int i = 0; |
+ if( zE==0 ) return; |
+ U8_NEXT(zE, i, nE, uEsc); |
+ if( i!=nE){ |
+ sqlite3_result_error(context, |
+ "ESCAPE expression must be a single character", -1); |
+ return; |
+ } |
+ } |
+ |
+ if( zA && zB ){ |
+ sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); |
+ } |
+} |
+ |
+/* |
+** This function is called when an ICU function called from within |
+** the implementation of an SQL scalar function returns an error. |
+** |
+** The scalar function context passed as the first argument is |
+** loaded with an error message based on the following two args. |
+*/ |
+static void icuFunctionError( |
+ sqlite3_context *pCtx, /* SQLite scalar function context */ |
+ const char *zName, /* Name of ICU function that failed */ |
+ UErrorCode e /* Error code returned by ICU function */ |
+){ |
+ char zBuf[128]; |
+ sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); |
+ zBuf[127] = '\0'; |
+ sqlite3_result_error(pCtx, zBuf, -1); |
+} |
+ |
+/* |
+** Function to delete compiled regexp objects. Registered as |
+** a destructor function with sqlite3_set_auxdata(). |
+*/ |
+static void icuRegexpDelete(void *p){ |
+ URegularExpression *pExpr = (URegularExpression *)p; |
+ uregex_close(pExpr); |
+} |
+ |
+/* |
+** Implementation of SQLite REGEXP operator. This scalar function takes |
+** two arguments. The first is a regular expression pattern to compile |
+** the second is a string to match against that pattern. If either |
+** argument is an SQL NULL, then NULL Is returned. Otherwise, the result |
+** is 1 if the string matches the pattern, or 0 otherwise. |
+** |
+** SQLite maps the regexp() function to the regexp() operator such |
+** that the following two are equivalent: |
+** |
+** zString REGEXP zPattern |
+** regexp(zPattern, zString) |
+** |
+** Uses the following ICU regexp APIs: |
+** |
+** uregex_open() |
+** uregex_matches() |
+** uregex_close() |
+*/ |
+static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ |
+ UErrorCode status = U_ZERO_ERROR; |
+ URegularExpression *pExpr; |
+ UBool res; |
+ const UChar *zString = sqlite3_value_text16(apArg[1]); |
+ |
+ (void)nArg; /* Unused parameter */ |
+ |
+ /* If the left hand side of the regexp operator is NULL, |
+ ** then the result is also NULL. |
+ */ |
+ if( !zString ){ |
+ return; |
+ } |
+ |
+ pExpr = sqlite3_get_auxdata(p, 0); |
+ if( !pExpr ){ |
+ const UChar *zPattern = sqlite3_value_text16(apArg[0]); |
+ if( !zPattern ){ |
+ return; |
+ } |
+ pExpr = uregex_open(zPattern, -1, 0, 0, &status); |
+ |
+ if( U_SUCCESS(status) ){ |
+ sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); |
+ }else{ |
+ assert(!pExpr); |
+ icuFunctionError(p, "uregex_open", status); |
+ return; |
+ } |
+ } |
+ |
+ /* Configure the text that the regular expression operates on. */ |
+ uregex_setText(pExpr, zString, -1, &status); |
+ if( !U_SUCCESS(status) ){ |
+ icuFunctionError(p, "uregex_setText", status); |
+ return; |
+ } |
+ |
+ /* Attempt the match */ |
+ res = uregex_matches(pExpr, 0, &status); |
+ if( !U_SUCCESS(status) ){ |
+ icuFunctionError(p, "uregex_matches", status); |
+ return; |
+ } |
+ |
+ /* Set the text that the regular expression operates on to a NULL |
+ ** pointer. This is not really necessary, but it is tidier than |
+ ** leaving the regular expression object configured with an invalid |
+ ** pointer after this function returns. |
+ */ |
+ uregex_setText(pExpr, 0, 0, &status); |
+ |
+ /* Return 1 or 0. */ |
+ sqlite3_result_int(p, res ? 1 : 0); |
+} |
+ |
+/* |
+** Implementations of scalar functions for case mapping - upper() and |
+** lower(). Function upper() converts its input to upper-case (ABC). |
+** Function lower() converts to lower-case (abc). |
+** |
+** ICU provides two types of case mapping, "general" case mapping and |
+** "language specific". Refer to ICU documentation for the differences |
+** between the two. |
+** |
+** To utilise "general" case mapping, the upper() or lower() scalar |
+** functions are invoked with one argument: |
+** |
+** upper('ABC') -> 'abc' |
+** lower('abc') -> 'ABC' |
+** |
+** To access ICU "language specific" case mapping, upper() or lower() |
+** should be invoked with two arguments. The second argument is the name |
+** of the locale to use. Passing an empty string ("") or SQL NULL value |
+** as the second argument is the same as invoking the 1 argument version |
+** of upper() or lower(). |
+** |
+** lower('I', 'en_us') -> 'i' |
+** lower('I', 'tr_tr') -> '\u131' (small dotless i) |
+** |
+** http://www.icu-project.org/userguide/posix.html#case_mappings |
+*/ |
+static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ |
+ const UChar *zInput; /* Pointer to input string */ |
+ UChar *zOutput = 0; /* Pointer to output buffer */ |
+ int nInput; /* Size of utf-16 input string in bytes */ |
+ int nOut; /* Size of output buffer in bytes */ |
+ int cnt; |
+ int bToUpper; /* True for toupper(), false for tolower() */ |
+ UErrorCode status; |
+ const char *zLocale = 0; |
+ |
+ assert(nArg==1 || nArg==2); |
+ bToUpper = (sqlite3_user_data(p)!=0); |
+ if( nArg==2 ){ |
+ zLocale = (const char *)sqlite3_value_text(apArg[1]); |
+ } |
+ |
+ zInput = sqlite3_value_text16(apArg[0]); |
+ if( !zInput ){ |
+ return; |
+ } |
+ nOut = nInput = sqlite3_value_bytes16(apArg[0]); |
+ if( nOut==0 ){ |
+ sqlite3_result_text16(p, "", 0, SQLITE_STATIC); |
+ return; |
+ } |
+ |
+ for(cnt=0; cnt<2; cnt++){ |
+ UChar *zNew = sqlite3_realloc(zOutput, nOut); |
+ if( zNew==0 ){ |
+ sqlite3_free(zOutput); |
+ sqlite3_result_error_nomem(p); |
+ return; |
+ } |
+ zOutput = zNew; |
+ status = U_ZERO_ERROR; |
+ if( bToUpper ){ |
+ nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); |
+ }else{ |
+ nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); |
+ } |
+ |
+ if( U_SUCCESS(status) ){ |
+ sqlite3_result_text16(p, zOutput, nOut, xFree); |
+ }else if( status==U_BUFFER_OVERFLOW_ERROR ){ |
+ assert( cnt==0 ); |
+ continue; |
+ }else{ |
+ icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); |
+ } |
+ return; |
+ } |
+ assert( 0 ); /* Unreachable */ |
+} |
+ |
+/* |
+** Collation sequence destructor function. The pCtx argument points to |
+** a UCollator structure previously allocated using ucol_open(). |
+*/ |
+static void icuCollationDel(void *pCtx){ |
+ UCollator *p = (UCollator *)pCtx; |
+ ucol_close(p); |
+} |
+ |
+/* |
+** Collation sequence comparison function. The pCtx argument points to |
+** a UCollator structure previously allocated using ucol_open(). |
+*/ |
+static int icuCollationColl( |
+ void *pCtx, |
+ int nLeft, |
+ const void *zLeft, |
+ int nRight, |
+ const void *zRight |
+){ |
+ UCollationResult res; |
+ UCollator *p = (UCollator *)pCtx; |
+ res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); |
+ switch( res ){ |
+ case UCOL_LESS: return -1; |
+ case UCOL_GREATER: return +1; |
+ case UCOL_EQUAL: return 0; |
+ } |
+ assert(!"Unexpected return value from ucol_strcoll()"); |
+ return 0; |
+} |
+ |
+/* |
+** Implementation of the scalar function icu_load_collation(). |
+** |
+** This scalar function is used to add ICU collation based collation |
+** types to an SQLite database connection. It is intended to be called |
+** as follows: |
+** |
+** SELECT icu_load_collation(<locale>, <collation-name>); |
+** |
+** Where <locale> is a string containing an ICU locale identifier (i.e. |
+** "en_AU", "tr_TR" etc.) and <collation-name> is the name of the |
+** collation sequence to create. |
+*/ |
+static void icuLoadCollation( |
+ sqlite3_context *p, |
+ int nArg, |
+ sqlite3_value **apArg |
+){ |
+ sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); |
+ UErrorCode status = U_ZERO_ERROR; |
+ const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ |
+ const char *zName; /* SQL Collation sequence name (eg. "japanese") */ |
+ UCollator *pUCollator; /* ICU library collation object */ |
+ int rc; /* Return code from sqlite3_create_collation_x() */ |
+ |
+ assert(nArg==2); |
+ (void)nArg; /* Unused parameter */ |
+ zLocale = (const char *)sqlite3_value_text(apArg[0]); |
+ zName = (const char *)sqlite3_value_text(apArg[1]); |
+ |
+ if( !zLocale || !zName ){ |
+ return; |
+ } |
+ |
+ pUCollator = ucol_open(zLocale, &status); |
+ if( !U_SUCCESS(status) ){ |
+ icuFunctionError(p, "ucol_open", status); |
+ return; |
+ } |
+ assert(p); |
+ |
+ rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, |
+ icuCollationColl, icuCollationDel |
+ ); |
+ if( rc!=SQLITE_OK ){ |
+ ucol_close(pUCollator); |
+ sqlite3_result_error(p, "Error registering collation function", -1); |
+ } |
+} |
+ |
+/* |
+** Register the ICU extension functions with database db. |
+*/ |
+SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ |
+ static const struct IcuScalar { |
+ const char *zName; /* Function name */ |
+ unsigned char nArg; /* Number of arguments */ |
+ unsigned short enc; /* Optimal text encoding */ |
+ unsigned char iContext; /* sqlite3_user_data() context */ |
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); |
+ } scalars[] = { |
+ {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, |
+ {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, |
+ {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, |
+ {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, |
+ {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, |
+ {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, |
+ {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, |
+ {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, |
+ {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, |
+ {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, |
+ {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, |
+ {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, |
+ }; |
+ int rc = SQLITE_OK; |
+ int i; |
+ |
+ |
+ for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ |
+ const struct IcuScalar *p = &scalars[i]; |
+ rc = sqlite3_create_function( |
+ db, p->zName, p->nArg, p->enc, |
+ p->iContext ? (void*)db : (void*)0, |
+ p->xFunc, 0, 0 |
+ ); |
+ } |
+ |
+ return rc; |
+} |
+ |
+#if !SQLITE_CORE |
+#ifdef _WIN32 |
+__declspec(dllexport) |
+#endif |
+SQLITE_API int sqlite3_icu_init( |
+ sqlite3 *db, |
+ char **pzErrMsg, |
+ const sqlite3_api_routines *pApi |
+){ |
+ SQLITE_EXTENSION_INIT2(pApi) |
+ return sqlite3IcuInit(db); |
+} |
+#endif |
+ |
+#endif |
+ |
+/************** End of icu.c *************************************************/ |
+/************** Begin file fts3_icu.c ****************************************/ |
+/* |
+** 2007 June 22 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+************************************************************************* |
+** This file implements a tokenizer for fts3 based on the ICU library. |
+*/ |
+/* #include "fts3Int.h" */ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
+#ifdef SQLITE_ENABLE_ICU |
+ |
+/* #include <assert.h> */ |
+/* #include <string.h> */ |
+/* #include "fts3_tokenizer.h" */ |
+ |
+#include <unicode/ubrk.h> |
+/* #include <unicode/ucol.h> */ |
+/* #include <unicode/ustring.h> */ |
+#include <unicode/utf16.h> |
+ |
+typedef struct IcuTokenizer IcuTokenizer; |
+typedef struct IcuCursor IcuCursor; |
+ |
+struct IcuTokenizer { |
+ sqlite3_tokenizer base; |
+ char *zLocale; |
+}; |
+ |
+struct IcuCursor { |
+ sqlite3_tokenizer_cursor base; |
+ |
+ UBreakIterator *pIter; /* ICU break-iterator object */ |
+ int nChar; /* Number of UChar elements in pInput */ |
+ UChar *aChar; /* Copy of input using utf-16 encoding */ |
+ int *aOffset; /* Offsets of each character in utf-8 input */ |
+ |
+ int nBuffer; |
+ char *zBuffer; |
+ |
+ int iToken; |
+}; |
+ |
+/* |
+** Create a new tokenizer instance. |
+*/ |
+static int icuCreate( |
+ int argc, /* Number of entries in argv[] */ |
+ const char * const *argv, /* Tokenizer creation arguments */ |
+ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ |
+){ |
+ IcuTokenizer *p; |
+ int n = 0; |
+ |
+ if( argc>0 ){ |
+ n = strlen(argv[0])+1; |
+ } |
+ p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n); |
+ if( !p ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(p, 0, sizeof(IcuTokenizer)); |
+ |
+ if( n ){ |
+ p->zLocale = (char *)&p[1]; |
+ memcpy(p->zLocale, argv[0], n); |
+ } |
+ |
+ *ppTokenizer = (sqlite3_tokenizer *)p; |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Destroy a tokenizer |
+*/ |
+static int icuDestroy(sqlite3_tokenizer *pTokenizer){ |
+ IcuTokenizer *p = (IcuTokenizer *)pTokenizer; |
+ sqlite3_free(p); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Prepare to begin tokenizing a particular string. The input |
+** string to be tokenized is pInput[0..nBytes-1]. A cursor |
+** used to incrementally tokenize this string is returned in |
+** *ppCursor. |
+*/ |
+static int icuOpen( |
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */ |
+ const char *zInput, /* Input string */ |
+ int nInput, /* Length of zInput in bytes */ |
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ |
+){ |
+ IcuTokenizer *p = (IcuTokenizer *)pTokenizer; |
+ IcuCursor *pCsr; |
+ |
+ const int32_t opt = U_FOLD_CASE_DEFAULT; |
+ UErrorCode status = U_ZERO_ERROR; |
+ int nChar; |
+ |
+ UChar32 c; |
+ int iInput = 0; |
+ int iOut = 0; |
+ |
+ *ppCursor = 0; |
+ |
+ if( zInput==0 ){ |
+ nInput = 0; |
+ zInput = ""; |
+ }else if( nInput<0 ){ |
+ nInput = strlen(zInput); |
+ } |
+ nChar = nInput+1; |
+ pCsr = (IcuCursor *)sqlite3_malloc( |
+ sizeof(IcuCursor) + /* IcuCursor */ |
+ ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ |
+ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ |
+ ); |
+ if( !pCsr ){ |
+ return SQLITE_NOMEM; |
+ } |
+ memset(pCsr, 0, sizeof(IcuCursor)); |
+ pCsr->aChar = (UChar *)&pCsr[1]; |
+ pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; |
+ |
+ pCsr->aOffset[iOut] = iInput; |
+ U8_NEXT(zInput, iInput, nInput, c); |
+ while( c>0 ){ |
+ int isError = 0; |
+ c = u_foldCase(c, opt); |
+ U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); |
+ if( isError ){ |
+ sqlite3_free(pCsr); |
+ return SQLITE_ERROR; |
+ } |
+ pCsr->aOffset[iOut] = iInput; |
+ |
+ if( iInput<nInput ){ |
+ U8_NEXT(zInput, iInput, nInput, c); |
+ }else{ |
+ c = 0; |
+ } |
+ } |
+ |
+ pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); |
+ if( !U_SUCCESS(status) ){ |
+ sqlite3_free(pCsr); |
+ return SQLITE_ERROR; |
+ } |
+ pCsr->nChar = iOut; |
+ |
+ ubrk_first(pCsr->pIter); |
+ *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Close a tokenization cursor previously opened by a call to icuOpen(). |
+*/ |
+static int icuClose(sqlite3_tokenizer_cursor *pCursor){ |
+ IcuCursor *pCsr = (IcuCursor *)pCursor; |
+ ubrk_close(pCsr->pIter); |
+ sqlite3_free(pCsr->zBuffer); |
+ sqlite3_free(pCsr); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Extract the next token from a tokenization cursor. |
+*/ |
+static int icuNext( |
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ |
+ const char **ppToken, /* OUT: *ppToken is the token text */ |
+ int *pnBytes, /* OUT: Number of bytes in token */ |
+ int *piStartOffset, /* OUT: Starting offset of token */ |
+ int *piEndOffset, /* OUT: Ending offset of token */ |
+ int *piPosition /* OUT: Position integer of token */ |
+){ |
+ IcuCursor *pCsr = (IcuCursor *)pCursor; |
+ |
+ int iStart = 0; |
+ int iEnd = 0; |
+ int nByte = 0; |
+ |
+ while( iStart==iEnd ){ |
+ UChar32 c; |
+ |
+ iStart = ubrk_current(pCsr->pIter); |
+ iEnd = ubrk_next(pCsr->pIter); |
+ if( iEnd==UBRK_DONE ){ |
+ return SQLITE_DONE; |
+ } |
+ |
+ while( iStart<iEnd ){ |
+ int iWhite = iStart; |
+ U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c); |
+ if( u_isspace(c) ){ |
+ iStart = iWhite; |
+ }else{ |
+ break; |
+ } |
+ } |
+ assert(iStart<=iEnd); |
+ } |
+ |
+ do { |
+ UErrorCode status = U_ZERO_ERROR; |
+ if( nByte ){ |
+ char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); |
+ if( !zNew ){ |
+ return SQLITE_NOMEM; |
+ } |
+ pCsr->zBuffer = zNew; |
+ pCsr->nBuffer = nByte; |
+ } |
+ |
+ u_strToUTF8( |
+ pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ |
+ &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ |
+ &status /* Output success/failure */ |
+ ); |
+ } while( nByte>pCsr->nBuffer ); |
+ |
+ *ppToken = pCsr->zBuffer; |
+ *pnBytes = nByte; |
+ *piStartOffset = pCsr->aOffset[iStart]; |
+ *piEndOffset = pCsr->aOffset[iEnd]; |
+ *piPosition = pCsr->iToken++; |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The set of routines that implement the simple tokenizer |
+*/ |
+static const sqlite3_tokenizer_module icuTokenizerModule = { |
+ 0, /* iVersion */ |
+ icuCreate, /* xCreate */ |
+ icuDestroy, /* xCreate */ |
+ icuOpen, /* xOpen */ |
+ icuClose, /* xClose */ |
+ icuNext, /* xNext */ |
+ 0, /* xLanguageid */ |
+}; |
+ |
+/* |
+** Set *ppModule to point at the implementation of the ICU tokenizer. |
+*/ |
+SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( |
+ sqlite3_tokenizer_module const**ppModule |
+){ |
+ *ppModule = &icuTokenizerModule; |
+} |
+ |
+#endif /* defined(SQLITE_ENABLE_ICU) */ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
+ |
+/************** End of fts3_icu.c ********************************************/ |
+ |
+/* Chain include. */ |
+#include "sqlite3.08.c" |